import argparse import multiprocessing import pytest from pathlib import Path from concurrent import futures import shutil import sys from typing import NoReturn, Sequence, Optional from typing_extensions import Protocol import subprocess from . import paths, toolchain from .dds import DDSWrapper from .bootstrap import BootstrapMode, get_bootstrap_exe def make_argparser() -> argparse.ArgumentParser: """Create an argument parser for the dds-ci command-line""" parser = argparse.ArgumentParser() parser.add_argument('-B', '--bootstrap-with', help='How are we to obtain a bootstrapped DDS executable?', metavar='{download,build,skip,lazy}', type=BootstrapMode, default=BootstrapMode.Lazy) parser.add_argument('--rapid', help='Run CI for fast development iterations', action='store_true') parser.add_argument('--test-toolchain', '-TT', type=Path, metavar='', help='The toolchain to use for the first build, which will be passed through the tests') parser.add_argument('--main-toolchain', '-T', type=Path, dest='toolchain', metavar='', help='The toolchain to use for the final build') parser.add_argument('--jobs', '-j', type=int, help='Number of parallel jobs to use when building and testing', default=multiprocessing.cpu_count() + 2) parser.add_argument('--clean', action='store_true', help="Don't remove prior build/deps results") parser.add_argument('--no-test', action='store_false', dest='do_test', help='Skip testing and just build the final result') return parser class CommandArguments(Protocol): """ The result of parsing argv with the dds-ci argument parser. """ #: Whether the user wants us to clean result before building clean: bool #: The bootstrap method the user has requested bootstrap_with: BootstrapMode #: The toolchain to use when building the 'dds' executable that will be tested. test_toolchain: Optional[Path] #: The toolchain to use when building the main 'dds' executable to publish toolchain: Optional[Path] #: The maximum number of parallel jobs for build and test jobs: int #: Whether we should run the pytest tests do_test: bool #: Rapid-CI is for 'dds' development purposes rapid: bool def parse_argv(argv: Sequence[str]) -> CommandArguments: """Parse the given dds-ci command-line argument list""" return make_argparser().parse_args(argv) def test_build(dds: DDSWrapper, args: CommandArguments) -> DDSWrapper: """ Execute the build that generates the test-mode executable. Uses the given 'dds' to build the new dds. Returns a DDSWrapper around the generated test executable. """ test_tc = args.test_toolchain or toolchain.get_default_audit_toolchain() print(f'Test build is building with toolchain: {test_tc}') build_dir = paths.BUILD_DIR with toolchain.fixup_toolchain(test_tc) as new_tc: dds.build(toolchain=new_tc, root=paths.PROJECT_ROOT, build_root=build_dir, jobs=args.jobs, timeout=60 * 15) return DDSWrapper(build_dir / ('dds' + paths.EXE_SUFFIX)) def run_pytest(dds: DDSWrapper, args: CommandArguments) -> int: """ Execute pytest, testing against the given 'test_dds' executable. Returns the exit code of pytest. """ basetemp = Path('/tmp/dds-ci') basetemp.mkdir(exist_ok=True, parents=True) return pytest.main([ '-v', '--durations=10', '-n', str(args.jobs), f'--basetemp={basetemp}', f'--dds-exe={dds.path}', f'--junit-xml={paths.BUILD_DIR}/pytest-junit.xml', str(paths.PROJECT_ROOT / 'tests/'), ]) def main_build(dds: DDSWrapper, args: CommandArguments) -> int: """ Execute the main build of dds using the given 'dds' executable to build itself. """ main_tc = args.toolchain or ( # If we are in rapid-dev mode, use the test toolchain, which had audit/debug enabled toolchain.get_default_toolchain() if not args.rapid else toolchain.get_default_audit_toolchain()) print(f'Building with toolchain: {main_tc}') with toolchain.fixup_toolchain(main_tc) as new_tc: try: dds.build(toolchain=new_tc, root=paths.PROJECT_ROOT, build_root=paths.BUILD_DIR, jobs=args.jobs, timeout=60 * 15) except subprocess.CalledProcessError as e: if args.rapid: return e.returncode raise return 0 def ci_with_dds(dds: DDSWrapper, args: CommandArguments) -> int: """ Execute CI using the given prior 'dds' executable. """ if args.clean: dds.clean(build_dir=paths.BUILD_DIR) dds.catalog_json_import(paths.PROJECT_ROOT / 'old-catalog.json') if args.rapid: return main_build(dds, args) pool = futures.ThreadPoolExecutor() test_fut = pool.submit(lambda: 0) if args.do_test: # Build the test executable: test_dds = test_build(dds, args) # Move the generated exe and start tests. We'll start building the main # EXE and don't want to overwrite the test one while the tests are running dds_cp = paths.BUILD_DIR / ('dds.test' + paths.EXE_SUFFIX) test_dds.path.rename(dds_cp) test_dds.path = dds_cp # Workaround: dds doesn't rebuild the test-driver on toolchain changes: shutil.rmtree(paths.BUILD_DIR / '_test-driver') test_fut = pool.submit(lambda: run_pytest(test_dds, args)) main_fut = pool.submit(lambda: main_build(dds, args)) for fut in futures.as_completed({test_fut, main_fut}): if fut.result(): return fut.result() return 0 def main(argv: Sequence[str]) -> int: args = parse_argv(argv) with get_bootstrap_exe(args.bootstrap_with) as f: return ci_with_dds(f, args) def start() -> NoReturn: sys.exit(main(sys.argv[1:])) if __name__ == "__main__": start()