Modify the bootstrap build process to now build newer versions with gcc-9, older with gcc-8, and use the new catalog feature to build deps

tools/bootstrap.py

@@ -2,13 +2,28 @@ import argparse
 from pathlib import Path
 import subprocess
 import os
-from typing import Sequence
+from typing import Sequence, NamedTuple
 import sys
 import shutil
 
+
+class BootstrapPhase(NamedTuple):
+    ref: str
+    nix_compiler: str
+    win_compiler: str
+
+    @property
+    def platform_compiler(self):
+        if os.name == 'nt':
+            return self.win_compiler
+        else:
+            return self.nix_compiler
+
+
 BOOTSTRAP_PHASES = [
-    'bootstrap-p1',
-    'bootstrap-p4',
+    BootstrapPhase('bootstrap-p1', 'g++-8', 'cl.exe'),
+    BootstrapPhase('bootstrap-p4', 'g++-8', 'cl.exe'),
+    BootstrapPhase('release/0.0.1', 'g++-9', 'cl.exe'),
 ]
 
 HERE = Path(__file__).parent.absolute()
@@ -23,32 +38,35 @@ EXE_SUFFIX = '.exe' if os.name == 'nt' else ''
 def _run_quiet(cmd, **kwargs) -> None:
     cmd = [str(s) for s in cmd]
     res = subprocess.run(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        **kwargs,
+    )
     if res.returncode != 0:
         print(f'Subprocess command {cmd} failed '
               f'[{res.returncode}]:\n{res.stdout.decode()}')
         raise subprocess.CalledProcessError(res.returncode, cmd)
 
 
-def _clone_bootstrap_phase(ph: str) -> Path:
-    print(f'Clone revision: {ph}')
-    bts_dir = BOOTSTRAP_BASE_DIR / ph
+def _clone_bootstrap_phase(ref: str) -> Path:
+    print(f'Clone revision: {ref}')
+    bts_dir = BOOTSTRAP_BASE_DIR / ref
     if bts_dir.exists():
         shutil.rmtree(bts_dir)
     _run_quiet([
         'git',
         'clone',
         '--depth=1',
-        f'--branch={ph}',
+        f'--branch={ref}',
         f'file://{PROJECT_ROOT}',
         bts_dir,
     ])
     return bts_dir
 
 
-def _build_bootstrap_phase(ph: str, bts_dir: Path,
-                           args: argparse.Namespace) -> None:
-    print(f'Build revision: {ph} [This may take a moment]')
+def _build_bootstrap_phase(ph: BootstrapPhase, bts_dir: Path) -> None:
+    print(f'Build revision: {ph.ref} [This may take a moment]')
     env = os.environ.copy()
     env['DDS_BOOTSTRAP_PREV_EXE'] = str(PREBUILT_DIR / 'dds')
     _run_quiet(
@@ -56,7 +74,7 @@ def _build_bootstrap_phase(ph: str, bts_dir: Path,
             sys.executable,
             '-u',
             str(bts_dir / 'tools/build.py'),
-            f'--cxx={args.cxx}',
+            f'--cxx={ph.platform_compiler}',
         ],
         env=env,
         cwd=bts_dir,
@@ -76,20 +94,16 @@ def _pull_executable(bts_dir: Path) -> Path:
     return dest
 
 
-def _run_boot_phase(phase: str, args: argparse.Namespace) -> Path:
-    bts_dir = _clone_bootstrap_phase(phase)
-    _build_bootstrap_phase(phase, bts_dir, args)
+def _run_boot_phase(phase: BootstrapPhase) -> Path:
+    bts_dir = _clone_bootstrap_phase(phase.ref)
+    _build_bootstrap_phase(phase, bts_dir)
     return _pull_executable(bts_dir)
 
 
 def main(argv: Sequence[str]) -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        '--cxx', help='The C++ compiler to use for the build', required=True)
-    args = parser.parse_args(argv)
     for idx, phase in enumerate(BOOTSTRAP_PHASES):
         print(f'Bootstrap phase [{idx+1}/{len(BOOTSTRAP_PHASES)}]')
-        exe = _run_boot_phase(phase, args)
+        exe = _run_boot_phase(phase)
 
     print(f'A bootstrapped DDS executable has been generated: {exe}')
     return 0

tools/ci.py

@@ -15,7 +15,6 @@ from dds_ci import paths, proc
 
 
 class CIOptions(NamedTuple):
-    cxx: Path
     toolchain: str
     skip_deps: bool
 
@@ -26,7 +25,6 @@ def _do_bootstrap_build(opts: CIOptions) -> None:
         sys.executable,
         '-u',
         str(paths.TOOLS_DIR / 'bootstrap.py'),
-        f'--cxx={opts.cxx}',
     ])
 
 
@@ -67,8 +65,6 @@ def main(argv: Sequence[str]) -> int:
         choices=('download', 'build', 'skip'),
         required=True,
     )
-    parser.add_argument(
-        '--cxx', help='The name/path of the C++ compiler to use.')
     parser.add_argument(
         '--toolchain',
         '-T',
@@ -81,15 +77,9 @@ def main(argv: Sequence[str]) -> int:
         'dependencies. (They must already be present)')
     args = parser.parse_args(argv)
 
-    opts = CIOptions(
-        cxx=Path(args.cxx or 'unspecified'),
-        toolchain=args.toolchain,
-        skip_deps=args.skip_deps)
+    opts = CIOptions(toolchain=args.toolchain, skip_deps=args.skip_deps)
 
     if args.bootstrap_with == 'build':
-        if args.cxx is None:
-            raise RuntimeError(
-                '`--cxx` must be given when using `--bootstrap-with=build`')
         _do_bootstrap_build(opts)
     elif args.bootstrap_with == 'download':
         _do_bootstrap_download()
@@ -98,21 +88,27 @@ def main(argv: Sequence[str]) -> int:
     else:
         assert False, 'impossible'
 
+    cat_path = paths.BUILD_DIR / 'catalog.db'
     ci_repo_dir = paths.BUILD_DIR / '_ci-repo'
-    if not opts.skip_deps:
-        if ci_repo_dir.exists():
-            shutil.rmtree(ci_repo_dir)
-        self_deps_get(paths.PREBUILT_DDS, ci_repo_dir)
-        self_deps_build(paths.PREBUILT_DDS, opts.toolchain, ci_repo_dir,
-                        paths.PROJECT_ROOT / 'remote.dds')
+    if ci_repo_dir.exists():
+        shutil.rmtree(ci_repo_dir)
 
+    proc.check_run([
+        paths.PREBUILT_DDS,
+        'catalog',
+        'import',
+        ('--catalog', cat_path),
+        ('--json', paths.PROJECT_ROOT / 'catalog.json'),
+    ])
     self_build(
         paths.PREBUILT_DDS,
         toolchain=opts.toolchain,
-        dds_flags=['--warnings', '--tests', '--apps'])
+        dds_flags=[
+            ('--catalog', cat_path),
+            ('--repo-dir', ci_repo_dir),
+        ])
     print('Main build PASSED!')
 
-    cat_path = paths.BUILD_DIR / 'catalog.db'
     proc.check_run([
         paths.CUR_BUILT_DDS,
         'catalog',