Merge branch 'release/0.0.1' into develop

azure-pipelines.yml

@@ -12,7 +12,7 @@ jobs:
           echo Executing Build and Tests
           reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1
           python -m pip install pytest pytest-xdist || exit 1
-          python -u tools/ci.py -B download --cxx cl.exe -T tools\msvc.dds || exit 1
+          python -u tools/ci.py -B build -T tools\msvc.dds || exit 1
         displayName: Full CI
       - publish: _build/dds.exe
         artifact: DDS Executable - Windows VS2019
@@ -24,10 +24,10 @@ jobs:
       - script: |
           set -eu
           sudo apt update -y
-          sudo apt install -y python3-minimal g++-9 ccache
+          sudo apt install -y python3-minimal g++-9 g++-8 ccache
           python3 -m pip install pytest pytest-xdist
         displayName: Prepare System
-      - script: python3 -u tools/ci.py -B download -T tools/gcc-9.dds
+      - script: python3 -u tools/ci.py -B build -T tools/gcc-9.dds
         displayName: Full CI
       - publish: _build/dds
         artifact: DDS Executable - Linux
@@ -41,7 +41,7 @@ jobs:
       - script: |
           set -eu
           python3 -m pip install pytest pytest-xdist
-          python3 -u tools/ci.py -B download -T tools/gcc-9.dds
+          python3 -u tools/ci.py -B build -T tools/gcc-9.dds
         displayName: Build and Run Unit Tests
       - publish: _build/dds
         artifact: DDS Executable - macOS

tools/bootstrap.py

@@ -2,13 +2,28 @@ import argparse
 from pathlib import Path
 import subprocess
 import os
-from typing import Sequence
+from typing import Sequence, NamedTuple
 import sys
 import shutil
 
+
+class BootstrapPhase(NamedTuple):
+    ref: str
+    nix_compiler: str
+    win_compiler: str
+
+    @property
+    def platform_compiler(self):
+        if os.name == 'nt':
+            return self.win_compiler
+        else:
+            return self.nix_compiler
+
+
 BOOTSTRAP_PHASES = [
-    'bootstrap-p1',
-    'bootstrap-p4',
+    BootstrapPhase('bootstrap-p1', 'g++-8', 'cl.exe'),
+    BootstrapPhase('bootstrap-p4', 'g++-8', 'cl.exe'),
+    BootstrapPhase('bootstrap-p5', 'g++-9', 'cl.exe'),
 ]
 
 HERE = Path(__file__).parent.absolute()
@@ -23,40 +38,43 @@ EXE_SUFFIX = '.exe' if os.name == 'nt' else ''
 def _run_quiet(cmd, **kwargs) -> None:
     cmd = [str(s) for s in cmd]
     res = subprocess.run(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        **kwargs,
+    )
     if res.returncode != 0:
         print(f'Subprocess command {cmd} failed '
               f'[{res.returncode}]:\n{res.stdout.decode()}')
         raise subprocess.CalledProcessError(res.returncode, cmd)
 
 
-def _clone_bootstrap_phase(ph: str) -> Path:
-    print(f'Clone revision: {ph}')
-    bts_dir = BOOTSTRAP_BASE_DIR / ph
+def _clone_bootstrap_phase(ref: str) -> Path:
+    print(f'Clone revision: {ref}')
+    bts_dir = BOOTSTRAP_BASE_DIR / ref
     if bts_dir.exists():
         shutil.rmtree(bts_dir)
     _run_quiet([
         'git',
         'clone',
         '--depth=1',
-        f'--branch={ph}',
+        f'--branch={ref}',
         f'file://{PROJECT_ROOT}',
         bts_dir,
     ])
     return bts_dir
 
 
-def _build_bootstrap_phase(ph: str, bts_dir: Path,
-                           args: argparse.Namespace) -> None:
-    print(f'Build revision: {ph} [This may take a moment]')
+def _build_bootstrap_phase(ph: BootstrapPhase, bts_dir: Path) -> None:
+    print(f'Build revision: {ph.ref} [This may take a moment]')
     env = os.environ.copy()
-    env['DDS_BOOTSTRAP_PREV_EXE'] = str(PREBUILT_DIR / 'dds')
+    env['DDS_BOOTSTRAP_PREV_EXE'] = str(PREBUILT_DIR / F'dds{EXE_SUFFIX}')
     _run_quiet(
         [
             sys.executable,
             '-u',
             str(bts_dir / 'tools/build.py'),
-            f'--cxx={args.cxx}',
+            f'--cxx={ph.platform_compiler}',
         ],
         env=env,
         cwd=bts_dir,
@@ -76,20 +94,16 @@ def _pull_executable(bts_dir: Path) -> Path:
     return dest
 
 
-def _run_boot_phase(phase: str, args: argparse.Namespace) -> Path:
-    bts_dir = _clone_bootstrap_phase(phase)
-    _build_bootstrap_phase(phase, bts_dir, args)
+def _run_boot_phase(phase: BootstrapPhase) -> Path:
+    bts_dir = _clone_bootstrap_phase(phase.ref)
+    _build_bootstrap_phase(phase, bts_dir)
     return _pull_executable(bts_dir)
 
 
 def main(argv: Sequence[str]) -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        '--cxx', help='The C++ compiler to use for the build', required=True)
-    args = parser.parse_args(argv)
     for idx, phase in enumerate(BOOTSTRAP_PHASES):
         print(f'Bootstrap phase [{idx+1}/{len(BOOTSTRAP_PHASES)}]')
-        exe = _run_boot_phase(phase, args)
+        exe = _run_boot_phase(phase)
 
     print(f'A bootstrapped DDS executable has been generated: {exe}')
     return 0

tools/build.py

@@ -1,14 +1,11 @@
 #!/usr/bin/env python3
 
 import argparse
-from contextlib import contextmanager
 import os
 from pathlib import Path
 from typing import Sequence
-import subprocess
 import sys
 import shutil
-import tempfile
 
 from dds_ci import paths
 from self_build import self_build
@@ -19,40 +16,11 @@ ROOT = Path(__file__).parent.parent.absolute()
 BUILD_DIR = ROOT / '_build'
 
 
-@contextmanager
-def _generate_toolchain(cxx: str):
-    with tempfile.NamedTemporaryFile(
-            suffix='-dds-toolchain.dds', mode='wb', delete=False) as f:
-        comp_id = 'GNU'
-        flags = ''
-        link_flags = ''
-        if cxx in ('cl', 'cl.exe'):
-            comp_id = 'MSVC'
-            flags += '/experimental:preprocessor '
-            link_flags += 'rpcrt4.lib '
-        else:
-            flags += '-fconcepts'
-        flags += ' -DSPDLOG_COMPILED_LIB'
-        content = f'''
-            Compiler-ID: {comp_id}
-            C++-Compiler: {cxx}
-            C++-Version: C++17
-            Debug: True
-            Flags: {flags}
-            Link-Flags: {link_flags}
-        '''
-        print('Using generated toolchain file: ' + content)
-        f.write(content.encode('utf-8'))
-        f.close()
-        yield Path(f.name)
-        os.unlink(f.name)
-
-
 def main(argv: Sequence[str]) -> int:
+    # Prior versions of this script took a --cxx argument, but we don't care anymore
     parser = argparse.ArgumentParser()
-    parser.add_argument(
-        '--cxx', help='Path/name of the C++ compiler to use.', required=True)
-    args = parser.parse_args(argv)
+    parser.add_argument('--cxx', help=argparse.SUPPRESS)
+    parser.parse_args(argv)
 
     dds_bootstrap_env_key = 'DDS_BOOTSTRAP_PREV_EXE'
     if dds_bootstrap_env_key not in os.environ:
@@ -67,10 +35,14 @@ def main(argv: Sequence[str]) -> int:
     print(f'Using previously built DDS executable: {dds_exe}')
     self_deps_get(dds_exe, paths.SELF_TEST_REPO_DIR)
 
-    with _generate_toolchain(args.cxx) as tc_fpath:
-        self_deps_build(dds_exe, tc_fpath, paths.SELF_TEST_REPO_DIR,
-                        ROOT / 'remote.dds')
-        self_build(dds_exe, toolchain=tc_fpath, dds_flags=['--apps'])
+    if os.name == 'nt':
+        tc_fpath = ROOT / 'tools/msvc.dds'
+    else:
+        tc_fpath = ROOT / 'tools/gcc-9.dds'
+
+    self_deps_build(dds_exe, str(tc_fpath), paths.SELF_TEST_REPO_DIR,
+                    ROOT / 'remote.dds')
+    self_build(dds_exe, toolchain=str(tc_fpath), dds_flags=['--apps'])
 
     return 0
 

tools/ci.py

@@ -15,9 +15,7 @@ from dds_ci import paths, proc
 
 
 class CIOptions(NamedTuple):
-    cxx: Path
     toolchain: str
-    skip_deps: bool
 
 
 def _do_bootstrap_build(opts: CIOptions) -> None:
@@ -26,7 +24,6 @@ def _do_bootstrap_build(opts: CIOptions) -> None:
         sys.executable,
         '-u',
         str(paths.TOOLS_DIR / 'bootstrap.py'),
-        f'--cxx={opts.cxx}',
     ])
 
 
@@ -67,29 +64,16 @@ def main(argv: Sequence[str]) -> int:
         choices=('download', 'build', 'skip'),
         required=True,
     )
-    parser.add_argument(
-        '--cxx', help='The name/path of the C++ compiler to use.')
     parser.add_argument(
         '--toolchain',
         '-T',
         help='The toolchain to use for the CI process',
         required=True)
-    parser.add_argument(
-        '--skip-deps',
-        action='store_true',
-        help='If specified, will skip getting and building '
-        'dependencies. (They must already be present)')
     args = parser.parse_args(argv)
 
-    opts = CIOptions(
-        cxx=Path(args.cxx or 'unspecified'),
-        toolchain=args.toolchain,
-        skip_deps=args.skip_deps)
+    opts = CIOptions(toolchain=args.toolchain)
 
     if args.bootstrap_with == 'build':
-        if args.cxx is None:
-            raise RuntimeError(
-                '`--cxx` must be given when using `--bootstrap-with=build`')
         _do_bootstrap_build(opts)
     elif args.bootstrap_with == 'download':
         _do_bootstrap_download()
@@ -98,21 +82,27 @@ def main(argv: Sequence[str]) -> int:
     else:
         assert False, 'impossible'
 
+    cat_path = paths.BUILD_DIR / 'catalog.db'
     ci_repo_dir = paths.BUILD_DIR / '_ci-repo'
-    if not opts.skip_deps:
-        if ci_repo_dir.exists():
-            shutil.rmtree(ci_repo_dir)
-        self_deps_get(paths.PREBUILT_DDS, ci_repo_dir)
-        self_deps_build(paths.PREBUILT_DDS, opts.toolchain, ci_repo_dir,
-                        paths.PROJECT_ROOT / 'remote.dds')
+    if ci_repo_dir.exists():
+        shutil.rmtree(ci_repo_dir)
 
+    proc.check_run([
+        paths.PREBUILT_DDS,
+        'catalog',
+        'import',
+        ('--catalog', cat_path),
+        ('--json', paths.PROJECT_ROOT / 'catalog.json'),
+    ])
     self_build(
         paths.PREBUILT_DDS,
         toolchain=opts.toolchain,
-        dds_flags=['--warnings', '--tests', '--apps'])
+        dds_flags=[
+            ('--catalog', cat_path),
+            ('--repo-dir', ci_repo_dir),
+        ])
     print('Main build PASSED!')
 
-    cat_path = paths.BUILD_DIR / 'catalog.db'
     proc.check_run([
         paths.CUR_BUILT_DDS,
         'catalog',