Merge branch 'feature/ci-download-prebuilt' into develop

azure-pipelines.yml

@@ -10,7 +10,7 @@ jobs:
           echo Loading VS environment
           call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\Common7\\Tools\\vsdevcmd" -arch=x64 || exit 1
           echo Executing Build and Tests
-          python -u tools/ci.py --cxx cl.exe -T tools\\msvc.dds || exit 1
+          python -u tools/ci.py -B download --cxx cl.exe -T tools\\msvc.dds || exit 1
         displayName: Full CI
       - publish: _build/dds.exe
         artifact: DDS Executable - Windows VS2019
@@ -21,7 +21,7 @@ jobs:
     steps:
       - script: sudo apt update -y && sudo apt install -y python3-minimal g++-8
         displayName: Prepare System
-      - script: python3 -u tools/ci.py --cxx g++-8 -T tools/gcc-8.dds
+      - script: python3 -u tools/ci.py -B download --cxx g++-8 -T tools/gcc-8.dds
         displayName: Full CI
       - publish: _build/dds
         artifact: DDS Executable - Linux
@@ -32,7 +32,7 @@ jobs:
     steps:
       - script: brew install gcc@8
         displayName: Prepare System
-      - script: python3 -u tools/ci.py --cxx g++-8 -T tools/gcc-8.dds
+      - script: python3 -u tools/ci.py -B download --cxx g++-8 -T tools/gcc-8.dds
         displayName: Build and Run Unit Tests
       - publish: _build/dds
         artifact: DDS Executable - macOS

tools/bootstrap.py

@@ -20,9 +20,10 @@ PREBUILT_DIR = PROJECT_ROOT / '_prebuilt'
 EXE_SUFFIX = '.exe' if os.name == 'nt' else ''
 
 
-def _run_quiet(args) -> None:
-    cmd = [str(s) for s in args]
-    res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+def _run_quiet(cmd, **kwargs) -> None:
+    cmd = [str(s) for s in cmd]
+    res = subprocess.run(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
     if res.returncode != 0:
         print(f'Subprocess command {cmd} failed '
               f'[{res.returncode}]:\n{res.stdout.decode()}')
@@ -30,7 +31,7 @@ def _run_quiet(args) -> None:
 
 
 def _clone_bootstrap_phase(ph: str) -> Path:
-    print(f'Cloning: {ph}')
+    print(f'Clone revision: {ph}')
     bts_dir = BOOTSTRAP_BASE_DIR / ph
     if bts_dir.exists():
         shutil.rmtree(bts_dir)
@@ -45,11 +46,12 @@ def _clone_bootstrap_phase(ph: str) -> Path:
     return bts_dir
 
 
-def _build_bootstrap_phase(ph: str, bts_dir: Path, args: argparse.Namespace) -> None:
-    print(f'Running build: {ph} (Please wait a moment...)')
+def _build_bootstrap_phase(ph: str, bts_dir: Path,
+                           args: argparse.Namespace) -> None:
+    print(f'Build revision: {ph} [This may take a moment]')
     env = os.environ.copy()
     env['DDS_BOOTSTRAP_PREV_EXE'] = str(PREBUILT_DIR / 'dds')
-    subprocess.check_call(
+    _run_quiet(
         [
             sys.executable,
             '-u',
@@ -84,7 +86,8 @@ def main(argv: Sequence[str]) -> int:
     parser.add_argument(
         '--cxx', help='The C++ compiler to use for the build', required=True)
     args = parser.parse_args(argv)
-    for phase in BOOTSTRAP_PHASES:
+    for idx, phase in enumerate(BOOTSTRAP_PHASES):
+        print(f'Bootstrap phase [{idx+1}/{len(BOOTSTRAP_PHASES)}]')
         exe = _run_boot_phase(phase, args)
 
     print(f'A bootstrapped DDS executable has been generated: {exe}')

tools/ci.py

@@ -1,8 +1,10 @@
 import argparse
+import os
 import sys
 from pathlib import Path
 from typing import Sequence, NamedTuple
 import subprocess
+import urllib.request
 
 HERE = Path(__file__).parent.absolute()
 TOOLS_DIR = HERE
@@ -11,13 +13,12 @@ PREBUILT_DDS = PROJECT_ROOT / '_prebuilt/dds'
 
 
 class CIOptions(NamedTuple):
-    skip_bootstrap: bool
     cxx: Path
     toolchain: str
 
 
-def _do_bootstrap(opts: CIOptions) -> None:
-    print('Running bootstrap')
+def _do_bootstrap_build(opts: CIOptions) -> None:
+    print('Bootstrapping by a local build of prior versions...')
     subprocess.check_call([
         sys.executable,
         '-u',
@@ -26,13 +27,42 @@ def _do_bootstrap(opts: CIOptions) -> None:
     ])
 
 
+def _do_bootstrap_download() -> None:
+    filename = {
+        'win32': 'dds-win-x64.exe',
+        'linux': 'dds-linux-x64',
+        'darwin': 'dds-macos-x64',
+    }.get(sys.platform)
+    if filename is None:
+        raise RuntimeError(f'We do not have a prebuilt DDS binary for the "{sys.platform}" platform')
+    url = f'https://github.com/vector-of-bool/dds/releases/download/bootstrap-p2/{filename}'
+
+    print(f'Downloading prebuilt DDS executable: {url}')
+    stream = urllib.request.urlopen(url)
+    PREBUILT_DDS.parent.mkdir(exist_ok=True, parents=True)
+    with PREBUILT_DDS.open('wb') as fd:
+        while True:
+            buf = stream.read(1024 * 4)
+            if not buf:
+                break
+            fd.write(buf)
+
+    if os.name != 'nt':
+        # Mark the binary executable. By default it won't be
+        mode = PREBUILT_DDS.stat().st_mode
+        mode |= 0b001_001_001
+        PREBUILT_DDS.chmod(mode)
+
+
 def main(argv: Sequence[str]) -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        '--skip-bootstrap',
-        action='store_true',
+        '-B',
+        '--bootstrap-with',
         help=
         'Skip the prebuild-bootstrap step. This requires a _prebuilt/dds to exist!',
+        choices=('download', 'build'),
+        required=True,
     )
     parser.add_argument(
         '--cxx',
@@ -44,12 +74,15 @@ def main(argv: Sequence[str]) -> int:
         help='The toolchain to use for the CI process',
         required=True)
     args = parser.parse_args(argv)
-    opts = CIOptions(
-        skip_bootstrap=args.skip_bootstrap,
-        cxx=Path(args.cxx),
-        toolchain=args.toolchain)
-    if not opts.skip_bootstrap:
-        _do_bootstrap(opts)
+
+    opts = CIOptions(cxx=Path(args.cxx), toolchain=args.toolchain)
+
+    if args.bootstrap_with == 'build':
+        _do_bootstrap_build(opts)
+    elif args.bootstrap_with == 'download':
+        _do_bootstrap_download()
+    else:
+        assert False, 'impossible'
 
     subprocess.check_call([
         str(PREBUILT_DDS),
@@ -66,11 +99,12 @@ def main(argv: Sequence[str]) -> int:
         f'-T{opts.toolchain}',
     ])
 
+    exe_suffix = '.exe' if os.name == 'nt' else ''
     subprocess.check_call([
         sys.executable,
         '-u',
         str(TOOLS_DIR / 'test.py'),
-        f'--exe={PROJECT_ROOT / "_build/dds"}',
+        f'--exe={PROJECT_ROOT / f"_build/dds{exe_suffix}"}',
         f'-T{opts.toolchain}',
     ])
 

tools/msvc.dds

@@ -1,5 +1,5 @@
 Compiler-ID: MSVC
-C++-Version: C++17
-Flags: /experimental:preprocessor /D FMT_HEADER_ONLY=1 /wd5105
+# C++-Version: C++17
+Flags: /experimental:preprocessor /D FMT_HEADER_ONLY=1 /wd5105 /std:c++latest
 Link-Flags: rpcrt4.lib
 Optimize: True