New bootstrap phase. Now building upon alpha.6

tools/dds_ci/bootstrap.py

@@ -1,12 +1,14 @@
 import enum
 from pathlib import Path
+import os
 from contextlib import contextmanager
-from typing import Iterator
+from typing import Iterator, Optional, Mapping
 import sys
 import urllib.request
+import platform
 import shutil
 
-from . import paths
+from . import paths, proc
 from .dds import DDSWrapper
 from .paths import new_tempdir
 
@@ -33,7 +35,7 @@ def _do_bootstrap_download() -> Path:
     }.get(sys.platform)
     if filename is None:
         raise RuntimeError(f'We do not have a prebuilt DDS binary for the "{sys.platform}" platform')
-    url = f'https://github.com/vector-of-bool/dds/releases/download/0.1.0-alpha.4/{filename}'
+    url = f'https://github.com/vector-of-bool/dds/releases/download/0.1.0-alpha.6/{filename}'
 
     print(f'Downloading prebuilt DDS executable: {url}')
     stream = urllib.request.urlopen(url)
@@ -77,9 +79,179 @@ def get_bootstrap_exe(mode: BootstrapMode) -> Iterator[DDSWrapper]:
     elif mode is BootstrapMode.Download:
         f = _do_bootstrap_download()
     elif mode is BootstrapMode.Build:
-        f = _do_bootstrap_build()  # type: ignore  # TODO
+        f = _do_bootstrap_build()
     elif mode is BootstrapMode.Skip:
         f = paths.PREBUILT_DDS
 
     with pin_exe(f) as dds:
         yield DDSWrapper(dds)
+
+
+def _do_bootstrap_build() -> Path:
+    return _bootstrap_p6()
+
+
+def _bootstrap_p6() -> Path:
+    prev_dds = _bootstrap_alpha_4()
+    p6_dir = paths.PREBUILT_DIR / 'p6'
+    ret_dds = _dds_in(p6_dir)
+    if ret_dds.exists():
+        return ret_dds
+
+    _clone_self_at(p6_dir, '0.1.0-alpha.6')
+    tc = 'msvc-rel.jsonc' if platform.system() == 'Windows' else 'gcc-9-rel.jsonc'
+
+    catalog_arg = f'--catalog={p6_dir}/_catalog.db'
+    repo_arg = f'--repo-dir={p6_dir}/_repo'
+
+    proc.check_run(
+        [prev_dds, 'catalog', 'import', catalog_arg, '--json=old-catalog.json'],
+        cwd=p6_dir,
+    )
+    proc.check_run(
+        [
+            prev_dds,
+            'build',
+            catalog_arg,
+            repo_arg,
+            ('--toolchain', p6_dir / 'tools' / tc),
+        ],
+        cwd=p6_dir,
+    )
+    return ret_dds
+
+
+def _bootstrap_alpha_4() -> Path:
+    prev_dds = _bootstrap_alpha_3()
+    a4_dir = paths.PREBUILT_DIR / 'alpha-4'
+    ret_dds = _dds_in(a4_dir)
+    if ret_dds.exists():
+        return ret_dds
+
+    _clone_self_at(a4_dir, '0.1.0-alpha.4')
+    build_py = a4_dir / 'tools/build.py'
+    proc.check_run(
+        [
+            sys.executable,
+            '-u',
+            build_py,
+        ],
+        env=_prev_dds_env(prev_dds),
+        cwd=a4_dir,
+    )
+    return ret_dds
+
+
+def _bootstrap_alpha_3() -> Path:
+    prev_dds = _bootstrap_p5()
+    a3_dir = paths.PREBUILT_DIR / 'alpha-3'
+    ret_dds = _dds_in(a3_dir)
+    if ret_dds.exists():
+        return ret_dds
+
+    _clone_self_at(a3_dir, '0.1.0-alpha.3')
+    build_py = a3_dir / 'tools/build.py'
+    proc.check_run(
+        [
+            sys.executable,
+            '-u',
+            build_py,
+        ],
+        env=_prev_dds_env(prev_dds),
+        cwd=a3_dir,
+    )
+    return ret_dds
+
+
+def _bootstrap_p5() -> Path:
+    prev_dds = _bootstrap_p4()
+    p5_dir = paths.PREBUILT_DIR / 'p5'
+    ret_dds = _dds_in(p5_dir)
+    if ret_dds.exists():
+        return ret_dds
+
+    _clone_self_at(p5_dir, 'bootstrap-p5')
+    build_py = p5_dir / 'tools/build.py'
+    proc.check_run(
+        [
+            sys.executable,
+            '-u',
+            build_py,
+        ],
+        env=_prev_dds_env(prev_dds),
+        cwd=p5_dir,
+    )
+    return ret_dds
+
+
+def _bootstrap_p4() -> Path:
+    prev_dds = _bootstrap_p1()
+    p4_dir = paths.PREBUILT_DIR / 'p4'
+    p4_dir.mkdir(exist_ok=True, parents=True)
+    ret_dds = _dds_in(p4_dir)
+    if ret_dds.exists():
+        return ret_dds
+
+    _clone_self_at(p4_dir, 'bootstrap-p4')
+    build_py = p4_dir / 'tools/build.py'
+    proc.check_run(
+        [
+            sys.executable,
+            '-u',
+            build_py,
+            '--cxx=cl.exe' if platform.system() == 'Windows' else '--cxx=g++-8',
+        ],
+        env=_prev_dds_env(prev_dds),
+    )
+    return ret_dds
+
+
+def _bootstrap_p1() -> Path:
+    p1_dir = paths.PREBUILT_DIR / 'p1'
+    p1_dir.mkdir(exist_ok=True, parents=True)
+    ret_dds = _dds_in(p1_dir)
+    if ret_dds.exists():
+        return ret_dds
+
+    _clone_self_at(p1_dir, 'bootstrap-p1')
+    build_py = p1_dir / 'tools/build.py'
+    proc.check_run([
+        sys.executable,
+        '-u',
+        build_py,
+        '--cxx=cl.exe' if platform.system() == 'Windows' else '--cxx=g++-8',
+    ])
+    _build_prev(p1_dir)
+    return ret_dds
+
+
+def _clone_self_at(dir: Path, ref: str) -> None:
+    if dir.is_dir():
+        shutil.rmtree(dir)
+    dir.mkdir(exist_ok=True, parents=True)
+    proc.check_run(['git', 'clone', '-qq', paths.PROJECT_ROOT, f'--branch={ref}', dir])
+
+
+def _dds_in(dir: Path) -> Path:
+    return dir.joinpath('_build/dds' + paths.EXE_SUFFIX)
+
+
+def _prev_dds_env(dds: Path) -> Mapping[str, str]:
+    env = os.environ.copy()
+    env.update({'DDS_BOOTSTRAP_PREV_EXE': str(dds)})
+    return env
+
+
+def _build_prev(dir: Path, prev_dds: Optional[Path] = None) -> None:
+    build_py = dir / 'tools/build.py'
+    proc.check_run(
+        [
+            sys.executable,
+            '-u',
+            build_py,
+            '--cxx=cl.exe' if platform.system() == 'Windows' else '--cxx=g++-8',
+        ],
+        cwd=dir,
+        env=None if prev_dds is None else os.environ.clone().update({'DDS_BOOTSTRAP_PREV_EXE',
+                                                                     str(prev_dds)}),
+    )

tools/dds_ci/dds.py

@@ -33,16 +33,12 @@ class DDSWrapper:
     @property
     def pkg_db_path_arg(self) -> str:
         """The arguments for --catalog"""
-        return f'--catalog={self.pkg_db_path}'
+        return f'--pkg-db-path={self.pkg_db_path}'
 
     @property
     def cache_dir_arg(self) -> str:
         """The arguments for --repo-dir"""
-        return f'--repo-dir={self.repo_dir}'
-
-    @property
-    def project_dir_flag(self) -> str:
-        return '--project-dir'
+        return f'--pkg-cache-dir={self.repo_dir}'
 
     def set_repo_scratch(self, path: Pathish) -> None:
         self.repo_dir = Path(path) / 'data'
@@ -66,13 +62,6 @@ class DDSWrapper:
         env['DDS_NO_ADD_INITIAL_REPO'] = '1'
         proc.check_run([self.path, args], cwd=cwd or self.default_cwd, env=env, timeout=timeout)
 
-    def catalog_json_import(self, path: Path) -> None:
-        """Run 'catalog import' to import the given JSON. Only applicable to older 'dds'"""
-        self.run(['catalog', 'import', self.pkg_db_path_arg, f'--json={path}'])
-
-    def catalog_get(self, what: str) -> None:
-        self.run(['catalog', 'get', self.pkg_db_path_arg, what])
-
     def pkg_get(self, what: str) -> None:
         self.run(['pkg', 'get', self.pkg_db_path_arg, what])
 
@@ -114,7 +103,7 @@ class DDSWrapper:
                 self.cache_dir_arg,
                 self.pkg_db_path_arg,
                 f'--jobs={jobs}',
-                f'{self.project_dir_flag}={root}',
+                f'--project={root}',
                 f'--out={build_root}',
                 f'--tweaks-dir={tweaks_dir}' if tweaks_dir else (),
                 more_args or (),
@@ -138,7 +127,7 @@ class DDSWrapper:
             self.cache_dir_arg,
             paths,
             f'--toolchain={toolchain}',
-            f'{self.project_dir_flag}={project_dir}',
+            f'--project={project_dir}',
             f'--out={out}',
         ])
 
@@ -151,20 +140,3 @@ class DDSWrapper:
             self.cache_dir_arg,
             args,
         ])
-
-
-class NewDDSWrapper(DDSWrapper):
-    """
-    Wraps the new 'dds' executable with some convenience APIs
-    """
-    @property
-    def cache_dir_arg(self) -> str:
-        return f'--pkg-cache-dir={self.repo_dir}'
-
-    @property
-    def pkg_db_path_arg(self) -> str:
-        return f'--pkg-db-path={self.pkg_db_path}'
-
-    @property
-    def project_dir_flag(self) -> str:
-        return '--project'

tools/dds_ci/main.py

@@ -134,7 +134,7 @@ def ci_with_dds(dds: DDSWrapper, args: CommandArguments) -> int:
     if args.clean:
         dds.clean(build_dir=paths.BUILD_DIR)
 
-    dds.catalog_json_import(paths.PROJECT_ROOT / 'old-catalog.json')
+    dds.repo_add('https://repo-1.dds.pizza')
 
     if args.rapid:
         return main_build(dds, args)

tools/dds_ci/testing/fixtures.py

@@ -14,7 +14,7 @@ from _pytest.tmpdir import TempPathFactory
 from _pytest.fixtures import FixtureRequest
 
 from dds_ci import toolchain, paths
-from ..dds import DDSWrapper, NewDDSWrapper
+from ..dds import DDSWrapper
 from ..util import Pathish
 tc_mod = toolchain
 
@@ -225,11 +225,11 @@ def tmp_project(request: FixtureRequest, worker_id: str, project_opener: Project
 
 
 @pytest.fixture(scope='session')
-def dds(dds_exe: Path) -> NewDDSWrapper:
+def dds(dds_exe: Path) -> DDSWrapper:
     """
     A :class:`~dds_ci.dds.DDSWrapper` around the dds executable under test
     """
-    wr = NewDDSWrapper(dds_exe)
+    wr = DDSWrapper(dds_exe)
     return wr