Merge branch 'develop' into feature/get-deps

azure-pipelines.yml

@@ -10,7 +10,7 @@ jobs:
           echo Loading VS environment
           call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\Enterprise\\Common7\\Tools\\vsdevcmd" -arch=x64 || exit 1
           echo Executing Build and Tests
-          python -u tools/ci.py --cxx cl.exe -T tools\\msvc.dds || exit 1
+          python -u tools/ci.py -B download --cxx cl.exe -T tools\\msvc.dds || exit 1
         displayName: Full CI
       - publish: _build/dds.exe
         artifact: DDS Executable - Windows VS2019
@@ -21,7 +21,7 @@ jobs:
     steps:
       - script: sudo apt update -y && sudo apt install -y python3-minimal g++-8
         displayName: Prepare System
-      - script: python3 -u tools/ci.py --cxx g++-8 -T tools/gcc-8.dds
+      - script: python3 -u tools/ci.py -B download --cxx g++-8 -T tools/gcc-8.dds
         displayName: Full CI
       - publish: _build/dds
         artifact: DDS Executable - Linux
@@ -32,7 +32,7 @@ jobs:
     steps:
       - script: brew install gcc@8
         displayName: Prepare System
-      - script: python3 -u tools/ci.py --cxx g++-8 -T tools/gcc-8.dds
+      - script: python3 -u tools/ci.py -B download --cxx g++-8 -T tools/gcc-8.dds
         displayName: Build and Run Unit Tests
       - publish: _build/dds
         artifact: DDS Executable - macOS

src/dds/library_manifest.cpp

@@ -20,6 +20,7 @@ library_manifest library_manifest::load_from_file(const fs::path& fpath) {
              lm::read_accumulate("Uses", uses_strings),
              lm::read_accumulate("Links", links_strings),
              lm::read_required("Name", ret.name),
+             lm::ignore_x_keys(),
              lm::reject_unknown());
 
     extend(ret.uses, ranges::views::transform(uses_strings, lm::split_usage_string));

src/dds/source.hpp

@@ -3,6 +3,7 @@
 #include <dds/util/fs.hpp>
 
 #include <optional>
+#include <vector>
 
 namespace dds {
 

src/libman/parse.cpp

@@ -79,7 +79,7 @@ lm::pair_list lm::parse_file(fs::path fpath) { return parse_string(dds::slurp_fi
 void lm::write_pairs(fs::path fpath, const std::vector<pair>& pairs) {
     auto fstream = dds::open(fpath, std::ios::out | std::ios::binary);
     for (auto& pair : pairs) {
-        fstream << pair.key() << ": " << pair.value() << '\n';
+        fstream << pair.key << ": " << pair.value << '\n';
     }
 }
 

src/libman/parse.hpp

@@ -13,25 +13,13 @@
 namespace lm {
 
 class pair {
-    std::string _key;
-    std::string _value;
-
 public:
+    std::string key;
+    std::string value;
+
     pair(std::string_view k, std::string_view v)
-        : _key(k)
-        , _value(v) {}
-
-    auto& key() const noexcept { return _key; }
-    auto& value() const noexcept { return _value; }
-
-    template <std::size_t I>
-    std::string_view get() const {
-        if constexpr (I == 0) {
-            return key();
-        } else if constexpr (I == 1) {
-            return value();
-        }
-    }
+        : key(k)
+        , value(v) {}
 };
 
 class pair_iterator {
@@ -53,7 +41,7 @@ public:
     pair_iterator& operator++() & noexcept {
         assert(_iter != _end);
         ++_iter;
-        while (_iter != _end && _iter->key() != _key) {
+        while (_iter != _end && _iter->key != _key) {
             ++_iter;
         }
         return *this;
@@ -88,7 +76,7 @@ public:
 
     const pair* find(const std::string_view& key) const noexcept {
         for (auto&& item : items()) {
-            if (item.key() == key) {
+            if (item.key == key) {
                 return &item;
             }
         }
@@ -98,7 +86,7 @@ public:
     pair_iterator iter(std::string_view key) const noexcept {
         auto       iter = items().begin();
         const auto end  = items().end();
-        while (iter != end && iter->key() != key) {
+        while (iter != end && iter->key != key) {
             ++iter;
         }
         return pair_iterator{iter, end, key};
@@ -262,6 +250,13 @@ public:
     }
 };
 
+class ignore_x_keys {
+public:
+    bool operator()(std::string_view, std::string_view key, std::string_view) const {
+        return key.find("X-") == 0;
+    }
+};
+
 class reject_unknown {
 public:
     int operator()(std::string_view context, std::string_view key, std::string_view) const {
@@ -291,15 +286,3 @@ auto read(std::string_view context [[maybe_unused]], const pair_list& pairs, Ite
 }
 
 }  // namespace lm
-
-namespace std {
-
-template <>
-struct tuple_size<lm::pair> : std::integral_constant<int, 2> {};
-
-template <std::size_t N>
-struct tuple_element<N, lm::pair> {
-    using type = std::string_view;
-};
-
-}  // namespace std

src/libman/parse.test.cpp

@@ -18,13 +18,13 @@ void test_simple() {
     kvs    = parse_string(lm_src);
     CHECK(kvs.size() == 1);
     REQUIRE(kvs.find("foo"));
-    CHECK(kvs.find("foo")->value() == "bar");
+    CHECK(kvs.find("foo")->value == "bar");
 
     lm_src = "foo:bar: baz";
     kvs    = parse_string(lm_src);
     CHECK(kvs.size() == 1);
     REQUIRE(kvs.find("foo:bar"));
-    CHECK(kvs.find("foo:bar")->value() == "baz");
+    CHECK(kvs.find("foo:bar")->value == "baz");
 
     CHECK(parse_string("#comment").size() == 0);
     CHECK(parse_string("\n\n").size() == 0);
@@ -46,33 +46,33 @@ void test_simple() {
         kvs = parse_string(s);
         CHECK(kvs.size() == 1);
         REQUIRE(kvs.find("Foo"));
-        CHECK(kvs.find("Foo")->value() == "");
+        CHECK(kvs.find("Foo")->value == "");
     }
 
     kvs = parse_string("foo: # Not a comment");
     CHECK(kvs.size() == 1);
     REQUIRE(kvs.find("foo"));
-    CHECK(kvs.find("foo")->value() == "# Not a comment");
+    CHECK(kvs.find("foo")->value == "# Not a comment");
 }
 
 void test_multi() {
     auto kvs = parse_string("Foo: bar\nbaz: qux");
     CHECK(kvs.size() == 2);
     REQUIRE(kvs.find("Foo"));
-    CHECK(kvs.find("Foo")->value() == "bar");
+    CHECK(kvs.find("Foo")->value == "bar");
     REQUIRE(kvs.find("baz"));
-    CHECK(kvs.find("baz")->value() == "qux");
+    CHECK(kvs.find("baz")->value == "qux");
 
     kvs = parse_string("foo: first\nfoo: second\n");
     CHECK(kvs.size() == 2);
     auto iter = kvs.iter("foo");
     REQUIRE(iter);
-    CHECK(iter->key() == "foo");
-    CHECK(iter->value() == "first");
+    CHECK(iter->key == "foo");
+    CHECK(iter->value == "first");
     ++iter;
     REQUIRE(iter);
-    CHECK(iter->key() == "foo");
-    CHECK(iter->value() == "second");
+    CHECK(iter->key == "foo");
+    CHECK(iter->value == "second");
     ++iter;
     CHECK(!iter);
 
@@ -86,7 +86,7 @@ void test_nested_kvlist() {
         CHECK(result.primary == "Foo");
         CHECK(result.pairs.size() == 1);
         REQUIRE(result.pairs.find("bar"));
-        CHECK(result.pairs.find("bar")->value() == "baz");
+        CHECK(result.pairs.find("bar")->value == "baz");
     };
     check_1("Foo; bar=baz");
     check_1("Foo ; bar=baz");
@@ -111,9 +111,9 @@ void test_nested_kvlist() {
         CHECK(result.primary == "Foo bar");
         CHECK(result.pairs.size() == 2);
         REQUIRE(result.pairs.find("baz"));
-        CHECK(result.pairs.find("baz")->value() == "meow");
+        CHECK(result.pairs.find("baz")->value == "meow");
         REQUIRE(result.pairs.find("quux"));
-        CHECK(result.pairs.find("quux")->value() == "");
+        CHECK(result.pairs.find("quux")->value == "");
     };
 
     check_3("Foo bar; baz=meow quux");

tools/bootstrap.py

@@ -20,9 +20,10 @@ PREBUILT_DIR = PROJECT_ROOT / '_prebuilt'
 EXE_SUFFIX = '.exe' if os.name == 'nt' else ''
 
 
-def _run_quiet(args) -> None:
-    cmd = [str(s) for s in args]
-    res = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+def _run_quiet(cmd, **kwargs) -> None:
+    cmd = [str(s) for s in cmd]
+    res = subprocess.run(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
     if res.returncode != 0:
         print(f'Subprocess command {cmd} failed '
               f'[{res.returncode}]:\n{res.stdout.decode()}')
@@ -30,7 +31,7 @@ def _run_quiet(args) -> None:
 
 
 def _clone_bootstrap_phase(ph: str) -> Path:
-    print(f'Cloning: {ph}')
+    print(f'Clone revision: {ph}')
     bts_dir = BOOTSTRAP_BASE_DIR / ph
     if bts_dir.exists():
         shutil.rmtree(bts_dir)
@@ -45,11 +46,12 @@ def _clone_bootstrap_phase(ph: str) -> Path:
     return bts_dir
 
 
-def _build_bootstrap_phase(ph: str, bts_dir: Path, args: argparse.Namespace) -> None:
-    print(f'Running build: {ph} (Please wait a moment...)')
+def _build_bootstrap_phase(ph: str, bts_dir: Path,
+                           args: argparse.Namespace) -> None:
+    print(f'Build revision: {ph} [This may take a moment]')
     env = os.environ.copy()
     env['DDS_BOOTSTRAP_PREV_EXE'] = str(PREBUILT_DIR / 'dds')
-    subprocess.check_call(
+    _run_quiet(
         [
             sys.executable,
             '-u',
@@ -84,7 +86,8 @@ def main(argv: Sequence[str]) -> int:
     parser.add_argument(
         '--cxx', help='The C++ compiler to use for the build', required=True)
     args = parser.parse_args(argv)
-    for phase in BOOTSTRAP_PHASES:
+    for idx, phase in enumerate(BOOTSTRAP_PHASES):
+        print(f'Bootstrap phase [{idx+1}/{len(BOOTSTRAP_PHASES)}]')
         exe = _run_boot_phase(phase, args)
 
     print(f'A bootstrapped DDS executable has been generated: {exe}')

tools/ci.py

@@ -1,8 +1,10 @@
 import argparse
+import os
 import sys
 from pathlib import Path
 from typing import Sequence, NamedTuple
 import subprocess
+import urllib.request
 
 HERE = Path(__file__).parent.absolute()
 TOOLS_DIR = HERE
@@ -11,13 +13,12 @@ PREBUILT_DDS = PROJECT_ROOT / '_prebuilt/dds'
 
 
 class CIOptions(NamedTuple):
-    skip_bootstrap: bool
     cxx: Path
     toolchain: str
 
 
-def _do_bootstrap(opts: CIOptions) -> None:
-    print('Running bootstrap')
+def _do_bootstrap_build(opts: CIOptions) -> None:
+    print('Bootstrapping by a local build of prior versions...')
     subprocess.check_call([
         sys.executable,
         '-u',
@@ -26,13 +27,42 @@ def _do_bootstrap(opts: CIOptions) -> None:
     ])
 
 
+def _do_bootstrap_download() -> None:
+    filename = {
+        'win32': 'dds-win-x64.exe',
+        'linux': 'dds-linux-x64',
+        'darwin': 'dds-macos-x64',
+    }.get(sys.platform)
+    if filename is None:
+        raise RuntimeError(f'We do not have a prebuilt DDS binary for the "{sys.platform}" platform')
+    url = f'https://github.com/vector-of-bool/dds/releases/download/bootstrap-p2/{filename}'
+
+    print(f'Downloading prebuilt DDS executable: {url}')
+    stream = urllib.request.urlopen(url)
+    PREBUILT_DDS.parent.mkdir(exist_ok=True, parents=True)
+    with PREBUILT_DDS.open('wb') as fd:
+        while True:
+            buf = stream.read(1024 * 4)
+            if not buf:
+                break
+            fd.write(buf)
+
+    if os.name != 'nt':
+        # Mark the binary executable. By default it won't be
+        mode = PREBUILT_DDS.stat().st_mode
+        mode |= 0b001_001_001
+        PREBUILT_DDS.chmod(mode)
+
+
 def main(argv: Sequence[str]) -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        '--skip-bootstrap',
-        action='store_true',
+        '-B',
+        '--bootstrap-with',
         help=
         'Skip the prebuild-bootstrap step. This requires a _prebuilt/dds to exist!',
+        choices=('download', 'build'),
+        required=True,
     )
     parser.add_argument(
         '--cxx',
@@ -44,12 +74,15 @@ def main(argv: Sequence[str]) -> int:
         help='The toolchain to use for the CI process',
         required=True)
     args = parser.parse_args(argv)
-    opts = CIOptions(
-        skip_bootstrap=args.skip_bootstrap,
-        cxx=Path(args.cxx),
-        toolchain=args.toolchain)
-    if not opts.skip_bootstrap:
-        _do_bootstrap(opts)
+
+    opts = CIOptions(cxx=Path(args.cxx), toolchain=args.toolchain)
+
+    if args.bootstrap_with == 'build':
+        _do_bootstrap_build(opts)
+    elif args.bootstrap_with == 'download':
+        _do_bootstrap_download()
+    else:
+        assert False, 'impossible'
 
     subprocess.check_call([
         str(PREBUILT_DDS),
@@ -66,11 +99,12 @@ def main(argv: Sequence[str]) -> int:
         f'-T{opts.toolchain}',
     ])
 
+    exe_suffix = '.exe' if os.name == 'nt' else ''
     subprocess.check_call([
         sys.executable,
         '-u',
         str(TOOLS_DIR / 'test.py'),
-        f'--exe={PROJECT_ROOT / "_build/dds"}',
+        f'--exe={PROJECT_ROOT / f"_build/dds{exe_suffix}"}',
         f'-T{opts.toolchain}',
     ])
 

tools/msvc.dds

@@ -1,5 +1,5 @@
 Compiler-ID: MSVC
-C++-Version: C++17
-Flags: /experimental:preprocessor /D FMT_HEADER_ONLY=1 /wd5105
+# C++-Version: C++17
+Flags: /experimental:preprocessor /D FMT_HEADER_ONLY=1 /wd5105 /std:c++latest
 Link-Flags: rpcrt4.lib
 Optimize: True