Dependency downloading using `dds deps get`

src/dds/build.cpp

@@ -112,10 +112,10 @@ load_usage_requirements(path_ref project_root, path_ref build_root, path_ref use
     return usage_requirement_map::from_lm_index(idx);
 }
 
-void prepare_catch2_driver(library_build_params&   lib_params,
-                           test_lib                test_driver,
-                           const build_params&     params,
-                           const package_manifest& man) {
+void prepare_catch2_driver(library_build_params& lib_params,
+                           test_lib              test_driver,
+                           const build_params&   params,
+                           const package_manifest&) {
     fs::path test_include_root = params.out_root / "_test_inc";
     lib_params.test_include_dirs.emplace_back(test_include_root);
 

src/dds/dds.main.cpp

@@ -1,5 +1,6 @@
 #include <dds/build.hpp>
 #include <dds/logging.hpp>
+#include <dds/repo/remote.hpp>
 #include <dds/repo/repo.hpp>
 #include <dds/sdist.hpp>
 #include <dds/toolchain/from_dds.hpp>
@@ -120,8 +121,8 @@ struct cli_repo {
                 auto same_name
                     = [](auto&& a, auto&& b) { return a.manifest.name == b.manifest.name; };
 
-                auto all_sdists  = repo.load_sdists();
-                auto grp_by_name = all_sdists                      //
+                auto all         = repo.iter_sdists();
+                auto grp_by_name = all                             //
                     | ranges::views::group_by(same_name)           //
                     | ranges::views::transform(ranges::to_vector)  //
                     | ranges::views::transform([](auto&& grp) {
@@ -132,8 +133,7 @@ struct cli_repo {
                 for (const auto& [name, grp] : grp_by_name) {
                     spdlog::info("{}:", name);
                     for (const dds::sdist& sd : grp) {
-                        spdlog::info("  - {}",
-                                     sd.manifest.version.to_string());
+                        spdlog::info("  - {}", sd.manifest.version.to_string());
                     }
                 }
 
@@ -381,6 +381,55 @@ struct cli_deps {
         }
     } ls{*this};
 
+    struct {
+        cli_deps&     parent;
+        args::Command cmd{parent.deps_group,
+                          "get",
+                          "Ensure we have local copies of the project dependencies"};
+        common_flags  _common{cmd};
+
+        repo_where_flag repo_where{cmd};
+        path_flag       remote_listing_file{
+            cmd,
+            "remote-listing",
+            "Path to a file containing listing of remote sdists and how to obtain them",
+            {'R', "remote-list"},
+            "remote.dds"};
+
+        int run() {
+            auto man    = parent.load_package_manifest();
+            auto rd     = dds::remote_directory::load_from_file(remote_listing_file.Get());
+            bool failed = false;
+            dds::repository::with_repository(  //
+                repo_where.Get(),
+                dds::repo_flags::write_lock | dds::repo_flags::create_if_absent,
+                [&](dds::repository repo) {
+                    for (auto& dep : man.dependencies) {
+                        auto exists = !!repo.find(dep.name, dep.version);
+                        if (!exists) {
+                            spdlog::info("Pull remote: {} {}", dep.name, dep.version.to_string());
+                            auto opt_remote = rd.find(dep.name, dep.version);
+                            if (opt_remote) {
+                                auto tsd = opt_remote->pull_sdist();
+                                repo.add_sdist(tsd.sdist, dds::if_exists::ignore);
+                            } else {
+                                spdlog::error("No remote listing for {} {}",
+                                              dep.name,
+                                              dep.version.to_string());
+                                failed = true;
+                            }
+                        } else {
+                            spdlog::info("Okay: {} {}", dep.name, dep.version.to_string());
+                        }
+                    }
+                });
+            if (failed) {
+                return 1;
+            }
+            return 0;
+        }
+    } get{*this};
+
     struct {
         cli_deps&     parent;
         args::Command cmd{parent.deps_group, "build", "Build project dependencies"};
@@ -435,6 +484,8 @@ struct cli_deps {
             return ls.run();
         } else if (build.cmd) {
             return build.run();
+        } else if (get.cmd) {
+            return get.run();
         }
         std::terminate();
     }

src/dds/deps.cpp

@@ -43,41 +43,13 @@ dependency dependency::parse_depends_string(std::string_view str) {
 }
 
 std::vector<sdist> dds::find_dependencies(const repository& repo, const dependency& dep) {
-    auto all_dists = repo.load_sdists();
-    detail::sort_sdists(all_dists);
     std::vector<sdist> acc;
-    detail::do_find_deps(all_dists, dep, acc);
+    detail::do_find_deps(repo, dep, acc);
     return acc;
 }
 
-auto tie_sdist(const sdist& sd) {
-    return std::tuple(sd.manifest.name, sd.manifest.version.to_string());
-}
-
-auto sdist_compare
-    = [](const sdist& lhs, const sdist& rhs) { return tie_sdist(lhs) < tie_sdist(rhs); };
-
-void detail::sort_sdists(std::vector<sdist>& sd) { std::sort(sd.begin(), sd.end(), sdist_compare); }
-
-namespace {
-
-const sdist*
-get_sdist(const std::vector<sdist>& sorted_sds, std::string_view name, std::string_view version) {
-    auto found
-        = std::partition_point(sorted_sds.begin(), sorted_sds.end(), [&](const auto& candidate) {
-              return tie_sdist(candidate) < std::tie(name, version);
-          });
-    if (found->manifest.name == name && found->manifest.version.to_string() == version) {
-        return &*found;
-    }
-    return nullptr;
-}
-}  // namespace
-
-void detail::do_find_deps(const std::vector<sdist>& sdists,
-                          const dependency&         dep,
-                          std::vector<sdist>&       sd) {
-    auto sdist_opt = get_sdist(sdists, dep.name, dep.version.to_string());
+void detail::do_find_deps(const repository& repo, const dependency& dep, std::vector<sdist>& sd) {
+    auto sdist_opt = repo.find(dep.name, dep.version);
     if (!sdist_opt) {
         throw std::runtime_error(
             fmt::format("Unable to find dependency to satisfy requirement: {} {}",
@@ -86,7 +58,7 @@ void detail::do_find_deps(const std::vector<sdist>& sdists,
     }
     const sdist& new_sd = *sdist_opt;
     for (const auto& inner_dep : new_sd.manifest.dependencies) {
-        do_find_deps(sdists, inner_dep, sd);
+        do_find_deps(repo, inner_dep, sd);
     }
     auto insert_point = std::partition_point(sd.begin(), sd.end(), [&](const sdist& cand) {
         return cand.path < new_sd.path;

src/dds/deps.hpp

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <dds/build/plan/full.hpp>
-#include <dds/repo/repo.hpp>
 
 #include <semver/version.hpp>
 
@@ -10,6 +9,7 @@
 namespace dds {
 
 struct sdist;
+class repository;
 
 enum class version_strength {
     exact,
@@ -27,8 +27,7 @@ struct dependency {
 
 namespace detail {
 
-void do_find_deps(const std::vector<sdist>&, const dependency& dep, std::vector<sdist>& acc);
-void sort_sdists(std::vector<sdist>& sds);
+void do_find_deps(const repository&, const dependency& dep, std::vector<sdist>& acc);
 
 }  // namespace detail
 
@@ -37,10 +36,8 @@ std::vector<sdist> find_dependencies(const repository& repo, const dependency& d
 template <typename Iter, typename Snt>
 inline std::vector<sdist> find_dependencies(const repository& repo, Iter it, Snt stop) {
     std::vector<sdist> acc;
-    auto               all_sds = repo.load_sdists();
-    detail::sort_sdists(all_sds);
     while (it != stop) {
-        detail::do_find_deps(all_sds, *it++, acc);
+        detail::do_find_deps(repo, *it++, acc);
     }
     return acc;
 }
@@ -49,4 +46,4 @@ build_plan create_deps_build_plan(const std::vector<sdist>& deps, build_env_ref
 
 void write_libman_index(path_ref where, const build_plan& plan, const build_env& env);
 
-}  // namespace dds
+}  // namespace dds

src/dds/repo/remote.cpp

@@ -0,0 +1,141 @@
+#include "./remote.hpp"
+
+#include <dds/deps.hpp>
+#include <dds/proc.hpp>
+#include <dds/repo/repo.hpp>
+#include <dds/sdist.hpp>
+#include <dds/temp.hpp>
+#include <dds/toolchain/toolchain.hpp>
+
+#include <spdlog/spdlog.h>
+
+#include <libman/parse.hpp>
+
+#include <algorithm>
+
+using namespace dds;
+
+namespace {
+
+struct read_listing_item {
+    std::string_view                                    _key;
+    std::set<remote_listing, remote_listing_compare_t>& out;
+
+    bool operator()(std::string_view context, std::string_view key, std::string_view value) {
+        if (key != _key) {
+            return false;
+        }
+
+        auto nested        = lm::nested_kvlist::parse(value);
+        auto name_ver_pair = split_shell_string(nested.primary);
+        if (name_ver_pair.size() != 2) {
+            throw std::runtime_error(
+                fmt::format("{}: Invalid Remote-Package identity: '{}'", context, nested.primary));
+        }
+        auto name    = name_ver_pair[0];
+        auto version = semver::version::parse(name_ver_pair[1]);
+        put_listing(context, name, version, nested.pairs);
+        return true;
+    }
+
+    void put_listing(std::string_view     context,
+                     std::string          name,
+                     semver::version      version,
+                     const lm::pair_list& pairs) {
+        if (pairs.find("git")) {
+            std::string url;
+            std::string ref;
+            lm::read(fmt::format("{}: Parsing Git remote listing", context),
+                     pairs,
+                     lm::read_required("url", url),
+                     lm::read_required("ref", ref),
+                     lm::read_check_eq("git", ""),
+                     lm::reject_unknown());
+            auto did_insert
+                = out.emplace(
+                         remote_listing{std::move(name), version, git_remote_listing{url, ref}})
+                      .second;
+            if (!did_insert) {
+                spdlog::warn("Duplicate remote package defintion for {} {}",
+                             name,
+                             version.to_string());
+            }
+        } else {
+            throw std::runtime_error(fmt::format("Unable to determine remote type of package {} {}",
+                                                 name,
+                                                 version.to_string()));
+        }
+    }
+};
+
+temporary_sdist do_pull_sdist(const git_remote_listing& git) {
+    auto tmpdir = dds::temporary_dir::create();
+    using namespace std::literals;
+    spdlog::info("Cloning repository: {} [{}] ...", git.url, git.ref);
+    auto command = {"git"s,
+                    "clone"s,
+                    "--depth=1"s,
+                    "--branch"s,
+                    git.ref,
+                    git.url,
+                    tmpdir.path().generic_string()};
+    auto git_res = run_proc(command);
+    if (!git_res.okay()) {
+        throw std::runtime_error(
+            fmt::format("Git clone operation failed [Git command: {}] [Exitted {}]:\n{}",
+                        quote_command(command),
+                        git_res.retc,
+                        git_res.output));
+    }
+    spdlog::info("Create sdist from clone ...");
+    sdist_params params;
+    params.project_dir = tmpdir.path();
+    auto sd_tmp_dir    = dds::temporary_dir::create();
+    params.dest_path   = sd_tmp_dir.path();
+    auto sd            = create_sdist(params);
+    return {sd_tmp_dir, sd};
+}
+
+}  // namespace
+
+temporary_sdist remote_listing::pull_sdist() const {
+    auto tsd = visit([](auto&& actual) { return do_pull_sdist(actual); });
+    if (tsd.sdist.manifest.name != name) {
+        throw std::runtime_error(
+            fmt::format("The name in the generated sdist ('{}') does not match the name listed in "
+                        "the remote listing file (expected '{}')",
+                        tsd.sdist.manifest.name,
+                        name));
+    }
+    if (tsd.sdist.manifest.version != version) {
+        throw std::runtime_error(
+            fmt::format("The version of the generated sdist is '{}', which does not match the "
+                        "expected version '{}'",
+                        tsd.sdist.manifest.version.to_string(),
+                        version.to_string()));
+    }
+    return tsd;
+}
+
+remote_directory remote_directory::load_from_file(path_ref filepath) {
+    auto        kvs = lm::parse_file(filepath);
+    listing_set listings;
+    lm::read(fmt::format("Loading remote package listing from {}", filepath.string()),
+             kvs,
+             read_listing_item{"Remote-Package", listings},
+             lm::reject_unknown());
+    return {std::move(listings)};
+}
+
+const remote_listing* remote_directory::find(std::string_view name, semver::version ver) const
+    noexcept {
+    auto found = _remotes.find(std::tie(name, ver));
+    if (found == _remotes.end()) {
+        return nullptr;
+    }
+    return &*found;
+}
+
+void remote_directory::ensure_all_local(const repository&) const {
+    spdlog::critical("Dependency download is not fully implemented!");
+}

src/dds/repo/remote.hpp

@@ -0,0 +1,73 @@
+#pragma once
+
+#include <dds/util/fs.hpp>
+
+#include <dds/sdist.hpp>
+#include <dds/temp.hpp>
+#include <semver/version.hpp>
+
+#include <set>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <variant>
+
+namespace dds {
+
+struct temporary_sdist {
+    temporary_dir tmpdir;
+    dds::sdist    sdist;
+};
+
+struct git_remote_listing {
+    std::string url;
+    std::string ref;
+
+    void clone(path_ref path) const;
+};
+
+struct remote_listing {
+    std::string                      name;
+    semver::version                  version;
+    std::variant<git_remote_listing> remote;
+
+    template <typename Func>
+    decltype(auto) visit(Func&& fn) const {
+        return std::visit(std::forward<Func>(fn), remote);
+    }
+
+    temporary_sdist pull_sdist() const;
+};
+
+inline constexpr struct remote_listing_compare_t {
+    using is_transparent = int;
+    auto tie(const remote_listing& rl) const { return std::tie(rl.name, rl.version); }
+    bool operator()(const remote_listing& lhs, const remote_listing& rhs) const {
+        return tie(lhs) < tie(rhs);
+    }
+    template <typename Name, typename Version>
+    bool operator()(const remote_listing& lhs, const std::tuple<Name, Version>& rhs) const {
+        return tie(lhs) < rhs;
+    }
+    template <typename Name, typename Version>
+    bool operator()(const std::tuple<Name, Version>& lhs, const remote_listing& rhs) const {
+        return lhs < tie(rhs);
+    }
+} remote_listing_compare;
+
+class remote_directory {
+    using listing_set = std::set<remote_listing, remote_listing_compare_t>;
+    listing_set _remotes;
+
+    remote_directory(listing_set s)
+        : _remotes(std::move(s)) {}
+
+public:
+    static remote_directory load_from_file(path_ref);
+
+    void ensure_all_local(const class repository& repo) const;
+
+    const remote_listing* find(std::string_view name, semver::version ver) const noexcept;
+};
+
+}  // namespace dds

src/dds/repo/repo.cpp

@@ -6,6 +6,7 @@
 
 #include <spdlog/spdlog.h>
 
+#include <range/v3/range/conversion.hpp>
 #include <range/v3/view/filter.hpp>
 #include <range/v3/view/transform.hpp>
 
@@ -13,26 +14,63 @@ using namespace dds;
 
 using namespace ranges;
 
+namespace {
+
+auto load_sdists(path_ref root) {
+    using namespace ranges;
+    using namespace ranges::views;
+
+    auto try_read_sdist = [](path_ref p) -> std::optional<sdist> {
+        if (starts_with(p.filename().string(), ".")) {
+            return std::nullopt;
+        }
+        try {
+            return sdist::from_directory(p);
+        } catch (const std::runtime_error& e) {
+            spdlog::error("Failed to load source distribution from directory '{}': {}",
+                          p.string(),
+                          e.what());
+            return std::nullopt;
+        }
+    };
+
+    return
+        // Get the top-level `name-version` dirs
+        fs::directory_iterator(root)  //
+        // // Convert each dir into an `sdist` object
+        | transform(try_read_sdist)  //
+        // // Drop items that failed to load
+        | filter([](auto&& opt) { return opt.has_value(); })  //
+        | transform([](auto&& opt) { return *opt; })          //
+        ;
+}
+
+}  // namespace
+
 void repository::_log_blocking(path_ref dirpath) noexcept {
     spdlog::warn("Another process has the repository directory locked [{}]", dirpath.string());
     spdlog::warn("Waiting for repository to be released...");
 }
 
-void repository::_init_repo_dir(path_ref dirpath) noexcept {
-    fs::create_directories(dirpath);
-}
+void repository::_init_repo_dir(path_ref dirpath) noexcept { fs::create_directories(dirpath); }
 
 fs::path repository::default_local_path() noexcept { return dds_data_dir() / "repo"; }
 
-repository repository::open_for_directory(path_ref dirpath) {
+repository repository::_open_for_directory(bool writeable, path_ref dirpath) {
     auto dist_dir = dirpath;
-    auto entries  = fs::directory_iterator(dist_dir) | to_vector;
-    return {dirpath};
+    auto entries  = load_sdists(dirpath) | to<sdist_set>;
+    return {writeable, dirpath, std::move(entries)};
 }
 
 void repository::add_sdist(const sdist& sd, if_exists ife_action) {
-    auto sd_dest
-        = _root / fmt::format("{}_{}", sd.manifest.name, sd.manifest.version.to_string());
+    if (!_write_enabled) {
+        spdlog::critical(
+            "DDS attempted to write into a repository that wasn't opened with a write-lock. This "
+            "is a hard bug and should be reported. For the safety and integrity of the local "
+            "repository, we'll hard-exit immediately.");
+        std::terminate();
+    }
+    auto sd_dest = _root / fmt::format("{}_{}", sd.manifest.name, sd.manifest.version.to_string());
     if (fs::exists(sd_dest)) {
         auto msg = fmt::format("Source distribution '{}' is already available in the local repo",
                                sd.path.string());
@@ -59,41 +97,10 @@ void repository::add_sdist(const sdist& sd, if_exists ife_action) {
     spdlog::info("Source distribution '{}' successfully exported", sd.ident());
 }
 
-std::vector<sdist> repository::load_sdists() const {
-    using namespace ranges;
-    using namespace ranges::views;
-
-    auto try_read_sdist = [](path_ref p) -> std::optional<sdist> {
-        if (starts_with(p.filename().string(), ".")) {
-            return std::nullopt;
-        }
-        try {
-            return sdist::from_directory(p);
-        } catch (const std::runtime_error& e) {
-            spdlog::error("Failed to load source distribution from directory '{}': {}",
-                          p.string(),
-                          e.what());
-            return std::nullopt;
-        }
-    };
-
-    return
-        // Get the top-level `name-version` dirs
-        fs::directory_iterator(_root)  //
-        // // Convert each dir into an `sdist` object
-        | transform(try_read_sdist)  //
-        // // Drop items that failed to load
-        | filter([](auto&& opt) { return opt.has_value(); })  //
-        | transform([](auto&& opt) { return *opt; })          //
-        | to_vector                                           //
-        ;
-}
-
-std::optional<sdist> repository::get_sdist(std::string_view name, std::string_view version) const {
-    auto expect_path = _root / fmt::format("{}_{}", name, version);
-    if (!fs::is_directory(expect_path)) {
-        return std::nullopt;
+const sdist* repository::find(std::string_view name, semver::version ver) const noexcept {
+    auto found = _sdists.find(std::tie(name, ver));
+    if (found == _sdists.end()) {
+        return nullptr;
     }
-
-    return sdist::from_directory(expect_path);
+    return &*found;
 }

src/dds/repo/repo.hpp

@@ -1,17 +1,17 @@
 #pragma once
 
+#include <dds/sdist.hpp>
 #include <dds/util/flock.hpp>
 #include <dds/util/fs.hpp>
 
 #include <functional>
 #include <optional>
+#include <set>
 #include <shared_mutex>
 #include <vector>
 
 namespace dds {
 
-struct sdist;
-
 enum repo_flags {
     none             = 0b00,
     read             = none,
@@ -30,13 +30,20 @@ inline repo_flags operator|(repo_flags a, repo_flags b) {
 }
 
 class repository {
-    fs::path _root;
+    using sdist_set = std::set<sdist, sdist_compare_t>;
+
+    bool      _write_enabled = false;
+    fs::path  _root;
+    sdist_set _sdists;
 
-    repository(path_ref p)
-        : _root(p) {}
+    repository(bool writeable, path_ref p, sdist_set sds)
+        : _write_enabled(writeable)
+        , _root(p)
+        , _sdists(std::move(sds)) {}
 
-    static void _log_blocking(path_ref dir) noexcept;
-    static void _init_repo_dir(path_ref dir) noexcept;
+    static void       _log_blocking(path_ref dir) noexcept;
+    static void       _init_repo_dir(path_ref dir) noexcept;
+    static repository _open_for_directory(bool writeable, path_ref);
 
 public:
     template <typename Func>
@@ -51,7 +58,9 @@ public:
         std::shared_lock  shared_lk{mut, std::defer_lock};
         std::unique_lock  excl_lk{mut, std::defer_lock};
 
-        if (flags & repo_flags::write_lock) {
+        bool writeable = (flags & repo_flags::write_lock) != repo_flags::none;
+
+        if (writeable) {
             if (!excl_lk.try_lock()) {
                 _log_blocking(dirpath);
                 excl_lk.lock();
@@ -63,16 +72,29 @@ public:
             }
         }
 
-        return std::invoke((Func &&) fn, open_for_directory(dirpath));
+        auto repo = _open_for_directory(writeable, dirpath);
+        return std::invoke((Func &&) fn, std::move(repo));
     }
 
-    static repository open_for_directory(path_ref);
-
     static fs::path default_local_path() noexcept;
 
-    void                 add_sdist(const sdist&, if_exists = if_exists::throw_exc);
-    std::optional<sdist> get_sdist(std::string_view name, std::string_view version) const;
-    std::vector<sdist>   load_sdists() const;
+    void add_sdist(const sdist&, if_exists = if_exists::throw_exc);
+
+    const sdist* find(std::string_view name, semver::version ver) const noexcept;
+
+    auto iter_sdists() const noexcept {
+        class ret {
+            const sdist_set& s;
+
+        public:
+            ret(const sdist_set& s)
+                : s(s) {}
+
+            auto begin() const { return s.cbegin(); }
+            auto end() const { return s.cend(); }
+        } r{_sdists};
+        return r;
+    }
 };
 
 }  // namespace dds

src/dds/sdist.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <tuple>
+
 #include <dds/package_manifest.hpp>
 #include <dds/util/fs.hpp>
 
@@ -28,6 +30,22 @@ struct sdist {
     }
 };
 
+inline constexpr struct sdist_compare_t {
+    bool operator()(const sdist& lhs, const sdist& rhs) const {
+        return std::tie(lhs.manifest.name, lhs.manifest.version)
+            < std::tie(rhs.manifest.name, rhs.manifest.version);
+    }
+    template <typename Name, typename Version>
+    bool operator()(const sdist& lhs, const std::tuple<Name, Version>& rhs) const {
+        return std::tie(lhs.manifest.name, lhs.manifest.version) < rhs;
+    }
+    template <typename Name, typename Version>
+    bool operator()(const std::tuple<Name, Version>& lhs, const sdist& rhs) const {
+        return lhs < std::tie(rhs.manifest.name, rhs.manifest.version);
+    }
+    using is_transparent = int;
+} sdist_compare;
+
 sdist create_sdist(const sdist_params&);
 sdist create_sdist_in_dir(path_ref, const sdist_params&);
 

src/libman/parse.cpp

@@ -84,10 +84,10 @@ void lm::write_pairs(fs::path fpath, const std::vector<pair>& pairs) {
 }
 
 nested_kvlist nested_kvlist::parse(const std::string_view line_) {
-    const auto line     = trim(line_);
+    const auto line     = trim_view(line_);
     const auto semi_pos = line.find(';');
-    const auto primary  = trim(line.substr(0, semi_pos));
-    auto       tail     = semi_pos == line.npos ? ""sv : trim(line.substr(semi_pos + 1));
+    const auto primary  = trim_view(line.substr(0, semi_pos));
+    auto       tail     = semi_pos == line.npos ? ""sv : trim_view(line.substr(semi_pos + 1));
 
     std::vector<pair> pairs;
     while (!tail.empty()) {
@@ -106,7 +106,7 @@ nested_kvlist nested_kvlist::parse(const std::string_view line_) {
         if (space_pos == tail.npos) {
             break;
         }
-        tail = trim(tail.substr(space_pos + 1));
+        tail = trim_view(tail.substr(space_pos + 1));
     }
 
     return nested_kvlist{std::string(primary), pair_list{std::move(pairs)}};