Parallelize downloading of dependencies from Git

src/dds.main.cpp

@@ -623,15 +623,8 @@ struct cli_build {
                 [&](dds::repository repo) {
                     // Download dependencies
                     auto deps = repo.solve(man.dependencies, cat);
+                    dds::get_all(deps, repo, cat);
                     for (const dds::package_id& pk : deps) {
-                        auto exists = !!repo.find(pk);
-                        if (!exists) {
-                            spdlog::info("Download dependency: {}", pk.to_string());
-                            auto opt_pkg = cat.get(pk);
-                            assert(opt_pkg);
-                            auto tsd = dds::get_package_sdist(*opt_pkg);
-                            repo.add_sdist(tsd.sdist, dds::if_exists::throw_exc);
-                        }
                         auto sdist_ptr = repo.find(pk);
                         assert(sdist_ptr);
                         dds::sdist_build_params deps_params;
@@ -717,15 +710,8 @@ struct cli_build_deps {
                 // Download dependencies
                 spdlog::info("Loading {} dependencies", all_deps.size());
                 auto deps = repo.solve(all_deps, cat);
+                dds::get_all(deps, repo, cat);
                 for (const dds::package_id& pk : deps) {
-                    auto exists = !!repo.find(pk);
-                    if (!exists) {
-                        spdlog::info("Download dependency: {}", pk.to_string());
-                        auto opt_pkg = cat.get(pk);
-                        assert(opt_pkg);
-                        auto tsd = dds::get_package_sdist(*opt_pkg);
-                        repo.add_sdist(tsd.sdist, dds::if_exists::throw_exc);
-                    }
                     auto sdist_ptr = repo.find(pk);
                     assert(sdist_ptr);
                     dds::sdist_build_params deps_params;

src/dds/catalog/get.cpp

@@ -2,6 +2,8 @@
 
 #include <dds/catalog/catalog.hpp>
 #include <dds/error/errors.hpp>
+#include <dds/repo/repo.hpp>
+#include <dds/util/parallel.hpp>
 
 #include <neo/assert.hpp>
 #include <nlohmann/json.hpp>
@@ -9,6 +11,8 @@
 #include <range/v3/algorithm/any_of.hpp>
 #include <range/v3/distance.hpp>
 #include <range/v3/numeric/accumulate.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/transform.hpp>
 #include <spdlog/spdlog.h>
 
 using namespace dds;
@@ -75,4 +79,33 @@ temporary_sdist dds::get_package_sdist(const package_info& pkg) {
             tsd.sdist.manifest.pkg_id.to_string());
     }
     return tsd;
-}
+}
+
+void dds::get_all(const std::vector<package_id>& pkgs, repository& repo, const catalog& cat) {
+    std::mutex repo_mut;
+
+    auto absent_pkg_infos = pkgs  //
+        | ranges::views::filter([&](auto pk) {
+                                std::scoped_lock lk{repo_mut};
+                                return !repo.find(pk);
+                            })
+        | ranges::views::transform([&](auto id) {
+                                auto info = cat.get(id);
+                                neo_assert(invariant,
+                                           info.has_value(),
+                                           "No catalog entry for package id?",
+                                           id.to_string());
+                                return *info;
+                            });
+
+    auto okay = parallel_run(absent_pkg_infos, 8, [&](package_info inf) {
+        spdlog::info("Download package: {}", inf.ident.to_string());
+        auto             tsd = get_package_sdist(inf);
+        std::scoped_lock lk{repo_mut};
+        repo.add_sdist(tsd.sdist, if_exists::throw_exc);
+    });
+
+    if (!okay) {
+        throw_external_error<errc::dependency_resolve_failure>("Downloading of packages failed.");
+    }
+}

src/dds/catalog/get.hpp

@@ -5,6 +5,8 @@
 
 namespace dds {
 
+class repository;
+class catalog;
 struct package_info;
 
 struct temporary_sdist {
@@ -14,4 +16,6 @@ struct temporary_sdist {
 
 temporary_sdist get_package_sdist(const package_info&);
 
-}  // namespace dds
+void get_all(const std::vector<package_id>& pkgs, dds::repository& repo, const catalog& cat);
+
+}  // namespace dds

src/dds/source/dist.cpp

@@ -18,7 +18,7 @@ namespace {
 
 void sdist_export_file(path_ref out_root, path_ref in_root, path_ref filepath) {
     auto relpath = fs::relative(filepath, in_root);
-    spdlog::info("Export file {}", relpath.string());
+    spdlog::debug("Export file {}", relpath.string());
     auto dest = out_root / relpath;
     fs::create_directories(dest.parent_path());
     fs::copy(filepath, dest);