Building of dependencies is now run as part of the main build

src/dds/build.cpp

@@ -9,9 +9,14 @@
 #include <libman/index.hpp>
 #include <libman/parse.hpp>
 
+#include <range/v3/algorithm/transform.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/transform.hpp>
 #include <spdlog/spdlog.h>
 
 #include <array>
+#include <map>
+#include <set>
 #include <stdexcept>
 
 using namespace dds;
@@ -185,30 +190,111 @@ void prepare_test_driver(library_build_params&   lib_params,
     }
 }
 
+void add_ureqs(usage_requirement_map& ureqs,
+               const sdist&           sd,
+               const library&         lib,
+               const library_plan&    lib_plan,
+               build_env_ref          env) {
+    lm::library& reqs = ureqs.add(sd.manifest.namespace_, lib.manifest().name);
+    reqs.include_paths.push_back(lib.public_include_dir());
+    reqs.name  = lib.manifest().name;
+    reqs.uses  = lib.manifest().uses;
+    reqs.links = lib.manifest().links;
+    if (lib_plan.create_archive()) {
+        reqs.linkable_path = lib_plan.create_archive()->calc_archive_file_path(env);
+    }
+    // TODO: preprocessor definitions
+}
+
+using sdist_index_type = std::map<std::string, std::reference_wrapper<const sdist>>;
+using sdist_names      = std::set<std::string>;
+
+void add_sdist_to_build(build_plan&             plan,
+                        const sdist&            sd,
+                        const sdist_index_type& sd_idx,
+                        build_env_ref           env,
+                        usage_requirement_map&  ureqs,
+                        sdist_names&            already_added) {
+    if (already_added.find(sd.manifest.pk_id.name) != already_added.end()) {
+        // This one has already been added
+        return;
+    }
+    spdlog::debug("Adding dependent build: {}", sd.manifest.pk_id.name);
+    // Ensure that ever dependency is loaded up first)
+    for (const auto& dep : sd.manifest.dependencies) {
+        auto other = sd_idx.find(dep.name);
+        assert(other != sd_idx.end()
+               && "Failed to load a transitive dependency shortly after initializing them. What?");
+        add_sdist_to_build(plan, other->second, sd_idx, env, ureqs, already_added);
+    }
+    // Record that we have been processed
+    already_added.insert(sd.manifest.pk_id.name);
+    // Finally, actually add the package:
+    auto& pkg  = plan.add_package(package_plan(sd.manifest.pk_id.name, sd.manifest.namespace_));
+    auto  libs = collect_libraries(sd.path);
+    for (const auto& lib : libs) {
+        shared_compile_file_rules comp_rules = lib.base_compile_rules();
+        library_build_params      lib_params;
+        auto                      lib_plan = library_plan::create(lib, lib_params, ureqs);
+        // Create usage requirements for this libary.
+        add_ureqs(ureqs, sd, lib, lib_plan, env);
+        // Add it to the plan:
+        pkg.add_library(std::move(lib_plan));
+    }
+}
+
+void add_deps_to_build(build_plan&             plan,
+                       usage_requirement_map&  ureqs,
+                       const build_params&     params,
+                       const package_manifest& man,
+                       build_env_ref           env) {
+    auto sd_idx = params.dep_sdists  //
+        | ranges::views::transform([](const auto& sd) {
+                      return std::pair(sd.manifest.pk_id.name, std::cref(sd));
+                  })  //
+        | ranges::to<sdist_index_type>();
+
+    sdist_names already_added;
+    for (const sdist& sd : params.dep_sdists) {
+        add_sdist_to_build(plan, sd, sd_idx, env, ureqs, already_added);
+    }
+}
+
 }  // namespace
 
 void dds::build(const build_params& params, const package_manifest& man) {
+    fs::create_directories(params.out_root);
+    auto           db = database::open(params.out_root / ".dds.db");
+    dds::build_env env{params.toolchain, params.out_root, db};
+
+    // The build plan we will fill out:
+    build_plan plan;
+
+    // Collect libraries for the current project
     auto libs = collect_libraries(params.root);
     if (!libs.size()) {
         spdlog::warn("Nothing found to build!");
         return;
     }
 
-    build_plan plan;
-    auto&      pkg = plan.add_package(package_plan(man.pk_id.name, man.namespace_));
+    usage_requirement_map ureqs;
 
-    usage_requirement_map ureqs
-        = load_usage_requirements(params.root, params.out_root, params.lm_index);
+    if (params.existing_lm_index) {
+        ureqs = load_usage_requirements(params.root, params.out_root, *params.existing_lm_index);
+    } else {
+        add_deps_to_build(plan, ureqs, params, man, env);
+    }
+
+    // Initialize the build plan for this project.
+    auto& pkg = plan.add_package(package_plan(man.pk_id.name, man.namespace_));
+
+    // assert(false && "Not ready yet!");
 
     library_build_params lib_params;
     lib_params.build_tests     = params.build_tests;
     lib_params.build_apps      = params.build_apps;
     lib_params.enable_warnings = params.enable_warnings;
 
-    fs::create_directories(params.out_root);
-    auto           db = database::open(params.out_root / ".dds.db");
-    dds::build_env env{params.toolchain, params.out_root, db};
-
     if (man.test_driver) {
         prepare_test_driver(lib_params, params, man, env);
     }

src/dds/build/params.hpp

@@ -2,21 +2,24 @@
 
 #include <dds/toolchain/toolchain.hpp>
 #include <dds/util/fs.hpp>
+#include <dds/sdist.hpp>
+
+#include <optional>
 
 namespace dds {
 
 struct build_params {
-    fs::path       root;
-    fs::path       out_root;
-    fs::path       lm_index;
-    dds::toolchain toolchain;
-    bool           do_export       = false;
-    bool           build_tests     = false;
-    bool           enable_warnings = false;
-    bool           build_apps      = false;
-    bool           build_deps      = false;
-    bool           generate_compdb = true;
-    int            parallel_jobs   = 0;
+    fs::path                root;
+    fs::path                out_root;
+    std::optional<fs::path> existing_lm_index;
+    dds::toolchain          toolchain;
+    std::vector<sdist>      dep_sdists;
+    bool                    do_export       = false;
+    bool                    build_tests     = false;
+    bool                    enable_warnings = false;
+    bool                    build_apps      = false;
+    bool                    generate_compdb = true;
+    int                     parallel_jobs   = 0;
 };
 
 }  // namespace dds

src/dds/dds.main.cpp

@@ -276,6 +276,8 @@ struct cli_catalog {
             return get.run();
         } else if (add.cmd) {
             return add.run();
+        } else if (list.cmd) {
+            return list.run();
         } else {
             assert(false);
             std::terminate();
@@ -478,6 +480,14 @@ struct cli_build {
 
     common_project_flags project{cmd};
 
+    catalog_path_flag cat_path{cmd};
+    repo_path_flag    repo_path{cmd};
+
+    args::Flag do_download_deps{cmd,
+                                "download-deps",
+                                "Download any missing dependencies from the catalog",
+                                {"download-deps"}};
+
     args::Flag     build_tests{cmd, "build_tests", "Build and run the tests", {"tests", 'T'}};
     args::Flag     build_apps{cmd, "build_apps", "Build applications", {"apps", 'A'}};
     args::Flag     export_{cmd, "export", "Generate a library export", {"export", 'E'}};
@@ -488,11 +498,11 @@ struct cli_build {
                                "Enable compiler warnings",
                                {"warnings", 'W'}};
 
-    path_flag lm_index{cmd,
-                       "lm_index",
-                       "Path to a libman index (usually INDEX.lmi)",
-                       {"lm-index", 'I'},
-                       dds::fs::path()};
+    path_flag
+        lm_index{cmd,
+                 "lm_index",
+                 "Path to an existing libman index from which to load deps (usually INDEX.lmi)",
+                 {"lm-index", 'I'}};
 
     args::ValueFlag<int> num_jobs{cmd,
                                   "jobs",
@@ -516,12 +526,20 @@ struct cli_build {
         params.build_apps      = build_apps.Get();
         params.enable_warnings = enable_warnings.Get();
         params.parallel_jobs   = num_jobs.Get();
-        params.lm_index        = lm_index.Get();
         dds::package_manifest man;
         const auto            man_filepath = params.root / "package.dds";
         if (exists(man_filepath)) {
             man = dds::package_manifest::load_from_file(man_filepath);
         }
+        if (lm_index) {
+            params.existing_lm_index = lm_index.Get();
+        } else {
+            // Build the dependencies
+            params.dep_sdists = dds::repository::with_repository(  //
+                this->repo_path.Get(),
+                dds::repo_flags::read | dds::repo_flags::create_if_absent,
+                [&](dds::repository repo) { return repo.solve(man.dependencies); });
+        }
         dds::build(params, man);
         return 0;
     }
@@ -636,9 +654,7 @@ struct cli_deps {
             auto deps = dds::repository::with_repository(  //
                 repo_where.Get(),
                 dds::repo_flags::read,
-                [&](dds::repository repo) {
-                    return repo.solve(man.dependencies);
-                });
+                [&](dds::repository repo) { return repo.solve(man.dependencies); });
 
             auto tc   = tc_filepath.get_toolchain();
             auto bdir = build_dir.Get();