Working on a new modularized build plan system

src/dds/build.cpp

@@ -1,6 +1,6 @@
 #include "./build.hpp"
 
-#include <dds/compile.hpp>
+#include <dds/build/compile.hpp>
 #include <dds/logging.hpp>
 #include <dds/proc.hpp>
 #include <dds/project.hpp>
@@ -50,15 +50,6 @@ auto iter_libraries(const project& pr) {
     return libs;
 }
 
-fs::path object_file_path(fs::path source_path, const build_params& params) {
-    auto obj_dir     = params.out_root / "obj";
-    auto obj_relpath = fs::relative(source_path, params.root);
-    obj_relpath.replace_filename(obj_relpath.filename().string()
-                                 + params.toolchain.object_suffix());
-    auto obj_path = obj_dir / obj_relpath;
-    return obj_path;
-}
-
 fs::path lib_archive_path(const build_params& params, const library& lib) {
     return params.out_root
         / (fmt::format("lib{}{}", lib.name(), params.toolchain.archive_suffix()));
@@ -183,8 +174,8 @@ void include_deps(const lm::index::library_index& lib_index,
     }
 }
 
-std::vector<file_compilation> file_compilations_of_lib(const build_params& params,
-                                                       const library&      lib) {
+std::vector<compile_file_plan> file_compilations_of_lib(const build_params& params,
+                                                        const library&      lib) {
     const auto& sources = lib.sources();
 
     std::vector<fs::path>    dep_includes;
@@ -224,30 +215,26 @@ std::vector<file_compilation> file_compilations_of_lib(const build_params& param
                 || (sf.kind == source_kind::test && params.build_tests));
     };
 
-    compilation_rules rules;
-    rules.base_path() = lib.base_dir() / "src";
+    shared_compile_file_rules rules;
     extend(rules.defs(), lib.manifest().private_defines);
     extend(rules.defs(), dep_defines);
     extend(rules.include_dirs(), lib.manifest().private_includes);
     extend(rules.include_dirs(), dep_includes);
     rules.include_dirs().push_back(fs::absolute(lib.base_dir() / "src"));
     rules.include_dirs().push_back(fs::absolute(lib.base_dir() / "include"));
+    rules.enable_warnings() = params.enable_warnings;
 
     return                               //
         sources                          //
         | filter(should_compile_source)  //
         | transform([&](auto&& src) {
-              return file_compilation{rules,
-                                      src,
-                                      object_file_path(src.path, params),
-                                      lib.name(),
-                                      params.enable_warnings};
+              return compile_file_plan{rules, src, lib.name()};
           })  //
         | to_vector;
 }
 
-std::vector<dds::file_compilation> collect_compiles(const build_params& params,
-                                                    const project&      project) {
+std::vector<dds::compile_file_plan> collect_compiles(const build_params& params,
+                                                     const project&      project) {
     auto libs = iter_libraries(project);
     return                                                                              //
         libs                                                                            //
@@ -401,13 +388,16 @@ link_project_lib(const build_params& params, const library& lib, const object_fi
     return res;
 }
 
-std::vector<link_results> link_project(const build_params&                  params,
-                                       const project&                       pr,
-                                       const std::vector<file_compilation>& compilations) {
-    auto obj_index =                                                                    //
-        ranges::views::all(compilations)                                                //
-        | transform([](auto&& comp) { return std::pair(comp.source.path, comp.obj); })  //
-        | ranges::to<object_file_index>()                                               //
+std::vector<link_results> link_project(const build_params&                   params,
+                                       const project&                        pr,
+                                       const std::vector<compile_file_plan>& compilations) {
+    auto obj_index =                      //
+        ranges::views::all(compilations)  //
+        | transform([&](const compile_file_plan& comp) -> std::pair<fs::path, fs::path> {
+              return std::pair(comp.source.path,
+                               params.out_root / comp.get_object_file_path(params.toolchain));
+          })                               //
+        | ranges::to<object_file_index>()  //
         ;
 
     auto libs = iter_libraries(pr);
@@ -423,7 +413,7 @@ void dds::build(const build_params& params, const package_manifest&) {
 
     auto compiles = collect_compiles(params, project);
 
-    dds::execute_all(compiles, params.toolchain, params.parallel_jobs);
+    dds::execute_all(compiles, params.toolchain, params.parallel_jobs, params.out_root);
 
     using namespace ranges::views;
 

src/dds/build.hpp

@@ -1,29 +1,13 @@
-#ifndef DDS_BUILD_HPP_INCLUDED
-#define DDS_BUILD_HPP_INCLUDED
+#pragma once
 
+#include <dds/build/params.hpp>
 #include <dds/package_manifest.hpp>
-#include <dds/toolchain.hpp>
 #include <dds/util/fs.hpp>
 
 #include <optional>
 
 namespace dds {
 
-struct build_params {
-    fs::path       root;
-    fs::path       out_root;
-    fs::path       lm_index;
-    dds::toolchain toolchain;
-    bool           do_export       = false;
-    bool           build_tests     = false;
-    bool           enable_warnings = false;
-    bool           build_apps      = false;
-    bool           build_deps      = false;
-    int            parallel_jobs   = 0;
-};
-
 void build(const build_params&, const package_manifest& man);
 
 }  // namespace dds
-
-#endif  // DDS_BUILD_HPP_INCLUDED

src/dds/build/archive.cpp

@@ -0,0 +1 @@
+#include "./archive.hpp"

src/dds/build/archive.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <dds/toolchain.hpp>
+#include <dds/util/fs.hpp>
+
+#include <vector>
+
+namespace dds {
+
+struct archive_rules {
+    std::vector<fs::path> objects;
+    fs::path              out;
+
+    void create_archive(const toolchain& tc) const;
+};
+
+}  // namespace dds

src/dds/compile.cpp → src/dds/build/compile.cpp

@@ -13,16 +13,17 @@
 
 using namespace dds;
 
-void file_compilation::compile(const toolchain& tc) const {
-    fs::create_directories(obj.parent_path());
+void compile_file_plan::compile(const toolchain& tc, path_ref out_prefix) const {
+    const auto obj_path = out_prefix / get_object_file_path(tc);
+    fs::create_directories(obj_path.parent_path());
 
     spdlog::info("[{}] Compile: {}",
-                 owner_name,
-                 fs::relative(source.path, rules.base_path()).string());
+                 qualifier,
+                 fs::relative(source.path, source.basis_path).string());
     auto start_time = std::chrono::steady_clock::now();
 
-    compile_file_spec spec{source.path, obj};
-    spec.enable_warnings = enable_warnings;
+    compile_file_spec spec{source.path, obj_path};
+    spec.enable_warnings = rules.enable_warnings();
 
     extend(spec.include_dirs, rules.include_dirs());
     extend(spec.definitions, rules.defs());
@@ -34,8 +35,8 @@ void file_compilation::compile(const toolchain& tc) const {
     auto dur_ms   = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
 
     spdlog::info("[{}] Compile: {} - {:n}ms",
-                 owner_name,
-                 fs::relative(source.path, rules.base_path()).string(),
+                 qualifier,
+                 fs::relative(source.path, source.basis_path).string(),
                  dur_ms.count());
 
     if (!compile_res.okay()) {
@@ -57,9 +58,16 @@ void file_compilation::compile(const toolchain& tc) const {
     }
 }
 
-void dds::execute_all(const std::vector<file_compilation>& compilations,
-                      const toolchain&                     tc,
-                      int                                  n_jobs) {
+fs::path compile_file_plan::get_object_file_path(const toolchain& tc) const noexcept {
+    auto relpath = fs::relative(source.path, source.basis_path);
+    relpath.replace_filename(relpath.filename().string() + tc.object_suffix());
+    return relpath;
+}
+
+void dds::execute_all(const std::vector<compile_file_plan>& compilations,
+                      const toolchain&                      tc,
+                      int                                   n_jobs,
+                      path_ref                              out_prefix) {
     // We don't bother with a nice thread pool, as the overhead of compiling
     // source files dwarfs the cost of interlocking.
     std::mutex mut;
@@ -81,7 +89,7 @@ void dds::execute_all(const std::vector<file_compilation>& compilations,
             auto& compilation = *comp_iter++;
             lk.unlock();
             try {
-                compilation.compile(tc);
+                compilation.compile(tc, out_prefix);
                 cancellation_point();
             } catch (...) {
                 lk.lock();
@@ -112,4 +120,4 @@ void dds::execute_all(const std::vector<file_compilation>& compilations,
     if (!exceptions.empty()) {
         throw compile_failure("Failed to compile library sources");
     }
-}
+}

src/dds/build/compile.hpp

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <dds/source.hpp>
+#include <dds/toolchain.hpp>
+#include <dds/util/fs.hpp>
+
+#include <memory>
+#include <optional>
+#include <stdexcept>
+
+namespace dds {
+
+struct compile_failure : std::runtime_error {
+    using runtime_error::runtime_error;
+};
+
+struct shared_compile_file_rules {
+    struct rules_impl {
+        std::vector<fs::path>    inc_dirs;
+        std::vector<std::string> defs;
+        bool                     enable_warnings = false;
+    };
+
+    std::shared_ptr<rules_impl> _impl = std::make_shared<rules_impl>();
+
+public:
+    shared_compile_file_rules() = default;
+
+    auto& include_dirs() noexcept { return _impl->inc_dirs; }
+    auto& include_dirs() const noexcept { return _impl->inc_dirs; }
+
+    auto& defs() noexcept { return _impl->defs; }
+    auto& defs() const noexcept { return _impl->defs; }
+
+    auto& enable_warnings() noexcept { return _impl->enable_warnings; }
+    auto& enable_warnings() const noexcept { return _impl->enable_warnings; }
+};
+
+struct compile_file_plan {
+    shared_compile_file_rules rules;
+    dds::source_file          source;
+    std::string               qualifier;
+
+    fs::path get_object_file_path(const toolchain& tc) const noexcept;
+    void     compile(const toolchain& tc, path_ref out_prefix) const;
+};
+
+void execute_all(const std::vector<compile_file_plan>&,
+                 const toolchain& tc,
+                 int              n_jobs,
+                 path_ref         out_prefix);
+
+}  // namespace dds

src/dds/build/executable.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <dds/util/fs.hpp>
+
+#include <vector>
+
+namespace dds {
+
+class link_executable_rules {
+    std::vector<fs::path> inputs;
+    fs::path              output;
+};
+
+}  // namespace dds

src/dds/build/params.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <dds/toolchain.hpp>
+#include <dds/util/fs.hpp>
+
+namespace dds {
+
+struct build_params {
+    fs::path       root;
+    fs::path       out_root;
+    fs::path       lm_index;
+    dds::toolchain toolchain;
+    bool           do_export       = false;
+    bool           build_tests     = false;
+    bool           enable_warnings = false;
+    bool           build_apps      = false;
+    bool           build_deps      = false;
+    int            parallel_jobs   = 0;
+};
+
+}  // namespace dds

src/dds/build/plan.cpp

@@ -0,0 +1,144 @@
+#include "./plan.hpp"
+
+#include <range/v3/action/join.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/join.hpp>
+#include <range/v3/view/repeat_n.hpp>
+#include <range/v3/view/transform.hpp>
+#include <range/v3/view/zip.hpp>
+
+#include <spdlog/spdlog.h>
+
+#include <mutex>
+#include <thread>
+
+using namespace dds;
+
+void build_plan::add_sroot(const sroot& root, const sroot_build_params& params) {
+    create_libraries.push_back(library_plan::create(root, params));
+}
+
+library_plan library_plan::create(const sroot& root, const sroot_build_params& params) {
+    std::vector<compile_file_plan>   compile_files;
+    std::vector<create_archive_plan> create_archives;
+    std::vector<create_exe_plan>     link_executables;
+
+    std::vector<source_file> app_sources;
+    std::vector<source_file> test_sources;
+    std::vector<source_file> lib_sources;
+
+    auto src_dir = root.src_dir();
+    if (src_dir.exists()) {
+        auto all_sources = src_dir.sources();
+        auto to_compile  = all_sources | ranges::views::filter([&](const source_file& sf) {
+                              return (sf.kind == source_kind::source
+                                      || (sf.kind == source_kind::app && params.build_apps)
+                                      || (sf.kind == source_kind::test && params.build_tests));
+                          });
+
+        for (const auto& sfile : to_compile) {
+            compile_file_plan cf_plan;
+            cf_plan.source    = sfile;
+            cf_plan.qualifier = params.main_name;
+            cf_plan.rules     = params.compile_rules;
+            compile_files.push_back(std::move(cf_plan));
+            if (sfile.kind == source_kind::test) {
+                test_sources.push_back(sfile);
+            } else if (sfile.kind == source_kind::app) {
+                app_sources.push_back(sfile);
+            } else {
+                lib_sources.push_back(sfile);
+            }
+        }
+    }
+
+    if (!app_sources.empty() || !test_sources.empty()) {
+        assert(false && "Apps/tests not implemented on this code path");
+    }
+
+    if (!lib_sources.empty()) {
+        create_archive_plan ar_plan;
+        ar_plan.in_sources = lib_sources                                   //
+            | ranges::views::transform([](auto&& sf) { return sf.path; })  //
+            | ranges::to_vector;
+        ar_plan.name    = params.main_name;
+        ar_plan.out_dir = params.out_dir;
+        create_archives.push_back(std::move(ar_plan));
+    }
+
+    return library_plan{compile_files, create_archives, link_executables, params.out_dir};
+}
+
+namespace {
+
+template <typename Range, typename Fn>
+bool parallel_run(Range&& rng, int n_jobs, Fn&& fn) {
+    // We don't bother with a nice thread pool, as the overhead of most build
+    // tasks dwarf the cost of interlocking.
+    std::mutex mut;
+
+    auto       iter = rng.begin();
+    const auto stop = rng.end();
+
+    std::vector<std::exception_ptr> exceptions;
+
+    auto run_one = [&]() mutable {
+        while (true) {
+            std::unique_lock lk{mut};
+            if (!exceptions.empty()) {
+                break;
+            }
+            if (iter == stop) {
+                break;
+            }
+            auto&& item = *iter++;
+            lk.unlock();
+            try {
+                fn(item);
+            } catch (...) {
+                lk.lock();
+                exceptions.push_back(std::current_exception());
+                break;
+            }
+        }
+    };
+
+    std::unique_lock         lk{mut};
+    std::vector<std::thread> threads;
+    if (n_jobs < 1) {
+        n_jobs = std::thread::hardware_concurrency() + 2;
+    }
+    std::generate_n(std::back_inserter(threads), n_jobs, [&] { return std::thread(run_one); });
+    lk.unlock();
+    for (auto& t : threads) {
+        t.join();
+    }
+    for (auto eptr : exceptions) {
+        try {
+            std::rethrow_exception(eptr);
+        } catch (const std::exception& e) {
+            spdlog::error(e.what());
+        }
+    }
+    return exceptions.empty();
+}
+
+}  // namespace
+
+void build_plan::compile_all(const toolchain& tc, int njobs, path_ref out_prefix) const {
+    std::vector<std::pair<fs::path, std::reference_wrapper<const compile_file_plan>>> comps;
+    for (const auto& lib : create_libraries) {
+        const auto lib_out_prefix = out_prefix / lib.out_subdir;
+        for (auto&& cf_plan : lib.compile_files) {
+            comps.emplace_back(lib_out_prefix, cf_plan);
+        }
+    }
+
+    auto okay = parallel_run(comps, njobs, [&](const auto& pair) {
+        const auto& [out_dir, cf_plan] = pair;
+        cf_plan.get().compile(tc, out_dir);
+    });
+    if (!okay) {
+        throw std::runtime_error("Compilation failed.");
+    }
+}

src/dds/build/plan.hpp

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <dds/build/compile.hpp>
+#include <dds/build/params.hpp>
+#include <dds/build/sroot.hpp>
+
+#include <dds/toolchain.hpp>
+#include <dds/util/fs.hpp>
+
+namespace dds {
+
+struct create_archive_plan {
+    std::vector<fs::path> in_sources;
+    std::string           name;
+    fs::path              out_dir;
+};
+
+struct create_exe_plan {
+    std::vector<fs::path> in_sources;
+    std::string           name;
+    fs::path              out_dir;
+};
+
+struct library_plan {
+    std::vector<compile_file_plan>   compile_files;
+    std::vector<create_archive_plan> create_archives;
+    std::vector<create_exe_plan>     link_executables;
+    fs::path                         out_subdir;
+
+    static library_plan create(const sroot& root, const sroot_build_params& params);
+};
+
+struct build_plan {
+    std::vector<library_plan> create_libraries;
+
+    // static build_plan generate(const build_params& params);
+    void add_sroot(const sroot& root, const sroot_build_params& params);
+
+    void compile_all(const toolchain& tc, int njobs, path_ref out_prefix) const;
+};
+
+}  // namespace dds

src/dds/build/source_dir.cpp

@@ -0,0 +1,24 @@
+#include "./source_dir.hpp"
+
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/transform.hpp>
+
+using namespace dds;
+
+std::vector<source_file> source_directory::sources() const {
+    using namespace ranges::views;
+    // Strips nullopt elements and lifts the value from the results
+    auto drop_nulls =                                       //
+        filter([](auto&& opt) { return opt.has_value(); })  //
+        | transform([](auto&& opt) { return *opt; });       //
+
+    // Collect all source files from the directory
+    return                                                                              //
+        fs::recursive_directory_iterator(path)                                          //
+        | filter([](auto&& entry) { return entry.is_regular_file(); })                  //
+        | transform([&](auto&& entry) { return source_file::from_path(entry, path); })  //
+        // source_file::from_path returns an optional. Drop nulls
+        | drop_nulls  //
+        | ranges::to_vector;
+}

src/dds/build/source_dir.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <dds/source.hpp>
+#include <dds/util/fs.hpp>
+
+#include <vector>
+
+namespace dds {
+
+struct source_directory {
+    fs::path path;
+
+    std::vector<source_file> sources() const;
+
+    bool exists() const noexcept { return fs::exists(path); }
+};
+
+}  // namespace dds

src/dds/build/sroot.cpp

@@ -0,0 +1,24 @@
+#include "./sroot.hpp"
+
+using namespace dds;
+
+shared_compile_file_rules sroot::base_compile_rules() const noexcept {
+    auto                      inc_dir = include_dir();
+    auto                      src_dir = this->src_dir();
+    shared_compile_file_rules ret;
+    if (inc_dir.exists()) {
+        ret.include_dirs().push_back(inc_dir.path);
+    }
+    if (src_dir.exists()) {
+        ret.include_dirs().push_back(src_dir.path);
+    }
+    return ret;
+}
+
+fs::path sroot::public_include_dir() const noexcept {
+    auto inc_dir = include_dir();
+    if (inc_dir.exists()) {
+        return inc_dir.path;
+    }
+    return src_dir().path;
+}

src/dds/build/sroot.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <dds/build/compile.hpp>
+#include <dds/build/source_dir.hpp>
+#include <dds/util/fs.hpp>
+
+#include <vector>
+
+namespace dds {
+
+struct sroot {
+    fs::path path;
+
+    source_directory src_dir() const noexcept { return source_directory{path / "src"}; };
+    source_directory include_dir() const noexcept { return source_directory{path / "include"}; }
+
+    fs::path public_include_dir() const noexcept;
+
+    shared_compile_file_rules base_compile_rules() const noexcept;
+};
+
+struct sroot_build_params {
+    std::string               main_name;
+    fs::path                  out_dir;
+    bool                      build_tests = false;
+    bool                      build_apps  = false;
+    std::vector<fs::path>     rt_link_libraries;
+    shared_compile_file_rules compile_rules;
+};
+
+}  // namespace dds

src/dds/compile.hpp

@@ -1,51 +0,0 @@
-#pragma once
-
-#include <dds/source.hpp>
-#include <dds/toolchain.hpp>
-#include <dds/util/fs.hpp>
-
-#include <memory>
-#include <optional>
-#include <stdexcept>
-
-namespace dds {
-
-struct compile_failure : std::runtime_error {
-    using runtime_error::runtime_error;
-};
-
-class compilation_rules {
-    struct rules_impl {
-        std::vector<fs::path>    inc_dirs;
-        std::vector<std::string> defs;
-        fs::path                 base_path;
-    };
-
-    std::shared_ptr<rules_impl> _impl = std::make_shared<rules_impl>();
-
-public:
-    compilation_rules() = default;
-
-    auto&       base_path() noexcept { return _impl->base_path; }
-    const auto& base_path() const noexcept { return _impl->base_path; }
-
-    auto&       include_dirs() noexcept { return _impl->inc_dirs; }
-    const auto& include_dirs() const noexcept { return _impl->inc_dirs; }
-
-    auto&       defs() noexcept { return _impl->defs; }
-    const auto& defs() const noexcept { return _impl->defs; }
-};
-
-struct file_compilation {
-    compilation_rules rules;
-    source_file       source;
-    fs::path          obj;
-    std::string       owner_name;
-    bool              enable_warnings = false;
-
-    void compile(const toolchain& tc) const;
-};
-
-void execute_all(const std::vector<file_compilation>&, const toolchain& tc, int n_jobs);
-
-}  // namespace dds

src/dds/ddslim.main.cpp

@@ -1,4 +1,5 @@
 #include <dds/build.hpp>
+#include <dds/build/plan.hpp>
 #include <dds/logging.hpp>
 #include <dds/repo/repo.hpp>
 #include <dds/sdist.hpp>
@@ -359,19 +360,6 @@ struct cli_deps {
 
         toolchain_flag tc_filepath{cmd};
 
-        void _build_one_dep(const dds::sdist& dep) {
-            spdlog::info("Build dependency {} {}",
-                         dep.manifest.name,
-                         dep.manifest.version.to_string());
-            dds::build_params params;
-            params.root      = dep.path;
-            params.toolchain = tc_filepath.get_toolchain();
-            params.out_root  = build_dir.Get()
-                / fmt::format("{}-{}", dep.manifest.name, dep.manifest.version.to_string());
-
-            dds::build(params, dep.manifest);
-        }
-
         int run() {
             auto man  = parent.load_package_manifest();
             auto deps = dds::repository::with_repository(  //
@@ -382,9 +370,9 @@ struct cli_deps {
                                              man.dependencies.begin(),
                                              man.dependencies.end());
                 });
-            for (auto&& dep : deps) {
-                _build_one_dep(dep);
-            }
+
+            auto plan = dds::create_deps_build_plan(deps);
+            plan.compile_all(tc_filepath.get_toolchain(), 6, build_dir.Get());
             return 0;
         }
     } build{*this};

src/dds/deps.cpp

@@ -1,13 +1,16 @@
 #include "./deps.hpp"
 
-#include <dds/sdist.hpp>
+#include <dds/build/sroot.hpp>
 #include <dds/repo/repo.hpp>
+#include <dds/sdist.hpp>
 #include <dds/util/string.hpp>
 
-#include <range/v3/algorithm/partition_point.hpp>
-#include <spdlog/fmt/fmt.h>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/transform.hpp>
+#include <spdlog/spdlog.h>
 
 #include <cctype>
+#include <map>
 
 using namespace dds;
 
@@ -51,7 +54,10 @@ void detail::do_find_deps(const repository& repo, const dependency& dep, std::ve
                         dep.version.to_string()));
     }
     sdist& new_sd = *sdist_opt;
-    auto insert_point = ranges::partition_point(sd, [&](const sdist& cand) {
+    for (const auto& inner_dep : new_sd.manifest.dependencies) {
+        do_find_deps(repo, inner_dep, sd);
+    }
+    auto insert_point = std::partition_point(sd.begin(), sd.end(), [&](const sdist& cand) {
         return cand.path < new_sd.path;
     });
     if (insert_point != sd.end() && insert_point->manifest.name == new_sd.manifest.name) {
@@ -62,4 +68,50 @@ void detail::do_find_deps(const repository& repo, const dependency& dep, std::ve
         return;
     }
     sd.insert(insert_point, std::move(new_sd));
-}
+}
+
+using sdist_index_type = std::map<std::string, std::reference_wrapper<const sdist>>;
+
+namespace {
+
+void add_dep_includes(shared_compile_file_rules& rules,
+                      const package_manifest&    man,
+                      const sdist_index_type&    sd_idx) {
+    for (const dependency& dep : man.dependencies) {
+        auto found = sd_idx.find(dep.name);
+        if (found == sd_idx.end()) {
+            throw std::runtime_error(
+                fmt::format("Unable to resolve dependency '{}' (required by '{}')",
+                            dep.name,
+                            man.name));
+        }
+        add_dep_includes(rules, found->second.get().manifest, sd_idx);
+        rules.include_dirs().push_back(sroot{found->second.get().path}.public_include_dir());
+    }
+}
+
+void add_sdist_to_dep_plan(build_plan& plan, const sdist& sd, const sdist_index_type& sd_idx) {
+    auto                      root       = dds::sroot{sd.path};
+    shared_compile_file_rules comp_rules = root.base_compile_rules();
+    add_dep_includes(comp_rules, sd.manifest, sd_idx);
+    sroot_build_params params;
+    params.main_name     = sd.manifest.name;
+    params.compile_rules = comp_rules;
+    plan.add_sroot(root, params);
+}
+
+}  // namespace
+
+build_plan dds::create_deps_build_plan(const std::vector<sdist>& deps) {
+    auto sd_idx = deps | ranges::views::transform([](const auto& sd) {
+                      return std::pair(sd.manifest.name, std::cref(sd));
+                  })
+        | ranges::to<sdist_index_type>();
+
+    build_plan plan;
+    for (const sdist& sd : deps) {
+        spdlog::info("Recording dependency: {}", sd.manifest.name);
+        add_sdist_to_dep_plan(plan, sd, sd_idx);
+    }
+    return plan;
+}

src/dds/deps.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <dds/build/plan.hpp>
+
 #include <semver/version.hpp>
 
 #include <string_view>
@@ -27,7 +29,7 @@ namespace detail {
 
 void do_find_deps(const repository&, const dependency& dep, std::vector<sdist>& acc);
 
-} // namespace detail
+}  // namespace detail
 
 std::vector<sdist> find_dependencies(const repository& repo, const dependency& dep);
 
@@ -40,4 +42,6 @@ inline std::vector<sdist> find_dependencies(const repository& repo, Iter it, Snt
     return acc;
 }
 
+build_plan create_deps_build_plan(const std::vector<sdist>& deps);
+
 }  // namespace dds

src/dds/library.cpp

@@ -1,5 +1,6 @@
 #include <dds/library.hpp>
 
+#include <dds/build/source_dir.hpp>
 #include <dds/util/algo.hpp>
 
 #include <spdlog/spdlog.h>
@@ -15,16 +16,16 @@ struct pf_info {
 };
 
 pf_info collect_pf_sources(path_ref path) {
-    auto include_dir = path / "include";
-    auto src_dir     = path / "src";
+    auto include_dir = source_directory{path / "include"};
+    auto src_dir     = source_directory{path / "src"};
 
     source_list sources;
 
-    if (fs::exists(include_dir)) {
-        if (!fs::is_directory(include_dir)) {
+    if (include_dir.exists()) {
+        if (!fs::is_directory(include_dir.path)) {
             throw std::runtime_error("The `include` at the root of the project is not a directory");
         }
-        auto inc_sources = source_file::collect_for_dir(include_dir);
+        auto inc_sources = include_dir.sources();
         // Drop any source files we found within `include/`
         erase_if(sources, [&](auto& info) {
             if (info.kind != source_kind::header) {
@@ -37,15 +38,15 @@ pf_info collect_pf_sources(path_ref path) {
         extend(sources, inc_sources);
     }
 
-    if (fs::exists(src_dir)) {
-        if (!fs::is_directory(src_dir)) {
+    if (src_dir.exists()) {
+        if (!fs::is_directory(src_dir.path)) {
             throw std::runtime_error("The `src` at the root of the project is not a directory");
         }
-        auto src_sources = source_file::collect_for_dir(src_dir);
+        auto src_sources = src_dir.sources();
         extend(sources, src_sources);
     }
 
-    return {std::move(sources), include_dir, src_dir};
+    return {std::move(sources), include_dir.path, src_dir.path};
 }
 
 }  // namespace

src/dds/sdist.hpp

@@ -21,10 +21,10 @@ struct sdist {
     browns::md5::digest_type md5;
     fs::path                 path;
 
-    sdist(package_manifest man, browns::md5::digest_type hash, path_ref path)
+    sdist(package_manifest man, browns::md5::digest_type hash, path_ref path_)
         : manifest(std::move(man))
         , md5(hash)
-        , path(path) {}
+        , path(path_) {}
 
     static sdist from_directory(path_ref p);
 

src/dds/source.cpp

@@ -4,10 +4,6 @@
 
 #include <spdlog/spdlog.h>
 
-#include <range/v3/range/conversion.hpp>
-#include <range/v3/view/filter.hpp>
-#include <range/v3/view/transform.hpp>
-
 #include <algorithm>
 #include <optional>
 #include <vector>
@@ -58,28 +54,11 @@ std::optional<source_kind> dds::infer_source_kind(path_ref p) noexcept {
     return source_kind::source;
 }
 
-std::optional<source_file> source_file::from_path(path_ref path) noexcept {
+std::optional<source_file> source_file::from_path(path_ref path, path_ref base_path) noexcept {
     auto kind = infer_source_kind(path);
     if (!kind.has_value()) {
         return std::nullopt;
     }
 
-    return source_file{path, *kind};
-}
-
-source_list source_file::collect_for_dir(path_ref src) {
-    using namespace ranges::views;
-    // Strips nullopt elements and lifts the value from the results
-    auto drop_nulls =                                       //
-        filter([](auto&& opt) { return opt.has_value(); })  //
-        | transform([](auto&& opt) { return *opt; });       //
-
-    // Collect all source files from the directory
-    return                                                                       //
-        fs::recursive_directory_iterator(src)                                    //
-        | filter([](auto&& entry) { return entry.is_regular_file(); })           //
-        | transform([](auto&& entry) { return source_file::from_path(entry); })  //
-        // source_file::from_path returns an optional. Drop nulls
-        | drop_nulls  //
-        | ranges::to_vector;
+    return source_file{path, base_path, *kind};
 }

src/dds/source.hpp

@@ -18,10 +18,10 @@ std::optional<source_kind> infer_source_kind(path_ref) noexcept;
 
 struct source_file {
     fs::path    path;
+    fs::path    basis_path;
     source_kind kind;
 
-    static std::optional<source_file> from_path(path_ref) noexcept;
-    static std::vector<source_file>   collect_for_dir(path_ref);
+    static std::optional<source_file> from_path(path_ref path, path_ref base_path) noexcept;
 };
 
 using source_list = std::vector<source_file>;