Merge branch 'feature/dep-tracking' into develop

library.dds

@@ -4,4 +4,5 @@ Uses: spdlog/spdlog
 Uses: Microsoft/wil
 Uses: Niebler/range-v3
 Uses: nlohmann/json
-Uses: neo/buffer
+Uses: neo/buffer
+Uses: neo/sqlite3

package.dds

@@ -6,5 +6,6 @@ Depends: spdlog 1.4.2
 Depends: ms-wil 2019.11.10
 Depends: range-v3 0.9.1
 Depends: nlohmann-json 3.7.1
+Depends: neo-sqlite3 0.2.0
 
 Test-Driver: Catch-Main

remote.dds

@@ -9,3 +9,4 @@ Remote-Package: ms-wil 2019.11.10; git url=https://github.com/vector-of-bool/wil
 
 # XXX: Don't depend on a moving revision!
 Remote-Package: neo-buffer 0.1.0; git url=https://github.com/vector-of-bool/neo-buffer.git ref=develop
+Remote-Package: neo-sqlite3 0.2.0; git url=https://github.com/vector-of-bool/neo-sqlite3.git ref=0.2.0

src/dds/build.cpp

@@ -7,9 +7,11 @@
 #include <dds/util/time.hpp>
 #include <libman/index.hpp>
 #include <libman/parse.hpp>
+#include <dds/build/plan/compile_exec.hpp>
 
 #include <spdlog/spdlog.h>
 
+#include <array>
 #include <stdexcept>
 
 using namespace dds;
@@ -116,16 +118,18 @@ load_usage_requirements(path_ref project_root, path_ref build_root, path_ref use
 void prepare_catch2_driver(library_build_params& lib_params,
                            test_lib              test_driver,
                            const build_params&   params,
-                           const package_manifest&) {
-    fs::path test_include_root = params.out_root / "_test_inc";
+                           build_env_ref         env_) {
+    fs::path test_include_root = params.out_root / "_catch-2.10.2";
     lib_params.test_include_dirs.emplace_back(test_include_root);
 
     auto catch_hpp = test_include_root / "catch2/catch.hpp";
-    fs::create_directories(catch_hpp.parent_path());
-    auto hpp_strm = open(catch_hpp, std::ios::out | std::ios::binary);
-    hpp_strm.write(detail::catch2_embedded_single_header_str,
-                   std::strlen(detail::catch2_embedded_single_header_str));
-    hpp_strm.close();
+    if (!fs::exists(catch_hpp)) {
+        fs::create_directories(catch_hpp.parent_path());
+        auto hpp_strm = open(catch_hpp, std::ios::out | std::ios::binary);
+        hpp_strm.write(detail::catch2_embedded_single_header_str,
+                    std::strlen(detail::catch2_embedded_single_header_str));
+        hpp_strm.close();
+    }
 
     if (test_driver == test_lib::catch_) {
         // Don't generate a test library helper
@@ -157,14 +161,14 @@ void prepare_catch2_driver(library_build_params& lib_params,
     assert(sf.has_value());
 
     compile_file_plan plan{comp_rules, std::move(*sf), "Catch2", "v1"};
-    build_env         env;
-    env.output_root = params.out_root / "_test-driver";
-    env.toolchain   = params.toolchain;
-    auto obj_file   = plan.calc_object_file_path(env);
+
+    build_env env2 = env_;
+    env2.output_root /= "_test-driver";
+    auto obj_file = plan.calc_object_file_path(env2);
 
     if (!fs::exists(obj_file)) {
         spdlog::info("Compiling Catch2 test driver (This will only happen once)...");
-        plan.compile(env);
+        compile_all(std::array{plan}, env2, 1);
     }
 
     lib_params.test_link_files.push_back(obj_file);
@@ -172,11 +176,12 @@ void prepare_catch2_driver(library_build_params& lib_params,
 
 void prepare_test_driver(library_build_params&   lib_params,
                          const build_params&     params,
-                         const package_manifest& man) {
+                         const package_manifest& man,
+                         build_env_ref           env) {
     auto& test_driver = *man.test_driver;
     if (test_driver == test_lib::catch_ || test_driver == test_lib::catch_main
         || test_driver == test_lib::catch_runner) {
-        prepare_catch2_driver(lib_params, test_driver, params, man);
+        prepare_catch2_driver(lib_params, test_driver, params, env);
     } else {
         assert(false && "Unreachable");
         std::terminate();
@@ -203,8 +208,12 @@ void dds::build(const build_params& params, const package_manifest& man) {
     lib_params.build_apps      = params.build_apps;
     lib_params.enable_warnings = params.enable_warnings;
 
+    fs::create_directories(params.out_root);
+    auto           db = database::open(params.out_root / ".dds.db");
+    dds::build_env env{params.toolchain, params.out_root, db};
+
     if (man.test_driver) {
-        prepare_test_driver(lib_params, params, man);
+        prepare_test_driver(lib_params, params, man, env);
     }
 
     for (const library& lib : libs) {
@@ -212,7 +221,6 @@ void dds::build(const build_params& params, const package_manifest& man) {
         pkg.add_library(library_plan::create(lib, lib_params, ureqs));
     }
 
-    dds::build_env env{params.toolchain, params.out_root};
     if (params.generate_compdb) {
         generate_compdb(plan, env);
     }

src/dds/build/deps.cpp

@@ -0,0 +1,102 @@
+#include "./deps.hpp"
+
+#include <dds/db/database.hpp>
+#include <dds/proc.hpp>
+#include <dds/util/shlex.hpp>
+#include <dds/util/string.hpp>
+
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/transform.hpp>
+#include <spdlog/spdlog.h>
+
+using namespace dds;
+
+deps_info dds::parse_mkfile_deps_file(path_ref where) {
+    auto content = slurp_file(where);
+    return parse_mkfile_deps_str(content);
+}
+
+deps_info dds::parse_mkfile_deps_str(std::string_view str) {
+    deps_info ret;
+
+    // Remove escaped newlines
+    auto no_newlines = replace(str, "\\\n", " ");
+
+    auto split = split_shell_string(str);
+    auto iter  = split.begin();
+    auto stop  = split.end();
+    if (iter == stop) {
+        spdlog::critical(
+            "Invalid deps listing. Shell split was empty. This is almost certainly a bug.");
+        return ret;
+    }
+    auto& head = *iter;
+    ++iter;
+    if (!ends_with(head, ":")) {
+        spdlog::critical(
+            "Invalid deps listing. Leader item is not colon-terminated. This is probably a bug. "
+            "(Are you trying to use C++ Modules? That's not ready yet, sorry. Set `Deps-Mode` to "
+            "`None` in your toolchain file.)");
+        return ret;
+    }
+    ret.output = head.substr(0, head.length() - 1);
+    ret.inputs.insert(ret.inputs.end(), iter, stop);
+    return ret;
+}
+
+msvc_deps_info dds::parse_msvc_output_for_deps(std::string_view output, std::string_view leader) {
+    auto lines = split_view(output, "\n");
+    std::string cleaned_output;
+    deps_info   deps;
+    for (const auto full_line : lines) {
+        auto trimmed = trim_view(full_line);
+        if (!starts_with(trimmed, leader)) {
+            cleaned_output += std::string(full_line);
+            cleaned_output.push_back('\n');
+            continue;
+        }
+        auto remaining = trim_view(trimmed.substr(leader.size()));
+        deps.inputs.emplace_back(fs::weakly_canonical(remaining));
+    }
+    if (!cleaned_output.empty()) {
+        // Remove the extra newline at the back
+        cleaned_output.pop_back();
+    }
+    return {deps, cleaned_output};
+}
+
+void dds::update_deps_info(database& db, const deps_info& deps) {
+    db.store_mtime(deps.output, fs::last_write_time(deps.output));
+    db.store_file_command(deps.output, {deps.command, deps.command_output});
+    db.forget_inputs_of(deps.output);
+    for (auto&& inp : deps.inputs) {
+        db.store_mtime(inp, fs::last_write_time(inp));
+        db.record_dep(inp, deps.output);
+    }
+}
+
+deps_rebuild_info dds::get_rebuild_info(database& db, path_ref output_path) {
+    std::unique_lock lk{db.mutex()};
+    auto             cmd_ = db.command_of(output_path);
+    if (!cmd_) {
+        return {};
+    }
+    auto& cmd     = *cmd_;
+    auto  inputs_ = db.inputs_of(output_path);
+    if (!inputs_) {
+        return {};
+    }
+    auto& inputs        = *inputs_;
+    auto  changed_files =  //
+        inputs             //
+        | ranges::views::filter([](const seen_file_info& input) {
+              return fs::last_write_time(input.path) != input.last_mtime;
+          })
+        | ranges::views::transform([](auto& info) { return info.path; })  //
+        | ranges::to_vector;
+    deps_rebuild_info ret;
+    ret.newer_inputs            = std::move(changed_files);
+    ret.previous_command        = cmd.command;
+    ret.previous_command_output = cmd.output;
+    return ret;
+}

src/dds/build/deps.hpp

@@ -0,0 +1,45 @@
+#pragma once
+
+#include <dds/util/fs.hpp>
+
+#include <string>
+#include <string_view>
+
+namespace dds {
+
+enum class deps_mode {
+    none,
+    msvc,
+    gnu,
+};
+
+struct deps_info {
+    fs::path              output;
+    std::vector<fs::path> inputs;
+    std::string           command;
+    std::string           command_output;
+};
+
+class database;
+
+deps_info parse_mkfile_deps_file(path_ref where);
+deps_info parse_mkfile_deps_str(std::string_view str);
+
+struct msvc_deps_info {
+    struct deps_info deps_info;
+    std::string      cleaned_output;
+};
+
+msvc_deps_info parse_msvc_output_for_deps(std::string_view output, std::string_view leader);
+
+void update_deps_info(database& db, const deps_info&);
+
+struct deps_rebuild_info {
+    std::vector<fs::path> newer_inputs;
+    std::string           previous_command;
+    std::string           previous_command_output;
+};
+
+deps_rebuild_info get_rebuild_info(database& db, path_ref output_path);
+
+}  // namespace dds

src/dds/build/deps.test.cpp

@@ -0,0 +1,50 @@
+#include <dds/build/deps.hpp>
+
+#include <catch2/catch.hpp>
+
+auto path_vec = [](auto... args) { return std::vector<dds::fs::path>{args...}; };
+
+TEST_CASE("Parse Makefile deps") {
+    auto deps = dds::parse_mkfile_deps_str("foo.o: bar.c");
+    CHECK(deps.output == "foo.o");
+    CHECK(deps.inputs == path_vec("bar.c"));
+    // Newline is okay
+    deps = dds::parse_mkfile_deps_str("foo.o: bar.c \\\n baz.c");
+    CHECK(deps.output == "foo.o");
+    CHECK(deps.inputs == path_vec("bar.c", "baz.c"));
+}
+
+TEST_CASE("Invalid deps") {
+    // Invalid deps does not terminate. This will generate an error message in
+    // the logs, but it is a non-fatal error that we can recover from.
+    auto deps = dds::parse_mkfile_deps_str("foo.o : cat");
+    CHECK(deps.output.empty());
+    CHECK(deps.inputs.empty());
+    deps = dds::parse_mkfile_deps_str("foo.c");
+    CHECK(deps.output.empty());
+    CHECK(deps.inputs.empty());
+}
+
+TEST_CASE("Parse MSVC deps") {
+    auto mscv_output = R"(
+Note: including file:    C:\foo\bar\filepath/thing.hpp
+Note: including file:  C:\foo\bar\filepath/baz.h
+Note: including file:      C:\foo\bar\filepath/quux.h
+    Note: including file:   C:\foo\bar\filepath/cats/quux.h
+Other line
+    indented line
+Something else
+)";
+
+    auto  res        = dds::parse_msvc_output_for_deps(mscv_output, "Note: including file:");
+    auto& deps       = res.deps_info;
+    auto  new_output = res.cleaned_output;
+    CHECK(new_output == "\nOther line\n    indented line\nSomething else\n");
+    CHECK(deps.inputs
+          == std::vector<dds::fs::path>({
+                 "C:\\foo\\bar\\filepath/thing.hpp",
+                 "C:\\foo\\bar\\filepath/baz.h",
+                 "C:\\foo\\bar\\filepath/quux.h",
+                 "C:\\foo\\bar\\filepath/cats/quux.h",
+             }));
+}

src/dds/build/plan/base.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <dds/db/database.hpp>
 #include <dds/toolchain/toolchain.hpp>
 #include <dds/util/fs.hpp>
 
@@ -8,6 +9,7 @@ namespace dds {
 struct build_env {
     dds::toolchain toolchain;
     fs::path       output_root;
+    database&      db;
 };
 
 using build_env_ref = const build_env&;

src/dds/build/plan/compile_exec.cpp

@@ -0,0 +1,209 @@
+#include "./compile_exec.hpp"
+
+#include <dds/build/deps.hpp>
+#include <dds/proc.hpp>
+#include <dds/util/string.hpp>
+#include <dds/util/time.hpp>
+
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/transform.hpp>
+#include <spdlog/spdlog.h>
+
+#include <algorithm>
+#include <cassert>
+#include <thread>
+
+using namespace dds;
+using namespace ranges;
+
+namespace {
+
+template <typename Range, typename Fn>
+bool parallel_run(Range&& rng, int n_jobs, Fn&& fn) {
+    // We don't bother with a nice thread pool, as the overhead of most build
+    // tasks dwarf the cost of interlocking.
+    std::mutex mut;
+
+    auto       iter = rng.begin();
+    const auto stop = rng.end();
+
+    std::vector<std::exception_ptr> exceptions;
+
+    auto run_one = [&]() mutable {
+        while (true) {
+            std::unique_lock lk{mut};
+            if (!exceptions.empty()) {
+                break;
+            }
+            if (iter == stop) {
+                break;
+            }
+            auto&& item = *iter;
+            ++iter;
+            lk.unlock();
+            try {
+                fn(item);
+            } catch (...) {
+                lk.lock();
+                exceptions.push_back(std::current_exception());
+                break;
+            }
+        }
+    };
+
+    std::unique_lock         lk{mut};
+    std::vector<std::thread> threads;
+    if (n_jobs < 1) {
+        n_jobs = std::thread::hardware_concurrency() + 2;
+    }
+    std::generate_n(std::back_inserter(threads), n_jobs, [&] { return std::thread(run_one); });
+    lk.unlock();
+    for (auto& t : threads) {
+        t.join();
+    }
+    for (auto eptr : exceptions) {
+        try {
+            std::rethrow_exception(eptr);
+        } catch (const std::exception& e) {
+            spdlog::error(e.what());
+        }
+    }
+    return exceptions.empty();
+}
+
+struct compile_file_full {
+    const compile_file_plan& plan;
+    fs::path                 object_file_path;
+    compile_command_info     cmd_info;
+};
+
+std::optional<deps_info> do_compile(const compile_file_full& cf, build_env_ref env) {
+    fs::create_directories(cf.object_file_path.parent_path());
+    auto source_path = cf.plan.source_path();
+    auto msg         = fmt::format("[{}] Compile: {:40}",
+                           cf.plan.qualifier(),
+                           fs::relative(source_path, cf.plan.source().basis_path).string());
+    spdlog::info(msg);
+    auto&& [dur_ms, proc_res]
+        = timed<std::chrono::milliseconds>([&] { return run_proc(cf.cmd_info.command); });
+    spdlog::info("{} - {:>7n}ms", msg, dur_ms.count());
+
+    const bool  compiled_okay   = proc_res.okay();
+    const auto  compile_retc    = proc_res.retc;
+    const auto  compile_signal  = proc_res.signal;
+    std::string compiler_output = std::move(proc_res.output);
+
+    std::optional<deps_info> ret_deps_info;
+
+    if (env.toolchain.deps_mode() == deps_mode::gnu) {
+        assert(cf.cmd_info.gnu_depfile_path.has_value());
+        auto& df_path = *cf.cmd_info.gnu_depfile_path;
+        if (!fs::is_regular_file(df_path)) {
+            spdlog::critical(
+                "The expected Makefile deps were not generated on disk. This is a bug! "
+                "(Expected "
+                "file to exist: [{}])",
+                df_path.string());
+        } else {
+            auto dep_info = dds::parse_mkfile_deps_file(df_path);
+            assert(dep_info.output == cf.object_file_path);
+            dep_info.command        = quote_command(cf.cmd_info.command);
+            dep_info.command_output = compiler_output;
+            ret_deps_info           = std::move(dep_info);
+        }
+    } else if (env.toolchain.deps_mode() == deps_mode::msvc) {
+        auto msvc_deps = parse_msvc_output_for_deps(compiler_output, "Note: including file:");
+        msvc_deps.deps_info.inputs.push_back(cf.plan.source_path());
+        msvc_deps.deps_info.output         = cf.object_file_path;
+        msvc_deps.deps_info.command        = quote_command(cf.cmd_info.command);
+        msvc_deps.deps_info.command_output = msvc_deps.cleaned_output;
+        ret_deps_info                      = std::move(msvc_deps.deps_info);
+        compiler_output                    = std::move(msvc_deps.cleaned_output);
+    }
+
+    // MSVC prints the filename of the source file. Dunno why, but they do.
+    if (compiler_output.find(source_path.filename().string()) == 0) {
+        compiler_output.erase(0, source_path.filename().string().length());
+        if (starts_with(compiler_output, "\r")) {
+            compiler_output.erase(0, 1);
+        }
+        if (starts_with(compiler_output, "\n")) {
+            compiler_output.erase(0, 1);
+        }
+    }
+
+    if (!compiled_okay) {
+        spdlog::error("Compilation failed: {}", source_path.string());
+        spdlog::error("Subcommand FAILED [Exitted {}]: {}\n{}",
+                      compile_retc,
+                      quote_command(cf.cmd_info.command),
+                      compiler_output);
+        if (compile_signal) {
+            spdlog::error("Process exited via signal {}", compile_signal);
+        }
+        throw compile_failure(fmt::format("Compilation failed for {}", source_path.string()));
+    }
+
+    if (!dds::trim_view(compiler_output).empty()) {
+        spdlog::warn("While compiling file {} [{}]:\n{}",
+                     source_path.string(),
+                     quote_command(cf.cmd_info.command),
+                     compiler_output);
+    }
+
+    // We must always generate deps info if it was possible:
+    assert(ret_deps_info.has_value() || env.toolchain.deps_mode() == deps_mode::none);
+    return ret_deps_info;
+}
+
+compile_file_full realize_plan(const compile_file_plan& plan, build_env_ref env) {
+    auto cmd_info = plan.generate_compile_command(env);
+    return compile_file_full{plan, plan.calc_object_file_path(env), cmd_info};
+}
+
+bool should_compile(const compile_file_full& comp, build_env_ref env) {
+    database& db      = env.db;
+    auto      rb_info = get_rebuild_info(db, comp.object_file_path);
+    if (rb_info.previous_command.empty()) {
+        // We have no previous compile command for this file. Assume it is new.
+        return true;
+    }
+    if (!rb_info.newer_inputs.empty()) {
+        // Inputs to this file have changed from a prior execution.
+        return true;
+    }
+    auto cur_cmd_str = quote_command(comp.cmd_info.command);
+    if (cur_cmd_str != rb_info.previous_command) {
+        // The command used to generate the output is new
+        return true;
+    }
+    // Nope. This file is up-to-date.
+    return false;
+}
+
+}  // namespace
+
+bool dds::detail::compile_all(const ref_vector<const compile_file_plan>& compiles,
+                              build_env_ref                              env,
+                              int                                        njobs) {
+    auto each_realized =                                                          //
+        compiles                                                                  //
+        | views::transform([&](auto&& plan) { return realize_plan(plan, env); })  //
+        | views::filter([&](auto&& real) { return should_compile(real, env); });
+
+    std::vector<deps_info> all_new_deps;
+    std::mutex             mut;
+    auto okay = parallel_run(each_realized, njobs, [&](const compile_file_full& full) {
+        auto new_dep = do_compile(full, env);
+        if (new_dep) {
+            std::unique_lock lk{mut};
+            all_new_deps.push_back(std::move(*new_dep));
+        }
+    });
+
+    auto tr = env.db.transaction();
+    for (auto& info : all_new_deps) {
+        update_deps_info(env.db, info);
+    }
+    return okay;
+}

src/dds/build/plan/compile_exec.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <dds/build/plan/base.hpp>
+#include <dds/build/plan/compile_file.hpp>
+#include <dds/util/algo.hpp>
+
+#include <functional>
+#include <vector>
+
+namespace dds {
+
+namespace detail {
+
+bool compile_all(const ref_vector<const compile_file_plan>& files, build_env_ref env, int njobs);
+
+}  // namespace detail
+
+template <typename Range>
+bool compile_all(Range&& rng, build_env_ref env, int njobs) {
+    ref_vector<const compile_file_plan> cfps;
+    for (auto&& cf : rng) {
+        cfps.push_back(cf);
+    }
+    return detail::compile_all(cfps, env, njobs);
+}
+
+}  // namespace dds

src/dds/build/plan/compile_file.cpp

@@ -12,8 +12,7 @@
 
 using namespace dds;
 
-std::vector<std::string> compile_file_plan::generate_compile_command(build_env_ref env) const
-    noexcept {
+compile_command_info compile_file_plan::generate_compile_command(build_env_ref env) const noexcept {
     compile_file_spec spec{_source.path, calc_object_file_path(env)};
     spec.enable_warnings = _rules.enable_warnings();
     extend(spec.include_dirs, _rules.include_dirs());
@@ -21,41 +20,9 @@ std::vector<std::string> compile_file_plan::generate_compile_command(build_env_r
     return env.toolchain.create_compile_command(spec);
 }
 
-void compile_file_plan::compile(const build_env& env) const {
-    const auto obj_path = calc_object_file_path(env);
-    fs::create_directories(obj_path.parent_path());
-
-    auto msg = fmt::format("[{}] Compile: {:40}",
-                           _qualifier,
-                           fs::relative(_source.path, _source.basis_path).string());
-
-    spdlog::info(msg);
-    auto cmd                     = generate_compile_command(env);
-    auto&& [dur_ms, compile_res] = timed<std::chrono::milliseconds>([&] { return run_proc(cmd); });
-    spdlog::info("{} - {:>7n}ms", msg, dur_ms.count());
-
-    if (!compile_res.okay()) {
-        spdlog::error("Compilation failed: {}", _source.path.string());
-        spdlog::error("Subcommand FAILED: {}\n{}", quote_command(cmd), compile_res.output);
-        throw compile_failure(fmt::format("Compilation failed for {}", _source.path.string()));
-    }
-
-    // MSVC prints the filename of the source file. Dunno why, but they do.
-    if (compile_res.output.find(_source.path.filename().string() + "\r\n") == 0) {
-        compile_res.output.erase(0, _source.path.filename().string().length() + 2);
-    }
-
-    if (!compile_res.output.empty()) {
-        spdlog::warn("While compiling file {} [{}]:\n{}",
-                     _source.path.string(),
-                     quote_command(cmd),
-                     compile_res.output);
-    }
-}
-
 fs::path compile_file_plan::calc_object_file_path(const build_env& env) const noexcept {
     auto relpath = fs::relative(_source.path, _source.basis_path);
     auto ret     = env.output_root / _subdir / relpath;
     ret.replace_filename(relpath.filename().string() + env.toolchain.object_suffix());
-    return ret;
+    return fs::weakly_canonical(ret);
 }

src/dds/build/plan/compile_file.hpp

@@ -55,13 +55,14 @@ public:
         , _qualifier(qual)
         , _subdir(subdir) {}
 
-    std::vector<std::string> generate_compile_command(build_env_ref) const noexcept;
-
     const source_file& source() const noexcept { return _source; }
     path_ref           source_path() const noexcept { return _source.path; }
+    auto&              rules() const noexcept { return _rules; }
+    auto&              qualifier() const noexcept { return _qualifier; }
 
-    fs::path calc_object_file_path(build_env_ref env) const noexcept;
-    void     compile(build_env_ref) const;
+    fs::path                 calc_object_file_path(build_env_ref env) const noexcept;
+    compile_command_info     generate_compile_command(build_env_ref) const noexcept;
+    std::optional<deps_info> compile(build_env_ref) const;
 };
 
 }  // namespace dds

src/dds/build/plan/full.cpp

@@ -1,6 +1,7 @@
 #include "./full.hpp"
 
 #include <dds/build/iter_compilations.hpp>
+#include <dds/build/plan/compile_exec.hpp>
 
 #include <range/v3/view/concat.hpp>
 #include <range/v3/view/filter.hpp>
@@ -36,7 +37,8 @@ bool parallel_run(Range&& rng, int n_jobs, Fn&& fn) {
             if (iter == stop) {
                 break;
             }
-            auto&& item = *iter++;
+            auto&& item = *iter;
+            ++iter;
             lk.unlock();
             try {
                 fn(item);
@@ -71,9 +73,7 @@ bool parallel_run(Range&& rng, int n_jobs, Fn&& fn) {
 }  // namespace
 
 void build_plan::compile_all(const build_env& env, int njobs) const {
-    auto okay = parallel_run(iter_compilations(*this), njobs, [&](const compile_file_plan& cf) {
-        cf.compile(env);
-    });
+    auto okay = dds::compile_all(iter_compilations(*this), env, njobs);
     if (!okay) {
         throw std::runtime_error("Compilation failed.");
     }

src/dds/compdb.cpp

@@ -12,10 +12,10 @@ void dds::generate_compdb(const build_plan& plan, build_env_ref env) {
     auto compdb = nlohmann::json::array();
 
     for (const compile_file_plan& cf : iter_compilations(plan)) {
-        auto command = cf.generate_compile_command(env);
+        auto cmd_info = cf.generate_compile_command(env);
         auto entry   = nlohmann::json::object({
             {"directory", env.output_root.string()},
-            {"arguments", command},
+            {"arguments", cmd_info.command},
             {"file", cf.source_path().string()},
         });
         compdb.push_back(std::move(entry));

src/dds/db/database.cpp

@@ -0,0 +1,245 @@
+#include "./database.hpp"
+
+#include <neo/sqlite3/exec.hpp>
+#include <neo/sqlite3/iter_tuples.hpp>
+#include <neo/sqlite3/single.hpp>
+#include <neo/sqlite3/transaction.hpp>
+
+#include <nlohmann/json.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/transform.hpp>
+#include <spdlog/spdlog.h>
+
+using namespace dds;
+
+namespace sqlite3 = neo::sqlite3;
+using sqlite3::exec;
+using namespace sqlite3::literals;
+
+namespace {
+
+void migrate_1(sqlite3::database& db) {
+    db.exec(R"(
+        CREATE TABLE dds_files (
+            file_id INTEGER PRIMARY KEY,
+            path TEXT NOT NULL UNIQUE,
+            mtime INTEGER NOT NULL
+        );
+        CREATE TABLE dds_deps (
+            input_file_id
+                INTEGER
+                NOT NULL
+                REFERENCES dds_files(file_id),
+            output_file_id
+                INTEGER
+                NOT NULL
+                REFERENCES dds_files(file_id),
+            UNIQUE(input_file_id, output_file_id)
+        );
+        CREATE TABLE dds_file_commands (
+            command_id INTEGER PRIMARY KEY,
+            file_id
+                INTEGER
+                UNIQUE
+                NOT NULL
+                REFERENCES dds_files(file_id),
+            command TEXT NOT NULL,
+            output TEXT NOT NULL
+        );
+    )");
+}
+
+void ensure_migrated(sqlite3::database& db) {
+    sqlite3::transaction_guard tr{db};
+    db.exec(R"(
+        PRAGMA foreign_keys = 1;
+        CREATE TABLE IF NOT EXISTS dds_meta AS
+            WITH init (meta) AS (VALUES ('{"version": 0}'))
+            SELECT * FROM init;
+        )");
+    auto meta_st     = db.prepare("SELECT meta FROM dds_meta");
+    auto [meta_json] = sqlite3::unpack_single<std::string>(meta_st);
+
+    auto meta = nlohmann::json::parse(meta_json);
+    if (!meta.is_object()) {
+        throw std::runtime_error("Correupted database file.");
+    }
+
+    auto version_ = meta["version"];
+    if (!version_.is_number_integer()) {
+        throw std::runtime_error("Corrupted database file [bad dds_meta.version]");
+    }
+    int version = version_;
+    if (version < 1) {
+        migrate_1(db);
+    }
+    meta["version"] = 1;
+    exec(db, "UPDATE dds_meta SET meta=?", std::forward_as_tuple(meta.dump()));
+}
+
+}  // namespace
+
+database database::open(const std::string& db_path) {
+    auto db = sqlite3::database::open(db_path);
+    try {
+        ensure_migrated(db);
+    } catch (const sqlite3::sqlite3_error& e) {
+        spdlog::error(
+            "Failed to load the databsae. It appears to be invalid/corrupted. We'll delete it and "
+            "create a new one. The exception message is: {}",
+            e.what());
+        fs::remove(db_path);
+        db = sqlite3::database::open(db_path);
+        try {
+            ensure_migrated(db);
+        } catch (const sqlite3::sqlite3_error& e) {
+            spdlog::critical(
+                "Failed to apply database migrations to recovery database. This is a critical "
+                "error. The exception message is: {}",
+                e.what());
+            std::terminate();
+        }
+    }
+    return database(std::move(db));
+}
+
+database::database(sqlite3::database db)
+    : _db(std::move(db)) {}
+
+std::optional<fs::file_time_type> database::last_mtime_of(path_ref file_) {
+    auto& st = _stmt_cache(R"(
+        SELECT mtime FROM dds_files WHERE path = ?
+    )"_sql);
+    st.reset();
+    auto path      = fs::weakly_canonical(file_);
+    st.bindings[1] = path.string();
+    auto maybe_res = sqlite3::unpack_single_opt<std::int64_t>(st);
+    if (!maybe_res) {
+        return std::nullopt;
+    }
+    auto [timestamp] = *maybe_res;
+    return fs::file_time_type(fs::file_time_type::duration(timestamp));
+}
+
+void database::store_mtime(path_ref file, fs::file_time_type time) {
+    auto& st = _stmt_cache(R"(
+        INSERT INTO dds_files (path, mtime)
+        VALUES (?1, ?2)
+        ON CONFLICT(path) DO UPDATE SET mtime = ?2
+    )"_sql);
+    sqlite3::exec(st,
+                  std::forward_as_tuple(fs::weakly_canonical(file).string(),
+                                        time.time_since_epoch().count()));
+}
+
+void database::record_dep(path_ref input, path_ref output) {
+    auto& st = _stmt_cache(R"(
+        WITH input AS (
+            SELECT file_id
+              FROM dds_files
+             WHERE path = ?1
+        ),
+        output AS (
+            SELECT file_id
+              FROM dds_files
+             WHERE path = ?2
+        )
+        INSERT OR IGNORE INTO dds_deps (input_file_id, output_file_id)
+        VALUES (
+            (SELECT * FROM input),
+            (SELECT * FROM output)
+        )
+    )"_sql);
+    sqlite3::exec(st,
+                  std::forward_as_tuple(fs::weakly_canonical(input).string(),
+                                        fs::weakly_canonical(output).string()));
+}
+
+void database::store_file_command(path_ref file, const command_info& cmd) {
+    auto& st = _stmt_cache(R"(
+        WITH file AS (
+            SELECT file_id
+            FROM dds_files
+            WHERE path = ?1
+        )
+        INSERT OR REPLACE
+          INTO dds_file_commands(file_id, command, output)
+        VALUES (
+            (SELECT * FROM file),
+            ?2,
+            ?3
+        )
+    )"_sql);
+    sqlite3::exec(st,
+                  std::forward_as_tuple(fs::weakly_canonical(file).string(),
+                                        std::string_view(cmd.command),
+                                        std::string_view(cmd.output)));
+}
+
+void database::forget_inputs_of(path_ref file) {
+    auto& st = _stmt_cache(R"(
+        WITH id_to_delete AS (
+            SELECT file_id
+            FROM dds_files
+            WHERE path = ?
+        )
+        DELETE FROM dds_deps
+         WHERE output_file_id IN id_to_delete
+    )"_sql);
+    sqlite3::exec(st, std::forward_as_tuple(fs::weakly_canonical(file).string()));
+}
+
+std::optional<std::vector<seen_file_info>> database::inputs_of(path_ref file_) {
+    auto  file = fs::weakly_canonical(file_);
+    auto& st   = _stmt_cache(R"(
+        WITH file AS (
+            SELECT file_id
+              FROM dds_files
+             WHERE path = ?
+        ),
+        input_ids AS (
+            SELECT input_file_id
+              FROM dds_deps
+             WHERE output_file_id IN file
+        )
+        SELECT path, mtime
+          FROM dds_files
+         WHERE file_id IN input_ids
+    )"_sql);
+    st.reset();
+    st.bindings[1] = file.string();
+    auto tup_iter  = sqlite3::iter_tuples<std::string, std::int64_t>(st);
+
+    std::vector<seen_file_info> ret;
+    for (auto& [path, mtime] : tup_iter) {
+        ret.emplace_back(
+            seen_file_info{path, fs::file_time_type(fs::file_time_type::duration(mtime))});
+    }
+
+    if (ret.empty()) {
+        return std::nullopt;
+    }
+    return ret;
+}
+
+std::optional<command_info> database::command_of(path_ref file_) {
+    auto  file = fs::weakly_canonical(file_);
+    auto& st   = _stmt_cache(R"(
+        WITH file AS (
+            SELECT file_id
+              FROM dds_files
+             WHERE path = ?
+        )
+        SELECT command, output
+          FROM dds_file_commands
+         WHERE file_id IN file
+    )"_sql);
+    st.reset();
+    st.bindings[1] = file.string();
+    auto opt_res   = sqlite3::unpack_single_opt<std::string, std::string>(st);
+    if (!opt_res) {
+        return std::nullopt;
+    }
+    auto& [cmd, out] = *opt_res;
+    return command_info{cmd, out};
+}

src/dds/db/database.hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <dds/util/fs.hpp>
+
+#include <neo/sqlite3/database.hpp>
+#include <neo/sqlite3/statement.hpp>
+#include <neo/sqlite3/statement_cache.hpp>
+#include <neo/sqlite3/transaction.hpp>
+
+#include <chrono>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <string_view>
+
+namespace dds {
+
+struct command_info {
+    std::string command;
+    std::string output;
+};
+
+struct seen_file_info {
+    fs::path           path;
+    fs::file_time_type last_mtime;
+};
+
+class database {
+    neo::sqlite3::database        _db;
+    neo::sqlite3::statement_cache _stmt_cache{_db};
+    mutable std::shared_mutex     _mutex;
+
+    explicit database(neo::sqlite3::database db);
+    database(const database&) = delete;
+
+public:
+    static database open(const std::string& db_path);
+    static database open(path_ref db_path) { return open(db_path.string()); }
+
+    auto& mutex() const noexcept { return _mutex; }
+
+    neo::sqlite3::transaction_guard transaction() noexcept {
+        return neo::sqlite3::transaction_guard(_db);
+    }
+
+    std::optional<fs::file_time_type> last_mtime_of(path_ref file);
+    void                              store_mtime(path_ref file, fs::file_time_type time);
+    void                              record_dep(path_ref input, path_ref output);
+    void                              store_file_command(path_ref file, const command_info& cmd);
+    void                              forget_inputs_of(path_ref file);
+
+    std::optional<std::vector<seen_file_info>> inputs_of(path_ref file);
+    std::optional<command_info>                command_of(path_ref file);
+};
+
+}  // namespace dds

src/dds/db/database.test.cpp

@@ -0,0 +1,21 @@
+#include <dds/db/database.hpp>
+
+#include <catch2/catch.hpp>
+
+using namespace std::literals;
+
+TEST_CASE("Create a database") { auto db = dds::database::open(":memory:"s); }
+TEST_CASE("Read an absent file's mtime") {
+    auto db        = dds::database::open(":memory:"s);
+    auto mtime_opt = db.last_mtime_of("bad/file/path");
+    CHECK_FALSE(mtime_opt.has_value());
+}
+
+TEST_CASE("Record a file") {
+    auto db = dds::database::open(":memory:"s);
+    auto time = dds::fs::file_time_type::clock::now();
+    db.store_mtime("file/something", time);
+    auto mtime_opt = db.last_mtime_of("file/something");
+    REQUIRE(mtime_opt.has_value());
+    CHECK(mtime_opt == time);
+}

src/dds/dds.main.cpp

@@ -467,9 +467,11 @@ struct cli_deps {
                                              man.dependencies.end());
                 });
 
-            auto           tc   = tc_filepath.get_toolchain();
-            auto           bdir = build_dir.Get();
-            dds::build_env env{std::move(tc), bdir};
+            auto tc   = tc_filepath.get_toolchain();
+            auto bdir = build_dir.Get();
+            dds::fs::create_directories(bdir);
+            auto           db = dds::database::open(bdir / ".dds.db");
+            dds::build_env env{std::move(tc), bdir, db};
 
             auto plan = dds::create_deps_build_plan(deps, env);
             plan.compile_all(env, 6);

src/dds/repo/remote.cpp

@@ -6,6 +6,7 @@
 #include <dds/sdist.hpp>
 #include <dds/temp.hpp>
 #include <dds/toolchain/toolchain.hpp>
+#include <dds/util/shlex.hpp>
 
 #include <spdlog/spdlog.h>
 

src/dds/toolchain/from_dds.cpp

@@ -3,6 +3,7 @@
 #include <dds/toolchain/prep.hpp>
 #include <dds/toolchain/toolchain.hpp>
 #include <dds/util/algo.hpp>
+#include <dds/util/shlex.hpp>
 #include <libman/parse.hpp>
 
 #include <spdlog/fmt/fmt.h>
@@ -90,6 +91,7 @@ toolchain dds::parse_toolchain_dds(const lm::pair_list& pairs, strv context) {
     opt_string     obj_suffix;
     opt_string     exe_prefix;
     opt_string     exe_suffix;
+    opt_string     deps_mode_str;
     optional<bool> do_debug;
     optional<bool> do_optimize;
     opt_str_seq    include_template;
@@ -128,6 +130,7 @@ toolchain dds::parse_toolchain_dds(const lm::pair_list& pairs, strv context) {
              lm::read_bool("Debug", do_debug),
              // Miscellaneous
              read_argv{"Compiler-Launcher", compile_launcher},
+             lm::read_opt("Deps-Mode", deps_mode_str),
              // Command templates
              read_argv{"C-Compile-File", c_compile_file},
              read_argv{"C++-Compile-File", cxx_compile_file},
@@ -170,6 +173,26 @@ toolchain dds::parse_toolchain_dds(const lm::pair_list& pairs, strv context) {
     bool is_msvc     = compiler_id_e == msvc;
     bool is_gnu_like = is_gnu || is_clang;
 
+    const enum deps_mode deps_mode = [&] {
+        if (!deps_mode_str.has_value()) {
+            if (is_gnu_like) {
+                return deps_mode::gnu;
+            } else if (is_msvc) {
+                return deps_mode::msvc;
+            } else {
+                return deps_mode::none;
+            }
+        } else if (deps_mode_str == "GNU") {
+            return deps_mode::gnu;
+        } else if (deps_mode_str == "MSVC") {
+            return deps_mode::msvc;
+        } else if (deps_mode_str == "None") {
+            return deps_mode::none;
+        } else {
+            fail(context, "Unknown Deps-Mode '{}'", *deps_mode_str);
+        }
+    }();
+
     // Now convert the flags we've been given into a real toolchain
     auto get_compiler = [&](language lang) -> string {
         if (lang == language::cxx && cxx_compiler_fpath) {
@@ -356,10 +379,10 @@ toolchain dds::parse_toolchain_dds(const lm::pair_list& pairs, strv context) {
                 rt_lib = "/MTd";
             }
             ret.emplace_back(rt_lib);
-            extend(ret, {"/nologo", "<FLAGS>", "/permissive-", "/c", "<IN>", "/Fo<OUT>"});
             if (lang == language::cxx) {
                 extend(ret, {"/EHsc"});
             }
+            extend(ret, {"/nologo", "/permissive-", "<FLAGS>", "/c", "<IN>", "/Fo<OUT>"});
         } else if (is_gnu_like) {
             if (do_optimize.has_value() && *do_optimize) {
                 extend(ret, {"-O2"});
@@ -382,6 +405,8 @@ toolchain dds::parse_toolchain_dds(const lm::pair_list& pairs, strv context) {
         return ret;
     };
 
+    tc.deps_mode = deps_mode;
+
     tc.c_compile = read_opt(c_compile_file, [&] {
         string_seq c;
         if (compile_launcher) {
@@ -408,7 +433,7 @@ toolchain dds::parse_toolchain_dds(const lm::pair_list& pairs, strv context) {
         }
         if (is_gnu_like) {
             return {"-I", "<PATH>"};
-        } else if (compiler_id == "MSVC") {
+        } else if (is_msvc) {
             return {"/I", "<PATH>"};
         }
         assert(false && "Include-Template deduction failed");
@@ -421,7 +446,7 @@ toolchain dds::parse_toolchain_dds(const lm::pair_list& pairs, strv context) {
         }
         if (is_gnu_like) {
             return {"-D", "<DEF>"};
-        } else if (compiler_id == "MSVC") {
+        } else if (is_msvc) {
             return {"/D", "<DEF>"};
         }
         assert(false && "Define-Template deduction failed");

src/dds/toolchain/from_dds.test.cpp

@@ -18,12 +18,12 @@ void check_tc_compile(std::string_view tc_content,
     cf.source_path  = "foo.cpp";
     cf.out_path     = "foo.o";
     auto cf_cmd     = tc.create_compile_command(cf);
-    auto cf_cmd_str = dds::quote_command(cf_cmd);
+    auto cf_cmd_str = dds::quote_command(cf_cmd.command);
     CHECK(cf_cmd_str == expected_compile);
 
     cf.enable_warnings = true;
     cf_cmd             = tc.create_compile_command(cf);
-    cf_cmd_str         = dds::quote_command(cf_cmd);
+    cf_cmd_str         = dds::quote_command(cf_cmd.command);
     CHECK(cf_cmd_str == expected_compile_warnings);
 
     dds::archive_spec ar_spec;
@@ -45,39 +45,41 @@ void check_tc_compile(std::string_view tc_content,
 
 TEST_CASE("Generating toolchain commands") {
     check_tc_compile("Compiler-ID: GNU",
-                     "g++ -fPIC -fdiagnostics-color -pthread -c foo.cpp -ofoo.o",
+                     "g++ -fPIC -fdiagnostics-color -pthread -MD -MF foo.o.d -MT foo.o -c foo.cpp -ofoo.o",
                      "g++ -fPIC -fdiagnostics-color -pthread -Wall -Wextra -Wpedantic -Wconversion "
-                     "-c foo.cpp -ofoo.o",
+                     "-MD -MF foo.o.d -MT foo.o -c foo.cpp -ofoo.o",
                      "ar rcs stuff.a foo.o bar.o",
                      "g++ -fPIC -fdiagnostics-color foo.o bar.a -pthread -lstdc++fs -omeow.exe");
 
     check_tc_compile(
         "Compiler-ID: GNU\nDebug: True",
-        "g++ -g -fPIC -fdiagnostics-color -pthread -c foo.cpp -ofoo.o",
+        "g++ -g -fPIC -fdiagnostics-color -pthread -MD -MF foo.o.d -MT foo.o -c foo.cpp -ofoo.o",
         "g++ -g -fPIC -fdiagnostics-color -pthread -Wall -Wextra -Wpedantic -Wconversion "
-        "-c foo.cpp -ofoo.o",
+        "-MD -MF foo.o.d -MT foo.o -c foo.cpp -ofoo.o",
         "ar rcs stuff.a foo.o bar.o",
         "g++ -fPIC -fdiagnostics-color foo.o bar.a -pthread -lstdc++fs -omeow.exe -g");
 
     check_tc_compile(
         "Compiler-ID: GNU\nDebug: True\nOptimize: True",
-        "g++ -O2 -g -fPIC -fdiagnostics-color -pthread -c foo.cpp -ofoo.o",
+        "g++ -O2 -g -fPIC -fdiagnostics-color -pthread -MD -MF foo.o.d -MT foo.o -c foo.cpp "
+        "-ofoo.o",
         "g++ -O2 -g -fPIC -fdiagnostics-color -pthread -Wall -Wextra -Wpedantic -Wconversion "
-        "-c foo.cpp -ofoo.o",
+        "-MD -MF foo.o.d -MT foo.o -c foo.cpp -ofoo.o",
         "ar rcs stuff.a foo.o bar.o",
         "g++ -fPIC -fdiagnostics-color foo.o bar.a -pthread -lstdc++fs -omeow.exe -O2 -g");
 
     check_tc_compile("Compiler-ID: MSVC",
-                     "cl.exe /MT /nologo /permissive- /c foo.cpp /Fofoo.o /EHsc",
-                     "cl.exe /MT /nologo /W4 /permissive- /c foo.cpp /Fofoo.o /EHsc",
+                     "cl.exe /MT /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o",
+                     "cl.exe /MT /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o",
                      "lib /nologo /OUT:stuff.a foo.o bar.o",
                      "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /MT");
 
-    check_tc_compile("Compiler-ID: MSVC\nDebug: True",
-                     "cl.exe /Z7 /DEBUG /MTd /nologo /permissive- /c foo.cpp /Fofoo.o /EHsc",
-                     "cl.exe /Z7 /DEBUG /MTd /nologo /W4 /permissive- /c foo.cpp /Fofoo.o /EHsc",
-                     "lib /nologo /OUT:stuff.a foo.o bar.o",
-                     "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /Z7 /DEBUG /MTd");
+    check_tc_compile(
+        "Compiler-ID: MSVC\nDebug: True",
+        "cl.exe /Z7 /DEBUG /MTd /EHsc /nologo /permissive- /showIncludes /c foo.cpp /Fofoo.o",
+        "cl.exe /Z7 /DEBUG /MTd /EHsc /nologo /permissive- /W4 /showIncludes /c foo.cpp /Fofoo.o",
+        "lib /nologo /OUT:stuff.a foo.o bar.o",
+        "cl.exe /nologo /EHsc foo.o bar.a /Femeow.exe /Z7 /DEBUG /MTd");
 
     auto tc = dds::parse_toolchain_dds(R"(
     Compiler-ID: GNU
@@ -87,31 +89,41 @@ TEST_CASE("Generating toolchain commands") {
     cfs.source_path = "foo.cpp";
     cfs.out_path    = "foo.o";
     auto cmd        = tc.create_compile_command(cfs);
-    CHECK(cmd
+    CHECK(cmd.command
           == std::vector<std::string>{"g++",
                                       "-fPIC",
                                       "-fdiagnostics-color",
                                       "-pthread",
+                                      "-MD",
+                                      "-MF",
+                                      "foo.o.d",
+                                      "-MT",
+                                      "foo.o",
                                       "-c",
                                       "foo.cpp",
                                       "-ofoo.o"});
 
     cfs.definitions.push_back("FOO=BAR");
     cmd = tc.create_compile_command(cfs);
-    CHECK(cmd
+    CHECK(cmd.command
           == std::vector<std::string>{"g++",
                                       "-fPIC",
                                       "-fdiagnostics-color",
                                       "-pthread",
                                       "-D",
                                       "FOO=BAR",
+                                      "-MD",
+                                      "-MF",
+                                      "foo.o.d",
+                                      "-MT",
+                                      "foo.o",
                                       "-c",
                                       "foo.cpp",
                                       "-ofoo.o"});
 
     cfs.include_dirs.push_back("fake-dir");
     cmd = tc.create_compile_command(cfs);
-    CHECK(cmd
+    CHECK(cmd.command
           == std::vector<std::string>{"g++",
                                       "-fPIC",
                                       "-fdiagnostics-color",
@@ -120,6 +132,11 @@ TEST_CASE("Generating toolchain commands") {
                                       "fake-dir",
                                       "-D",
                                       "FOO=BAR",
+                                      "-MD",
+                                      "-MF",
+                                      "foo.o.d",
+                                      "-MT",
+                                      "foo.o",
                                       "-c",
                                       "foo.cpp",
                                       "-ofoo.o"});

src/dds/toolchain/prep.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <dds/build/deps.hpp>
+
 #include <string>
 #include <vector>
 
@@ -24,6 +26,8 @@ struct toolchain_prep {
     std::string exe_prefix;
     std::string exe_suffix;
 
+    enum deps_mode deps_mode;
+
     toolchain realize() const;
 };
 

src/dds/toolchain/toolchain.cpp

@@ -1,5 +1,6 @@
 #include "./toolchain.hpp"
 
+#include <dds/toolchain/from_dds.hpp>
 #include <dds/toolchain/prep.hpp>
 #include <dds/util/algo.hpp>
 #include <dds/util/string.hpp>
@@ -32,65 +33,10 @@ toolchain toolchain::realize(const toolchain_prep& prep) {
     ret._object_suffix  = prep.object_suffix;
     ret._exe_prefix     = prep.exe_prefix;
     ret._exe_suffix     = prep.exe_suffix;
+    ret._deps_mode      = prep.deps_mode;
     return ret;
 }
 
-vector<string> dds::split_shell_string(std::string_view shell) {
-    char cur_quote  = 0;
-    bool is_escaped = false;
-
-    vector<string> acc;
-
-    const auto begin = shell.begin();
-    auto       iter  = begin;
-    const auto end   = shell.end();
-
-    opt_string token;
-    while (iter != end) {
-        const char c = *iter++;
-        if (is_escaped) {
-            if (c == '\n') {
-                // Ignore the newline
-            } else if (cur_quote || c != cur_quote || c == '\\') {
-                // Escaped `\` character
-                token = token.value_or("") + c;
-            } else {
-                // Regular escape sequence
-                token = token.value_or("") + '\\' + c;
-            }
-            is_escaped = false;
-        } else if (c == '\\') {
-            is_escaped = true;
-        } else if (cur_quote) {
-            if (c == cur_quote) {
-                // End of quoted token;
-                cur_quote = 0;
-            } else {
-                token = token.value_or("") + c;
-            }
-        } else if (c == '"' || c == '\'') {
-            // Beginning of a quoted token
-            cur_quote = c;
-            token     = "";
-        } else if (c == '\t' || c == ' ' || c == '\n' || c == '\r' || c == '\f') {
-            // We've reached unquoted whitespace
-            if (token.has_value()) {
-                acc.push_back(move(*token));
-            }
-            token.reset();
-        } else {
-            // Just a regular character
-            token = token.value_or("") + c;
-        }
-    }
-
-    if (token.has_value()) {
-        acc.push_back(move(*token));
-    }
-
-    return acc;
-}
-
 vector<string> toolchain::include_args(const fs::path& p) const noexcept {
     return replace(_inc_template, "<PATH>", p.string());
 }
@@ -99,9 +45,12 @@ vector<string> toolchain::definition_args(std::string_view s) const noexcept {
     return replace(_def_template, "<DEF>", s);
 }
 
-vector<string> toolchain::create_compile_command(const compile_file_spec& spec) const noexcept {
+compile_command_info toolchain::create_compile_command(const compile_file_spec& spec) const
+    noexcept {
     vector<string> flags;
 
+    using namespace std::literals;
+
     language lang = spec.lang;
     if (lang == language::automatic) {
         if (spec.source_path.extension() == ".c" || spec.source_path.extension() == ".C") {
@@ -127,6 +76,21 @@ vector<string> toolchain::create_compile_command(const compile_file_spec& spec)
         extend(flags, _warning_flags);
     }
 
+    std::optional<fs::path> gnu_depfile_path;
+
+    if (_deps_mode == deps_mode::gnu) {
+        gnu_depfile_path = spec.out_path;
+        gnu_depfile_path->replace_extension(gnu_depfile_path->extension().string() + ".d");
+        extend(flags,
+               {"-MD"sv,
+                "-MF"sv,
+                std::string_view(gnu_depfile_path->string()),
+                "-MT"sv,
+                std::string_view(spec.out_path.string())});
+    } else if (_deps_mode == deps_mode::msvc) {
+        flags.push_back("/showIncludes");
+    }
+
     vector<string> command;
     for (auto arg : cmd_template) {
         if (arg == "<FLAGS>") {
@@ -137,7 +101,7 @@ vector<string> toolchain::create_compile_command(const compile_file_spec& spec)
             command.push_back(arg);
         }
     }
-    return command;
+    return {command, gnu_depfile_path};
 }
 
 vector<string> toolchain::create_archive_command(const archive_spec& spec) const noexcept {
@@ -170,116 +134,89 @@ vector<string> toolchain::create_link_executable_command(const link_exe_spec& sp
     return cmd;
 }
 
-std::optional<toolchain> toolchain::get_builtin(std::string_view s) noexcept {
-    toolchain ret;
-
+std::optional<toolchain> toolchain::get_builtin(std::string_view tc_id) noexcept {
     using namespace std::literals;
 
-    if (starts_with(s, "ccache:")) {
-        s = s.substr("ccache:"sv.length());
-        ret._c_compile.push_back("ccache");
-        ret._cxx_compile.push_back("ccache");
-    }
-
-    if (starts_with(s, "gcc") || starts_with(s, "clang")) {
-        ret._inc_template   = {"-isystem", "<PATH>"};
-        ret._def_template   = {"-D", "<DEF>"};
-        ret._archive_suffix = ".a";
-        ret._object_suffix  = ".o";
-        ret._warning_flags  = {"-Wall", "-Wextra"};
-        ret._link_archive   = {"ar", "rcs", "<OUT>", "<IN>"};
+    std::string tc_content;
 
-        std::vector<std::string> common_flags = {
-            "<FLAGS>",
-            "-g",
-            "-fPIC",
-            "-fdiagnostics-color",
-            "-pthread",
-            "-c",
-            "-o",
-            "<OUT>",
-            "<IN>",
-        };
-        std::vector<std::string> c_flags;
-        std::vector<std::string> cxx_flags = {"-std=c++17"};
-
-        std::string c_compiler_base;
-        std::string cxx_compiler_base;
-        std::string compiler_suffix;
+    if (starts_with(tc_id, "ccache:")) {
+        tc_id = tc_id.substr("ccache:"sv.length());
+        tc_content += "Compiler-Launcher: ccache\n";
+    }
 
-        if (starts_with(s, "gcc")) {
-            c_compiler_base   = "gcc";
-            cxx_compiler_base = "g++";
-            common_flags.push_back("-O0");
-        } else if (starts_with(s, "clang")) {
-            c_compiler_base   = "clang";
-            cxx_compiler_base = "clang++";
+#define CXX_VER_TAG(str, version)                                                                  \
+    if (starts_with(tc_id, str)) {                                                         \
+        tc_id = tc_id.substr(std::string_view(str).length());                                      \
+        tc_content += "C++-Version: "s + version + "\n";                                                  \
+    }                                                                                              \
+    static_assert(true)
+
+    CXX_VER_TAG("c++98:", "C++98");
+    CXX_VER_TAG("c++03:", "C++03");
+    CXX_VER_TAG("c++11:", "C++11");
+    CXX_VER_TAG("c++14:", "C++14");
+    CXX_VER_TAG("c++17:", "C++17");
+    CXX_VER_TAG("c++20:", "C++20");
+
+    struct compiler_info {
+        string c;
+        string cxx;
+        string id;
+    };
+
+    auto opt_triple = [&]() -> std::optional<compiler_info> {
+        if (starts_with(tc_id, "gcc") || starts_with(tc_id, "clang")) {
+            const bool is_gcc   = starts_with(tc_id, "gcc");
+            const bool is_clang = starts_with(tc_id, "clang");
+
+            const auto [c_compiler_base, cxx_compiler_base, compiler_id] = [&]() -> compiler_info {
+                if (is_gcc) {
+                    return {"gcc", "g++", "GNU"};
+                } else if (is_clang) {
+                    return {"clang", "clang++", "Clang"};
+                }
+                assert(false && "Unreachable");
+                std::terminate();
+            }();
+
+            const auto compiler_suffix = [&]() -> std::string {
+                if (ends_with(tc_id, "-7")) {
+                    return "-7";
+                } else if (ends_with(tc_id, "-8")) {
+                    return "-8";
+                } else if (ends_with(tc_id, "-9")) {
+                    return "-9";
+                } else if (ends_with(tc_id, "-10")) {
+                    return "-10";
+                } else if (ends_with(tc_id, "-11")) {
+                    return "-11";
+                } else if (ends_with(tc_id, "-12")) {
+                    return "-12";
+                } else if (ends_with(tc_id, "-13")) {
+                    return "-13";
+                }
+                return "";
+            }();
+
+            auto c_compiler_name = c_compiler_base + compiler_suffix;
+            if (c_compiler_name != tc_id) {
+                return std::nullopt;
+            }
+            auto cxx_compiler_name = cxx_compiler_base + compiler_suffix;
+            return compiler_info{c_compiler_name, cxx_compiler_name, compiler_id};
+        } else if (tc_id == "msvc") {
+            return compiler_info{"cl.exe", "cl.exe", "MSVC"};
         } else {
-            assert(false && "Unreachable");
-            std::terminate();
-        }
-
-        if (ends_with(s, "-7")) {
-            compiler_suffix = "-7";
-        } else if (ends_with(s, "-8")) {
-            compiler_suffix = "-8";
-        } else if (ends_with(s, "-9")) {
-            compiler_suffix = "-9";
-        } else if (ends_with(s, "-10")) {
-            compiler_suffix = "-10";
-        }
-
-        auto c_compiler_name = c_compiler_base + compiler_suffix;
-        if (c_compiler_name != s) {
             return std::nullopt;
         }
-        auto cxx_compiler_name = cxx_compiler_base + compiler_suffix;
-
-        ret._c_compile.push_back(c_compiler_name);
-        extend(ret._c_compile, common_flags);
-        extend(ret._c_compile, c_flags);
+    }();
 
-        ret._cxx_compile.push_back(cxx_compiler_name);
-        extend(ret._cxx_compile, common_flags);
-        extend(ret._cxx_compile, cxx_flags);
-
-        ret._link_exe.push_back(cxx_compiler_name);
-        extend(ret._link_exe,
-               {
-                   "-g",
-                   "-fPIC",
-                   "-fdiagnostics-color",
-                   "<IN>",
-                   "-pthread",
-                   "-lstdc++fs",
-                   "-o",
-                   "<OUT>",
-               });
-    } else if (s == "msvc") {
-        ret._inc_template = {"/I<PATH>"};
-        ret._def_template = {"/D<DEF>"};
-        ret._c_compile    = {"cl.exe", "/nologo", "<FLAGS>", "/c", "<IN>", "/Fo<OUT>"};
-        ret._cxx_compile  = {"cl.exe",
-                            "/nologo",
-                            "<FLAGS>",
-                            "/std:c++latest",
-                            "/permissive-",
-                            "/EHsc",
-                            "/c",
-                            "<IN>",
-                            "/Fo<OUT>"};
-        std::vector<std::string_view> common_flags = {"/Z7", "/O2", "/MT", "/DEBUG"};
-        extend(ret._c_compile, common_flags);
-        extend(ret._cxx_compile, common_flags);
-        ret._archive_suffix = ".lib";
-        ret._object_suffix  = ".obj";
-        ret._exe_suffix     = ".exe";
-        ret._link_archive   = {"lib", "/nologo", "/OUT:<OUT>", "<IN>"};
-        ret._link_exe       = {"cl.exe", "/nologo", "/std:c++latest", "/EHsc", "<IN>", "/Fe<OUT>"};
-        ret._warning_flags  = {"/W4"};
-    } else {
+    if (!opt_triple) {
         return std::nullopt;
     }
 
-    return ret;
+    tc_content += "C-Compiler: "s + opt_triple->c + "\n";
+    tc_content += "C++-Compiler: "s + opt_triple->cxx + "\n";
+    tc_content += "Compiler-ID: " + opt_triple->id + "\n";
+    return parse_toolchain_dds(tc_content);
 }

src/dds/toolchain/toolchain.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <dds/build/deps.hpp>
 #include <dds/util/fs.hpp>
 
 #include <optional>
@@ -9,8 +10,6 @@
 
 namespace dds {
 
-std::vector<std::string> split_shell_string(std::string_view s);
-
 enum class language {
     automatic,
     c,
@@ -26,6 +25,11 @@ struct compile_file_spec {
     bool                     enable_warnings = false;
 };
 
+struct compile_command_info {
+    std::vector<std::string> command;
+    std::optional<fs::path>  gnu_depfile_path;
+};
+
 struct archive_spec {
     std::vector<fs::path> input_files;
     fs::path              out_path;
@@ -56,6 +60,8 @@ class toolchain {
     std::string _exe_prefix;
     std::string _exe_suffix;
 
+    enum deps_mode _deps_mode;
+
 public:
     toolchain() = default;
 
@@ -64,10 +70,11 @@ public:
     auto& archive_suffix() const noexcept { return _archive_suffix; }
     auto& object_suffix() const noexcept { return _object_suffix; }
     auto& executable_suffix() const noexcept { return _exe_suffix; }
+    auto  deps_mode() const noexcept { return _deps_mode; }
 
     std::vector<std::string> definition_args(std::string_view s) const noexcept;
     std::vector<std::string> include_args(const fs::path& p) const noexcept;
-    std::vector<std::string> create_compile_command(const compile_file_spec&) const noexcept;
+    compile_command_info     create_compile_command(const compile_file_spec&) const noexcept;
     std::vector<std::string> create_archive_command(const archive_spec&) const noexcept;
     std::vector<std::string> create_link_executable_command(const link_exe_spec&) const noexcept;
 

src/dds/util/algo.hpp

@@ -2,6 +2,8 @@
 
 #include <algorithm>
 #include <initializer_list>
+#include <vector>
+#include <functional>
 
 namespace dds {
 
@@ -33,4 +35,7 @@ void extend(Container& c, std::initializer_list<Item> il) {
     c.insert(c.end(), il.begin(), il.end());
 }
 
+template <typename T>
+using ref_vector = std::vector<std::reference_wrapper<T>>;
+
 }  // namespace dds

src/dds/util/shlex.cpp

@@ -0,0 +1,66 @@
+#include "./shlex.hpp"
+
+#include <optional>
+#include <string>
+#include <utility>
+
+using std::string;
+using std::vector;
+
+using namespace dds;
+
+vector<string> dds::split_shell_string(std::string_view shell) {
+    char cur_quote  = 0;
+    bool is_escaped = false;
+
+    vector<string> acc;
+
+    const auto begin = shell.begin();
+    auto       iter  = begin;
+    const auto end   = shell.end();
+
+    std::optional<string> token;
+    while (iter != end) {
+        const char c = *iter++;
+        if (is_escaped) {
+            if (c == '\n') {
+                // Ignore the newline
+            } else if (cur_quote || c != cur_quote || c == '\\') {
+                // Escaped `\` character
+                token = token.value_or("") + c;
+            } else {
+                // Regular escape sequence
+                token = token.value_or("") + '\\' + c;
+            }
+            is_escaped = false;
+        } else if (c == '\\') {
+            is_escaped = true;
+        } else if (cur_quote) {
+            if (c == cur_quote) {
+                // End of quoted token;
+                cur_quote = 0;
+            } else {
+                token = token.value_or("") + c;
+            }
+        } else if (c == '"' || c == '\'') {
+            // Beginning of a quoted token
+            cur_quote = c;
+            token     = "";
+        } else if (c == '\t' || c == ' ' || c == '\n' || c == '\r' || c == '\f') {
+            // We've reached unquoted whitespace
+            if (token.has_value()) {
+                acc.push_back(move(*token));
+            }
+            token.reset();
+        } else {
+            // Just a regular character
+            token = token.value_or("") + c;
+        }
+    }
+
+    if (token.has_value()) {
+        acc.push_back(move(*token));
+    }
+
+    return acc;
+}

src/dds/util/shlex.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace dds {
+
+std::vector<std::string> split_shell_string(std::string_view s);
+
+}  // namespace dds

src/dds/toolchain/toolchain.test.cpp → src/dds/util/shlex.test.cpp

@@ -1,4 +1,4 @@
-#include <dds/toolchain/toolchain.hpp>
+#include <dds/util/shlex.hpp>
 
 #include <catch2/catch.hpp>
 

src/dds/util/string.hpp

@@ -41,6 +41,18 @@ inline bool starts_with(std::string_view s, std::string_view key) { return s.fin
 
 inline bool contains(std::string_view s, std::string_view key) { return s.find(key) != s.npos; }
 
+inline std::vector<std::string_view> split_view(std::string_view str, std::string_view sep) {
+    std::vector<std::string_view> ret;
+    std::string_view::size_type   prev_pos = 0;
+    auto                          pos      = prev_pos;
+    while ((pos = str.find(sep, prev_pos)) != str.npos) {
+        ret.emplace_back(str.substr(prev_pos, pos - prev_pos));
+        prev_pos = pos + sep.length();
+    }
+    ret.emplace_back(str.substr(prev_pos));
+    return ret;
+}
+
 inline std::vector<std::string> split(std::string_view str, std::string_view sep) {
     std::vector<std::string>    ret;
     std::string_view::size_type prev_pos = 0;

tools/gcc-8.dds

@@ -1,6 +1,9 @@
 Compiler-ID: GNU
 C++-Version: C++17
+C-Compiler: gcc-8
 C++-Compiler: g++-8
-Flags: -D SPDLOG_COMPILED_LIB -fconcepts
+Flags: -D SPDLOG_COMPILED_LIB -fconcepts -Werror=return-type
 Optimize: True
+# Debug:  True
+# Link-Flags: -fsanitize=address
 Compiler-Launcher: ccache