Persist dependency information in an SQLite database, and use it do incremental compilation

library.dds

@@ -4,4 +4,5 @@ Uses: spdlog/spdlog
 Uses: Microsoft/wil
 Uses: Niebler/range-v3
 Uses: nlohmann/json
-Uses: neo/buffer
+Uses: neo/buffer
+Uses: neo/sqlite3

package.dds

@@ -6,5 +6,6 @@ Depends: spdlog 1.4.2
 Depends: ms-wil 2019.11.10
 Depends: range-v3 0.9.1
 Depends: nlohmann-json 3.7.1
+Depends: neo-sqlite3 0.2.0
 
 Test-Driver: Catch-Main

src/dds/build.cpp

@@ -7,6 +7,7 @@
 #include <dds/util/time.hpp>
 #include <libman/index.hpp>
 #include <libman/parse.hpp>
+#include <dds/build/plan/compile_exec.hpp>
 
 #include <spdlog/spdlog.h>
 
@@ -116,16 +117,18 @@ load_usage_requirements(path_ref project_root, path_ref build_root, path_ref use
 void prepare_catch2_driver(library_build_params& lib_params,
                            test_lib              test_driver,
                            const build_params&   params,
-                           const package_manifest&) {
-    fs::path test_include_root = params.out_root / "_test_inc";
+                           build_env_ref         env_) {
+    fs::path test_include_root = params.out_root / "_catch-2.10.2";
     lib_params.test_include_dirs.emplace_back(test_include_root);
 
     auto catch_hpp = test_include_root / "catch2/catch.hpp";
-    fs::create_directories(catch_hpp.parent_path());
-    auto hpp_strm = open(catch_hpp, std::ios::out | std::ios::binary);
-    hpp_strm.write(detail::catch2_embedded_single_header_str,
-                   std::strlen(detail::catch2_embedded_single_header_str));
-    hpp_strm.close();
+    if (!fs::exists(catch_hpp)) {
+        fs::create_directories(catch_hpp.parent_path());
+        auto hpp_strm = open(catch_hpp, std::ios::out | std::ios::binary);
+        hpp_strm.write(detail::catch2_embedded_single_header_str,
+                    std::strlen(detail::catch2_embedded_single_header_str));
+        hpp_strm.close();
+    }
 
     if (test_driver == test_lib::catch_) {
         // Don't generate a test library helper
@@ -157,14 +160,14 @@ void prepare_catch2_driver(library_build_params& lib_params,
     assert(sf.has_value());
 
     compile_file_plan plan{comp_rules, std::move(*sf), "Catch2", "v1"};
-    build_env         env;
-    env.output_root = params.out_root / "_test-driver";
-    env.toolchain   = params.toolchain;
-    auto obj_file   = plan.calc_object_file_path(env);
+
+    build_env env2 = env_;
+    env2.output_root /= "_test-driver";
+    auto obj_file = plan.calc_object_file_path(env2);
 
     if (!fs::exists(obj_file)) {
         spdlog::info("Compiling Catch2 test driver (This will only happen once)...");
-        plan.compile(env);
+        compile_all(std::array{plan}, env2, 1);
     }
 
     lib_params.test_link_files.push_back(obj_file);
@@ -172,11 +175,12 @@ void prepare_catch2_driver(library_build_params& lib_params,
 
 void prepare_test_driver(library_build_params&   lib_params,
                          const build_params&     params,
-                         const package_manifest& man) {
+                         const package_manifest& man,
+                         build_env_ref           env) {
     auto& test_driver = *man.test_driver;
     if (test_driver == test_lib::catch_ || test_driver == test_lib::catch_main
         || test_driver == test_lib::catch_runner) {
-        prepare_catch2_driver(lib_params, test_driver, params, man);
+        prepare_catch2_driver(lib_params, test_driver, params, env);
     } else {
         assert(false && "Unreachable");
         std::terminate();
@@ -203,8 +207,11 @@ void dds::build(const build_params& params, const package_manifest& man) {
     lib_params.build_apps      = params.build_apps;
     lib_params.enable_warnings = params.enable_warnings;
 
+    auto           db = database::open(params.out_root / ".dds.db");
+    dds::build_env env{params.toolchain, params.out_root, db};
+
     if (man.test_driver) {
-        prepare_test_driver(lib_params, params, man);
+        prepare_test_driver(lib_params, params, man, env);
     }
 
     for (const library& lib : libs) {
@@ -212,7 +219,6 @@ void dds::build(const build_params& params, const package_manifest& man) {
         pkg.add_library(library_plan::create(lib, lib_params, ureqs));
     }
 
-    dds::build_env env{params.toolchain, params.out_root};
     if (params.generate_compdb) {
         generate_compdb(plan, env);
     }

src/dds/build/deps.cpp

@@ -0,0 +1,81 @@
+#include "./deps.hpp"
+
+#include <dds/db/database.hpp>
+#include <dds/proc.hpp>
+#include <dds/util/shlex.hpp>
+#include <dds/util/string.hpp>
+
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/transform.hpp>
+#include <spdlog/spdlog.h>
+
+using namespace dds;
+
+deps_info dds::parse_mkfile_deps_file(path_ref where) {
+    auto content = slurp_file(where);
+    return parse_mkfile_deps_str(content);
+}
+
+deps_info dds::parse_mkfile_deps_str(std::string_view str) {
+    deps_info ret;
+
+    // Remove escaped newlines
+    auto no_newlines = replace(str, "\\\n", " ");
+
+    auto split = split_shell_string(str);
+    auto iter  = split.begin();
+    auto stop  = split.end();
+    if (iter == stop) {
+        spdlog::critical(
+            "Invalid deps listing. Shell split was empty. This is almost certainly a bug.");
+        return ret;
+    }
+    auto& head = *iter;
+    ++iter;
+    if (!ends_with(head, ":")) {
+        spdlog::critical(
+            "Invalid deps listing. Leader item is not colon-terminated. This is probably a bug. "
+            "(Are you trying to use C++ Modules? That's not ready yet, sorry. Set `Deps-Mode` to "
+            "`None` in your toolchain file.)");
+        return ret;
+    }
+    ret.output = head.substr(0, head.length() - 1);
+    ret.inputs.insert(ret.inputs.end(), iter, stop);
+    return ret;
+}
+
+void dds::update_deps_info(database& db, const deps_info& deps) {
+    db.store_mtime(deps.output, fs::last_write_time(deps.output));
+    db.store_file_command(deps.output, {deps.command, deps.command_output});
+    db.forget_inputs_of(deps.output);
+    for (auto&& inp : deps.inputs) {
+        db.store_mtime(inp, fs::last_write_time(inp));
+        db.record_dep(inp, deps.output);
+    }
+}
+
+deps_rebuild_info dds::get_rebuild_info(database& db, path_ref output_path) {
+    std::unique_lock lk{db.mutex()};
+    auto             cmd_ = db.command_of(output_path);
+    if (!cmd_) {
+        return {};
+    }
+    auto& cmd     = *cmd_;
+    auto  inputs_ = db.inputs_of(output_path);
+    if (!inputs_) {
+        return {};
+    }
+    auto& inputs        = *inputs_;
+    auto  changed_files =  //
+        inputs             //
+        | ranges::views::filter([](const seen_file_info& input) {
+              return fs::last_write_time(input.path) != input.last_mtime;
+          })
+        | ranges::views::transform([](auto& info) { return info.path; })  //
+        | ranges::to_vector;
+    deps_rebuild_info ret;
+    ret.newer_inputs            = std::move(changed_files);
+    ret.previous_command        = cmd.command;
+    ret.previous_command_output = cmd.output;
+    return ret;
+}

src/dds/build/deps.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <dds/toolchain/deps.hpp>
+#include <dds/util/fs.hpp>
+
+#include <string>
+#include <string_view>
+
+namespace dds {
+
+class database;
+
+deps_info parse_mkfile_deps_file(path_ref where);
+deps_info parse_mkfile_deps_str(std::string_view str);
+
+void update_deps_info(database& db, const deps_info&);
+
+struct deps_rebuild_info {
+    std::vector<fs::path> newer_inputs;
+    std::string           previous_command;
+    std::string           previous_command_output;
+};
+
+deps_rebuild_info
+get_rebuild_info(database& db, path_ref output_path);
+
+}  // namespace dds

src/dds/build/deps.test.cpp

@@ -0,0 +1,26 @@
+#include <dds/build/deps.hpp>
+
+#include <catch2/catch.hpp>
+
+auto path_vec = [](auto... args) { return std::vector<dds::fs::path>{args...}; };
+
+TEST_CASE("Parse Makefile deps") {
+    auto deps = dds::parse_mkfile_deps_str("foo.o: bar.c");
+    CHECK(deps.output == "foo.o");
+    CHECK(deps.inputs == path_vec("bar.c"));
+    // Newline is okay
+    deps = dds::parse_mkfile_deps_str("foo.o: bar.c \\\n baz.c");
+    CHECK(deps.output == "foo.o");
+    CHECK(deps.inputs == path_vec("bar.c", "baz.c"));
+}
+
+TEST_CASE("Invalid deps") {
+    // Invalid deps does not terminate. This will generate an error message in
+    // the logs, but it is a non-fatal error that we can recover from.
+    auto deps = dds::parse_mkfile_deps_str("foo.o : cat");
+    CHECK(deps.output.empty());
+    CHECK(deps.inputs.empty());
+    deps = dds::parse_mkfile_deps_str("foo.c");
+    CHECK(deps.output.empty());
+    CHECK(deps.inputs.empty());
+}

src/dds/build/plan/base.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <dds/db/database.hpp>
 #include <dds/toolchain/toolchain.hpp>
 #include <dds/util/fs.hpp>
 
@@ -8,6 +9,7 @@ namespace dds {
 struct build_env {
     dds::toolchain toolchain;
     fs::path       output_root;
+    database&      db;
 };
 
 using build_env_ref = const build_env&;

src/dds/build/plan/compile_exec.cpp

@@ -0,0 +1,184 @@
+#include "./compile_exec.hpp"
+
+#include <dds/build/deps.hpp>
+#include <dds/proc.hpp>
+#include <dds/util/time.hpp>
+
+#include <range/v3/view/filter.hpp>
+#include <range/v3/view/transform.hpp>
+#include <spdlog/spdlog.h>
+
+#include <algorithm>
+#include <cassert>
+#include <thread>
+
+using namespace dds;
+using namespace ranges;
+
+namespace {
+
+template <typename Range, typename Fn>
+bool parallel_run(Range&& rng, int n_jobs, Fn&& fn) {
+    // We don't bother with a nice thread pool, as the overhead of most build
+    // tasks dwarf the cost of interlocking.
+    std::mutex mut;
+
+    auto       iter = rng.begin();
+    const auto stop = rng.end();
+
+    std::vector<std::exception_ptr> exceptions;
+
+    auto run_one = [&]() mutable {
+        while (true) {
+            std::unique_lock lk{mut};
+            if (!exceptions.empty()) {
+                break;
+            }
+            if (iter == stop) {
+                break;
+            }
+            auto&& item = *iter;
+            ++iter;
+            lk.unlock();
+            try {
+                fn(item);
+            } catch (...) {
+                lk.lock();
+                exceptions.push_back(std::current_exception());
+                break;
+            }
+        }
+    };
+
+    std::unique_lock         lk{mut};
+    std::vector<std::thread> threads;
+    if (n_jobs < 1) {
+        n_jobs = std::thread::hardware_concurrency() + 2;
+    }
+    std::generate_n(std::back_inserter(threads), n_jobs, [&] { return std::thread(run_one); });
+    lk.unlock();
+    for (auto& t : threads) {
+        t.join();
+    }
+    for (auto eptr : exceptions) {
+        try {
+            std::rethrow_exception(eptr);
+        } catch (const std::exception& e) {
+            spdlog::error(e.what());
+        }
+    }
+    return exceptions.empty();
+}
+
+struct compile_file_full {
+    const compile_file_plan& plan;
+    fs::path                 object_file_path;
+    compile_command_info     cmd_info;
+};
+
+std::optional<deps_info> do_compile(const compile_file_full& cf, build_env_ref env) {
+    fs::create_directories(cf.object_file_path.parent_path());
+    auto source_path = cf.plan.source_path();
+    auto msg         = fmt::format("[{}] Compile: {:40}",
+                           cf.plan.qualifier(),
+                           fs::relative(source_path, cf.plan.source().basis_path).string());
+    spdlog::info(msg);
+    auto&& [dur_ms, compile_res]
+        = timed<std::chrono::milliseconds>([&] { return run_proc(cf.cmd_info.command); });
+    spdlog::info("{} - {:>7n}ms", msg, dur_ms.count());
+
+    if (!compile_res.okay()) {
+        spdlog::error("Compilation failed: {}", source_path.string());
+        spdlog::error("Subcommand FAILED: {}\n{}",
+                      quote_command(cf.cmd_info.command),
+                      compile_res.output);
+        throw compile_failure(fmt::format("Compilation failed for {}", source_path.string()));
+    }
+
+    std::optional<deps_info> ret_deps_info;
+
+    if (env.toolchain.deps_mode() == deps_mode::gnu) {
+        assert(cf.cmd_info.gnu_depfile_path.has_value());
+        auto& df_path = *cf.cmd_info.gnu_depfile_path;
+        if (!fs::is_regular_file(df_path)) {
+            spdlog::critical(
+                "The expected Makefile deps were not generated on disk. This is a bug! "
+                "(Expected "
+                "file to exist: [{}])",
+                df_path.string());
+        } else {
+            auto dep_info           = dds::parse_mkfile_deps_file(df_path);
+            dep_info.command        = quote_command(cf.cmd_info.command);
+            dep_info.command_output = compile_res.output;
+            ret_deps_info           = std::move(dep_info);
+        }
+    }
+
+    // MSVC prints the filename of the source file. Dunno why, but they do.
+    if (compile_res.output.find(source_path.filename().string() + "\r\n") == 0) {
+        compile_res.output.erase(0, source_path.filename().string().length() + 2);
+    }
+
+    if (!compile_res.output.empty()) {
+        spdlog::warn("While compiling file {} [{}]:\n{}",
+                     source_path.string(),
+                     quote_command(cf.cmd_info.command),
+                     compile_res.output);
+    }
+
+    return ret_deps_info;
+}
+
+compile_file_full realize_plan(const compile_file_plan& plan, build_env_ref env) {
+    auto cmd_info = plan.generate_compile_command(env);
+    return compile_file_full{plan, plan.calc_object_file_path(env), cmd_info};
+}
+
+bool should_compile(const compile_file_full& comp, build_env_ref env) {
+    database& db      = env.db;
+    auto      rb_info = get_rebuild_info(db, comp.object_file_path);
+    if (rb_info.previous_command.empty()) {
+        // We have no previous compile command for this file. Assume it is
+        // new.
+        return true;
+    }
+    if (!rb_info.newer_inputs.empty()) {
+        // Inputs to this file have changed from a prior execution.
+        return true;
+    }
+    auto cur_cmd_str = quote_command(comp.cmd_info.command);
+    if (cur_cmd_str != rb_info.previous_command) {
+        // The command used to generate the output is new
+        return true;
+    }
+    // Nope. This file is up-to-date.
+    return false;
+}
+
+}  // namespace
+
+bool dds::detail::compile_all(const ref_vector<const compile_file_plan>& compiles,
+                              build_env_ref                              env,
+                              int                                        njobs) {
+    auto each_realized =                                                          //
+        compiles                                                                  //
+        | views::transform([&](auto&& plan) { return realize_plan(plan, env); })  //
+        | views::filter([&](auto&& real) { return should_compile(real, env); });
+
+    std::vector<deps_info> new_deps;
+    std::mutex             mut;
+    auto okay = parallel_run(each_realized, njobs, [&](const compile_file_full& full) {
+        auto nd = do_compile(full, env);
+        if (nd) {
+            std::unique_lock lk{mut};
+            new_deps.push_back(std::move(*nd));
+        }
+    });
+
+    stopwatch sw;
+    for (auto& info : new_deps) {
+        auto tr = env.db.transaction();
+        update_deps_info(env.db, info);
+    }
+    return okay;
+}

src/dds/build/plan/compile_exec.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <dds/build/plan/base.hpp>
+#include <dds/build/plan/compile_file.hpp>
+#include <dds/util/algo.hpp>
+
+#include <functional>
+#include <vector>
+
+namespace dds {
+
+namespace detail {
+
+bool compile_all(const ref_vector<const compile_file_plan>& files, build_env_ref env, int njobs);
+
+}  // namespace detail
+
+template <typename Range>
+bool compile_all(Range&& rng, build_env_ref env, int njobs) {
+    ref_vector<const compile_file_plan> cfps;
+    for (auto&& cf : rng) {
+        cfps.push_back(cf);
+    }
+    return detail::compile_all(cfps, env, njobs);
+}
+
+}  // namespace dds

src/dds/build/plan/compile_file.cpp

@@ -20,44 +20,9 @@ compile_command_info compile_file_plan::generate_compile_command(build_env_ref e
     return env.toolchain.create_compile_command(spec);
 }
 
-std::optional<deps_info> compile_file_plan::compile(const build_env& env) const {
-    const auto obj_path = calc_object_file_path(env);
-    fs::create_directories(obj_path.parent_path());
-
-    auto msg = fmt::format("[{}] Compile: {:40}",
-                           _qualifier,
-                           fs::relative(_source.path, _source.basis_path).string());
-
-    spdlog::info(msg);
-    auto cmd = generate_compile_command(env);
-    auto&& [dur_ms, compile_res]
-        = timed<std::chrono::milliseconds>([&] { return run_proc(cmd.command); });
-    spdlog::info("{} - {:>7n}ms", msg, dur_ms.count());
-
-    if (!compile_res.okay()) {
-        spdlog::error("Compilation failed: {}", _source.path.string());
-        spdlog::error("Subcommand FAILED: {}\n{}", quote_command(cmd.command), compile_res.output);
-        throw compile_failure(fmt::format("Compilation failed for {}", _source.path.string()));
-    }
-
-    // MSVC prints the filename of the source file. Dunno why, but they do.
-    if (compile_res.output.find(_source.path.filename().string() + "\r\n") == 0) {
-        compile_res.output.erase(0, _source.path.filename().string().length() + 2);
-    }
-
-    if (!compile_res.output.empty()) {
-        spdlog::warn("While compiling file {} [{}]:\n{}",
-                     _source.path.string(),
-                     quote_command(cmd.command),
-                     compile_res.output);
-    }
-
-    return std::nullopt;
-}
-
 fs::path compile_file_plan::calc_object_file_path(const build_env& env) const noexcept {
     auto relpath = fs::relative(_source.path, _source.basis_path);
     auto ret     = env.output_root / _subdir / relpath;
     ret.replace_filename(relpath.filename().string() + env.toolchain.object_suffix());
-    return ret;
+    return fs::weakly_canonical(ret);
 }

src/dds/build/plan/compile_file.hpp

@@ -56,13 +56,13 @@ public:
         , _qualifier(qual)
         , _subdir(subdir) {}
 
-    compile_command_info generate_compile_command(build_env_ref) const noexcept;
-
     const source_file& source() const noexcept { return _source; }
     path_ref           source_path() const noexcept { return _source.path; }
     auto&              rules() const noexcept { return _rules; }
+    auto&              qualifier() const noexcept { return _qualifier; }
 
     fs::path                 calc_object_file_path(build_env_ref env) const noexcept;
+    compile_command_info     generate_compile_command(build_env_ref) const noexcept;
     std::optional<deps_info> compile(build_env_ref) const;
 };
 

src/dds/build/plan/full.cpp

@@ -1,6 +1,7 @@
 #include "./full.hpp"
 
 #include <dds/build/iter_compilations.hpp>
+#include <dds/build/plan/compile_exec.hpp>
 
 #include <range/v3/view/concat.hpp>
 #include <range/v3/view/filter.hpp>
@@ -36,7 +37,8 @@ bool parallel_run(Range&& rng, int n_jobs, Fn&& fn) {
             if (iter == stop) {
                 break;
             }
-            auto&& item = *iter++;
+            auto&& item = *iter;
+            ++iter;
             lk.unlock();
             try {
                 fn(item);
@@ -71,9 +73,7 @@ bool parallel_run(Range&& rng, int n_jobs, Fn&& fn) {
 }  // namespace
 
 void build_plan::compile_all(const build_env& env, int njobs) const {
-    auto okay = parallel_run(iter_compilations(*this), njobs, [&](const compile_file_plan& cf) {
-        cf.compile(env);
-    });
+    auto okay = dds::compile_all(iter_compilations(*this), env, njobs);
     if (!okay) {
         throw std::runtime_error("Compilation failed.");
     }

src/dds/db/database.cpp

@@ -0,0 +1,245 @@
+#include "./database.hpp"
+
+#include <neo/sqlite3/exec.hpp>
+#include <neo/sqlite3/iter_tuples.hpp>
+#include <neo/sqlite3/single.hpp>
+#include <neo/sqlite3/transaction.hpp>
+
+#include <nlohmann/json.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/transform.hpp>
+#include <spdlog/spdlog.h>
+
+using namespace dds;
+
+namespace sqlite3 = neo::sqlite3;
+using sqlite3::exec;
+using namespace sqlite3::literals;
+
+namespace {
+
+void migrate_1(sqlite3::database& db) {
+    db.exec(R"(
+        CREATE TABLE dds_files (
+            file_id INTEGER PRIMARY KEY,
+            path TEXT NOT NULL UNIQUE,
+            mtime INTEGER NOT NULL
+        );
+        CREATE TABLE dds_deps (
+            input_file_id
+                INTEGER
+                NOT NULL
+                REFERENCES dds_files(file_id),
+            output_file_id
+                INTEGER
+                NOT NULL
+                REFERENCES dds_files(file_id),
+            UNIQUE(input_file_id, output_file_id)
+        );
+        CREATE TABLE dds_file_commands (
+            command_id INTEGER PRIMARY KEY,
+            file_id
+                INTEGER
+                UNIQUE
+                NOT NULL
+                REFERENCES dds_files(file_id),
+            command TEXT NOT NULL,
+            output TEXT NOT NULL
+        );
+    )");
+}
+
+void ensure_migrated(sqlite3::database& db) {
+    sqlite3::transaction_guard tr{db};
+    db.exec(R"(
+        PRAGMA foreign_keys = 1;
+        CREATE TABLE IF NOT EXISTS dds_meta AS
+            WITH init (meta) AS (VALUES ('{"version": 0}'))
+            SELECT * FROM init;
+        )");
+    auto meta_st     = db.prepare("SELECT meta FROM dds_meta");
+    auto [meta_json] = sqlite3::unpack_single<std::string>(meta_st);
+
+    auto meta = nlohmann::json::parse(meta_json);
+    if (!meta.is_object()) {
+        throw std::runtime_error("Correupted database file.");
+    }
+
+    auto version_ = meta["version"];
+    if (!version_.is_number_integer()) {
+        throw std::runtime_error("Corrupted database file [bad dds_meta.version]");
+    }
+    int version = version_;
+    if (version < 1) {
+        migrate_1(db);
+    }
+    meta["version"] = 1;
+    exec(db, "UPDATE dds_meta SET meta=?", std::forward_as_tuple(meta.dump()));
+}
+
+}  // namespace
+
+database database::open(const std::string& db_path) {
+    auto db = sqlite3::database::open(db_path);
+    try {
+        ensure_migrated(db);
+    } catch (const sqlite3::sqlite3_error& e) {
+        spdlog::error(
+            "Failed to load the databsae. It appears to be invalid/corrupted. We'll delete it and "
+            "create a new one. The exception message is: {}",
+            e.what());
+        fs::remove(db_path);
+        db = sqlite3::database::open(db_path);
+        try {
+            ensure_migrated(db);
+        } catch (const sqlite3::sqlite3_error& e) {
+            spdlog::critical(
+                "Failed to apply database migrations to recovery database. This is a critical "
+                "error. The exception message is: {}",
+                e.what());
+            std::terminate();
+        }
+    }
+    return database(std::move(db));
+}
+
+database::database(sqlite3::database db)
+    : _db(std::move(db)) {}
+
+std::optional<fs::file_time_type> database::last_mtime_of(path_ref file_) {
+    auto& st = _stmt_cache(R"(
+        SELECT mtime FROM dds_files WHERE path = ?
+    )"_sql);
+    st.reset();
+    auto path      = fs::weakly_canonical(file_);
+    st.bindings[1] = path.string();
+    auto maybe_res = sqlite3::unpack_single_opt<std::int64_t>(st);
+    if (!maybe_res) {
+        return std::nullopt;
+    }
+    auto [timestamp] = *maybe_res;
+    return fs::file_time_type(fs::file_time_type::duration(timestamp));
+}
+
+void database::store_mtime(path_ref file, fs::file_time_type time) {
+    auto& st = _stmt_cache(R"(
+        INSERT INTO dds_files (path, mtime)
+        VALUES (?1, ?2)
+        ON CONFLICT(path) DO UPDATE SET mtime = ?2
+    )"_sql);
+    sqlite3::exec(st,
+                  std::forward_as_tuple(fs::weakly_canonical(file).string(),
+                                        time.time_since_epoch().count()));
+}
+
+void database::record_dep(path_ref input, path_ref output) {
+    auto& st = _stmt_cache(R"(
+        WITH input AS (
+            SELECT file_id
+              FROM dds_files
+             WHERE path = ?1
+        ),
+        output AS (
+            SELECT file_id
+              FROM dds_files
+             WHERE path = ?2
+        )
+        INSERT OR IGNORE INTO dds_deps (input_file_id, output_file_id)
+        VALUES (
+            (SELECT * FROM input),
+            (SELECT * FROM output)
+        )
+    )"_sql);
+    sqlite3::exec(st,
+                  std::forward_as_tuple(fs::weakly_canonical(input.string()),
+                                        fs::weakly_canonical(output.string())));
+}
+
+void database::store_file_command(path_ref file, const command_info& cmd) {
+    auto& st = _stmt_cache(R"(
+        WITH file AS (
+            SELECT file_id
+            FROM dds_files
+            WHERE path = ?1
+        )
+        INSERT OR REPLACE
+          INTO dds_file_commands(file_id, command, output)
+        VALUES (
+            (SELECT * FROM file),
+            ?2,
+            ?3
+        )
+    )"_sql);
+    sqlite3::exec(st,
+                  std::forward_as_tuple(fs::weakly_canonical(file).string(),
+                                        std::string_view(cmd.command),
+                                        std::string_view(cmd.output)));
+}
+
+void database::forget_inputs_of(path_ref file) {
+    auto& st = _stmt_cache(R"(
+        WITH id_to_delete AS (
+            SELECT file_id
+            FROM dds_files
+            WHERE path = ?
+        )
+        DELETE FROM dds_deps
+         WHERE output_file_id IN id_to_delete
+    )"_sql);
+    sqlite3::exec(st, std::forward_as_tuple(fs::weakly_canonical(file)));
+}
+
+std::optional<std::vector<seen_file_info>> database::inputs_of(path_ref file_) {
+    auto  file = fs::weakly_canonical(file_);
+    auto& st   = _stmt_cache(R"(
+        WITH file AS (
+            SELECT file_id
+              FROM dds_files
+             WHERE path = ?
+        ),
+        input_ids AS (
+            SELECT input_file_id
+              FROM dds_deps
+             WHERE output_file_id IN file
+        )
+        SELECT path, mtime
+          FROM dds_files
+         WHERE file_id IN input_ids
+    )"_sql);
+    st.reset();
+    st.bindings[1] = file.string();
+    auto tup_iter  = sqlite3::iter_tuples<std::string, std::int64_t>(st);
+
+    std::vector<seen_file_info> ret;
+    for (auto& [path, mtime] : tup_iter) {
+        ret.emplace_back(
+            seen_file_info{path, fs::file_time_type(fs::file_time_type::duration(mtime))});
+    }
+
+    if (ret.empty()) {
+        return std::nullopt;
+    }
+    return ret;
+}
+
+std::optional<command_info> database::command_of(path_ref file_) {
+    auto  file = fs::weakly_canonical(file_);
+    auto& st   = _stmt_cache(R"(
+        WITH file AS (
+            SELECT file_id
+              FROM dds_files
+             WHERE path = ?
+        )
+        SELECT command, output
+          FROM dds_file_commands
+         WHERE file_id IN file
+    )"_sql);
+    st.reset();
+    st.bindings[1] = file.string();
+    auto opt_res   = sqlite3::unpack_single_opt<std::string, std::string>(st);
+    if (!opt_res) {
+        return std::nullopt;
+    }
+    auto& [cmd, out] = *opt_res;
+    return command_info{cmd, out};
+}

src/dds/db/database.hpp

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <dds/util/fs.hpp>
+
+#include <neo/sqlite3/database.hpp>
+#include <neo/sqlite3/statement.hpp>
+#include <neo/sqlite3/statement_cache.hpp>
+#include <neo/sqlite3/transaction.hpp>
+
+#include <chrono>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <string_view>
+
+namespace dds {
+
+struct command_info {
+    std::string command;
+    std::string output;
+};
+
+struct seen_file_info {
+    fs::path           path;
+    fs::file_time_type last_mtime;
+};
+
+class database {
+    neo::sqlite3::database        _db;
+    neo::sqlite3::statement_cache _stmt_cache{_db};
+    mutable std::shared_mutex     _mutex;
+
+    explicit database(neo::sqlite3::database db);
+    database(const database&) = delete;
+
+public:
+    static database open(const std::string& db_path);
+    static database open(path_ref db_path) { return open(db_path.string()); }
+
+    auto& mutex() const noexcept { return _mutex; }
+
+    neo::sqlite3::transaction_guard transaction() noexcept {
+        return neo::sqlite3::transaction_guard(_db);
+    }
+
+    std::optional<fs::file_time_type> last_mtime_of(path_ref file);
+    void                              store_mtime(path_ref file, fs::file_time_type time);
+    void                              record_dep(path_ref input, path_ref output);
+    void                              store_file_command(path_ref file, const command_info& cmd);
+    void                              forget_inputs_of(path_ref file);
+
+    std::optional<std::vector<seen_file_info>> inputs_of(path_ref file);
+    std::optional<command_info>                command_of(path_ref file);
+};
+
+}  // namespace dds

src/dds/db/database.test.cpp

@@ -0,0 +1,21 @@
+#include <dds/db/database.hpp>
+
+#include <catch2/catch.hpp>
+
+using namespace std::literals;
+
+TEST_CASE("Create a database") { auto db = dds::database::open(":memory:"s); }
+TEST_CASE("Read an absent file's mtime") {
+    auto db        = dds::database::open(":memory:"s);
+    auto mtime_opt = db.last_mtime_of("bad/file/path");
+    CHECK_FALSE(mtime_opt.has_value());
+}
+
+TEST_CASE("Record a file") {
+    auto db = dds::database::open(":memory:"s);
+    auto time = dds::fs::file_time_type::clock::now();
+    db.store_mtime("file/something", time);
+    auto mtime_opt = db.last_mtime_of("file/something");
+    REQUIRE(mtime_opt.has_value());
+    CHECK(mtime_opt == time);
+}

src/dds/dds.main.cpp

@@ -469,7 +469,8 @@ struct cli_deps {
 
             auto           tc   = tc_filepath.get_toolchain();
             auto           bdir = build_dir.Get();
-            dds::build_env env{std::move(tc), bdir};
+            auto           db   = dds::database::open(bdir / ".dds.db");
+            dds::build_env env{std::move(tc), bdir, db};
 
             auto plan = dds::create_deps_build_plan(deps, env);
             plan.compile_all(env, 6);

src/dds/repo/remote.cpp

@@ -6,6 +6,7 @@
 #include <dds/sdist.hpp>
 #include <dds/temp.hpp>
 #include <dds/toolchain/toolchain.hpp>
+#include <dds/util/shlex.hpp>
 
 #include <spdlog/spdlog.h>
 

src/dds/toolchain/deps.hpp

@@ -13,9 +13,10 @@ enum class deps_mode {
 };
 
 struct deps_info {
-    fs::path                 output;
-    std::vector<fs::path>    inputs;
-    std::vector<std::string> command;
+    fs::path              output;
+    std::vector<fs::path> inputs;
+    std::string           command;
+    std::string           command_output;
 };
 
 }  // namespace dds

src/dds/toolchain/from_dds.cpp

@@ -3,6 +3,7 @@
 #include <dds/toolchain/prep.hpp>
 #include <dds/toolchain/toolchain.hpp>
 #include <dds/util/algo.hpp>
+#include <dds/util/shlex.hpp>
 #include <libman/parse.hpp>
 
 #include <spdlog/fmt/fmt.h>

src/dds/toolchain/toolchain.cpp

@@ -36,62 +36,6 @@ toolchain toolchain::realize(const toolchain_prep& prep) {
     return ret;
 }
 
-vector<string> dds::split_shell_string(std::string_view shell) {
-    char cur_quote  = 0;
-    bool is_escaped = false;
-
-    vector<string> acc;
-
-    const auto begin = shell.begin();
-    auto       iter  = begin;
-    const auto end   = shell.end();
-
-    opt_string token;
-    while (iter != end) {
-        const char c = *iter++;
-        if (is_escaped) {
-            if (c == '\n') {
-                // Ignore the newline
-            } else if (cur_quote || c != cur_quote || c == '\\') {
-                // Escaped `\` character
-                token = token.value_or("") + c;
-            } else {
-                // Regular escape sequence
-                token = token.value_or("") + '\\' + c;
-            }
-            is_escaped = false;
-        } else if (c == '\\') {
-            is_escaped = true;
-        } else if (cur_quote) {
-            if (c == cur_quote) {
-                // End of quoted token;
-                cur_quote = 0;
-            } else {
-                token = token.value_or("") + c;
-            }
-        } else if (c == '"' || c == '\'') {
-            // Beginning of a quoted token
-            cur_quote = c;
-            token     = "";
-        } else if (c == '\t' || c == ' ' || c == '\n' || c == '\r' || c == '\f') {
-            // We've reached unquoted whitespace
-            if (token.has_value()) {
-                acc.push_back(move(*token));
-            }
-            token.reset();
-        } else {
-            // Just a regular character
-            token = token.value_or("") + c;
-        }
-    }
-
-    if (token.has_value()) {
-        acc.push_back(move(*token));
-    }
-
-    return acc;
-}
-
 vector<string> toolchain::include_args(const fs::path& p) const noexcept {
     return replace(_inc_template, "<PATH>", p.string());
 }

src/dds/toolchain/toolchain.hpp

@@ -10,8 +10,6 @@
 
 namespace dds {
 
-std::vector<std::string> split_shell_string(std::string_view s);
-
 enum class language {
     automatic,
     c,

src/dds/util/algo.hpp

@@ -2,6 +2,8 @@
 
 #include <algorithm>
 #include <initializer_list>
+#include <vector>
+#include <functional>
 
 namespace dds {
 
@@ -33,4 +35,7 @@ void extend(Container& c, std::initializer_list<Item> il) {
     c.insert(c.end(), il.begin(), il.end());
 }
 
+template <typename T>
+using ref_vector = std::vector<std::reference_wrapper<T>>;
+
 }  // namespace dds

src/dds/util/shlex.cpp

@@ -0,0 +1,66 @@
+#include "./shlex.hpp"
+
+#include <optional>
+#include <string>
+#include <utility>
+
+using std::string;
+using std::vector;
+
+using namespace dds;
+
+vector<string> dds::split_shell_string(std::string_view shell) {
+    char cur_quote  = 0;
+    bool is_escaped = false;
+
+    vector<string> acc;
+
+    const auto begin = shell.begin();
+    auto       iter  = begin;
+    const auto end   = shell.end();
+
+    std::optional<string> token;
+    while (iter != end) {
+        const char c = *iter++;
+        if (is_escaped) {
+            if (c == '\n') {
+                // Ignore the newline
+            } else if (cur_quote || c != cur_quote || c == '\\') {
+                // Escaped `\` character
+                token = token.value_or("") + c;
+            } else {
+                // Regular escape sequence
+                token = token.value_or("") + '\\' + c;
+            }
+            is_escaped = false;
+        } else if (c == '\\') {
+            is_escaped = true;
+        } else if (cur_quote) {
+            if (c == cur_quote) {
+                // End of quoted token;
+                cur_quote = 0;
+            } else {
+                token = token.value_or("") + c;
+            }
+        } else if (c == '"' || c == '\'') {
+            // Beginning of a quoted token
+            cur_quote = c;
+            token     = "";
+        } else if (c == '\t' || c == ' ' || c == '\n' || c == '\r' || c == '\f') {
+            // We've reached unquoted whitespace
+            if (token.has_value()) {
+                acc.push_back(move(*token));
+            }
+            token.reset();
+        } else {
+            // Just a regular character
+            token = token.value_or("") + c;
+        }
+    }
+
+    if (token.has_value()) {
+        acc.push_back(move(*token));
+    }
+
+    return acc;
+}

src/dds/util/shlex.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace dds {
+
+std::vector<std::string> split_shell_string(std::string_view s);
+
+}  // namespace dds

src/dds/toolchain/toolchain.test.cpp → src/dds/util/shlex.test.cpp

@@ -1,4 +1,4 @@
-#include <dds/toolchain/toolchain.hpp>
+#include <dds/util/shlex.hpp>
 
 #include <catch2/catch.hpp>