Add support for tweak-headers

src/dds/build/builder.cpp

@@ -277,6 +277,31 @@ void write_cmake(build_env_ref env, const build_plan& plan, path_ref cmake_out)
     }
 }
 
+/**
+ * @brief Calculate a hash of the directory layout of the given directory.
+ *
+ * Because a tweaks-dir is specifically designed to have files added/removed within it, and
+ * its contents are inspected by `__has_include`, we need to have a way to invalidate any caches
+ * when the content of that directory changes. We don't care to hash the contents of the files,
+ * since those will already break any caches.
+ */
+std::string hash_tweaks_dir(const fs::path& tweaks_dir) {
+    if (!fs::is_directory(tweaks_dir)) {
+        return "0";  // No tweaks directory, no cache to bust
+    }
+    std::vector<fs::path> children{fs::recursive_directory_iterator{tweaks_dir},
+                                   fs::recursive_directory_iterator{}};
+    std::sort(children.begin(), children.end());
+    // A really simple inline djb2 hash
+    std::uint32_t hash = 5381;
+    for (auto& p : children) {
+        for (std::uint32_t c : fs::weakly_canonical(p).string()) {
+            hash = ((hash << 5) + hash) + c;
+        }
+    }
+    return std::to_string(hash);
+}
+
 template <typename Func>
 void with_build_plan(const build_params&              params,
                      const std::vector<sdist_target>& sdists,
@@ -292,11 +317,20 @@ void with_build_plan(const build_params&              params,
         params.out_root,
         db,
         toolchain_knobs{
-            .is_tty = stdout_is_a_tty(),
+            .is_tty     = stdout_is_a_tty(),
+            .tweaks_dir = params.tweaks_dir,
         },
         ureqs,
     };
 
+    if (env.knobs.tweaks_dir) {
+        env.knobs.cache_buster = hash_tweaks_dir(*env.knobs.tweaks_dir);
+        dds_log(trace,
+                "Build cache-buster value for tweaks-dir [{}] content is '{}'",
+                *env.knobs.tweaks_dir,
+                *env.knobs.cache_buster);
+    }
+
     if (st.generate_catch2_main) {
         auto catch_lib                  = prepare_test_driver(params, test_lib::catch_main, env);
         ureqs.add(".dds", "Catch-Main") = catch_lib;

src/dds/build/params.hpp

@@ -13,9 +13,10 @@ struct build_params {
     std::optional<fs::path> existing_lm_index;
     std::optional<fs::path> emit_lmi;
     std::optional<fs::path> emit_cmake{};
+    std::optional<fs::path> tweaks_dir{};
     dds::toolchain          toolchain;
     bool                    generate_compdb = true;
     int                     parallel_jobs   = 0;
 };
 
-}  // namespace dds
+}  // namespace dds

src/dds/cli/cmd/build.cpp

@@ -30,6 +30,7 @@ int build(const options& opts) {
         .out_root          = opts.out_path.value_or(fs::current_path() / "_build"),
         .existing_lm_index = opts.build.lm_index,
         .emit_lmi          = {},
+        .tweaks_dir        = opts.build.tweaks_dir,
         .toolchain         = opts.load_toolchain(),
         .parallel_jobs     = opts.jobs,
     });

src/dds/cli/cmd/build_deps.cpp

@@ -18,6 +18,7 @@ int build_deps(const options& opts) {
         .existing_lm_index = {},
         .emit_lmi          = opts.build.lm_index.value_or("INDEX.lmi"),
         .emit_cmake        = opts.build_deps.cmake_file,
+        .tweaks_dir        = opts.build.tweaks_dir,
         .toolchain         = opts.load_toolchain(),
         .parallel_jobs     = opts.jobs,
     };

src/dds/cli/cmd/compile_file.cpp

@@ -11,6 +11,7 @@ int compile_file(const options& opts) {
                               .out_root = opts.out_path.value_or(fs::current_path() / "_build"),
                               .existing_lm_index = opts.build.lm_index,
                               .emit_lmi          = {},
+                              .tweaks_dir        = opts.build.tweaks_dir,
                               .toolchain         = opts.load_toolchain(),
                               .parallel_jobs     = opts.jobs,
                           });

src/dds/cli/options.cpp

@@ -90,6 +90,17 @@ struct setup {
         .action   = put_into(opts.repoman.repo_dir),
     };
 
+    argument tweaks_dir_arg{
+        .long_spellings  = {"tweaks-dir"},
+        .short_spellings = {"TD"},
+        .help
+        = "Base directory of "
+          "\x1b]8;;https://vector-of-bool.github.io/2020/10/04/lib-configuration.html\x1b\\tweak "
+          "headers\x1b]8;;\x1b\\ that should be available to the build.",
+        .valname = "<dir>",
+        .action  = put_into(opts.build.tweaks_dir),
+    };
+
     void do_setup(argument_parser& parser) noexcept {
         parser.add_argument({
             .long_spellings  = {"log-level"},
@@ -189,6 +200,7 @@ struct setup {
         build_cmd.add_argument(lm_index_arg.dup()).help
             = "Path to a libman index file to use for loading project dependencies";
         build_cmd.add_argument(jobs_arg.dup());
+        build_cmd.add_argument(tweaks_dir_arg.dup());
     }
 
     void setup_compile_file_cmd(argument_parser& compile_file_cmd) noexcept {
@@ -199,6 +211,7 @@ struct setup {
             = "Set the maximum number of files to compile in parallel";
         compile_file_cmd.add_argument(lm_index_arg.dup());
         compile_file_cmd.add_argument(out_arg.dup());
+        compile_file_cmd.add_argument(tweaks_dir_arg.dup());
         compile_file_cmd.add_argument({
             .help       = "One or more source files to compile",
             .valname    = "<source-files>",
@@ -228,6 +241,7 @@ struct setup {
             .valname = "<file-path>",
             .action  = debate::put_into(opts.build_deps.cmake_file),
         });
+        build_deps_cmd.add_argument(tweaks_dir_arg.dup());
         build_deps_cmd.add_argument({
             .help       = "Dependency statement strings",
             .valname    = "<dependency>",

src/dds/cli/options.hpp

@@ -136,6 +136,7 @@ struct options {
         opt_path            lm_index;
         std::vector<string> add_repos;
         bool                update_repos = false;
+        opt_path            tweaks_dir;
     } build;
 
     /**

src/dds/toolchain/toolchain.cpp

@@ -97,6 +97,13 @@ compile_command_info toolchain::create_compile_command(const compile_file_spec&
         extend(flags, _tty_flags);
     }
 
+    if (knobs.cache_buster) {
+        // This is simply a CPP definition that is used to "bust" any caches that rely on inspecting
+        // the command-line of the compiler (including our own).
+        auto def = replace(_def_template, "[def]", "__dds_cachebust=" + *knobs.cache_buster);
+        extend(flags, def);
+    }
+
     dds_log(trace, "#include-search dirs:");
     for (auto&& inc_dir : spec.include_dirs) {
         dds_log(trace, "  - search: {}", inc_dir.string());
@@ -111,6 +118,13 @@ compile_command_info toolchain::create_compile_command(const compile_file_spec&
         extend(flags, inc_args);
     }
 
+    if (knobs.tweaks_dir) {
+        dds_log(trace, "  - search (tweaks): {}", knobs.tweaks_dir->string());
+        auto shortest       = shortest_path_from(*knobs.tweaks_dir, cwd);
+        auto tweak_inc_args = include_args(shortest);
+        extend(flags, tweak_inc_args);
+    }
+
     for (auto&& def : spec.definitions) {
         auto def_args = definition_args(def);
         extend(flags, def_args);

src/dds/toolchain/toolchain.hpp

@@ -18,6 +18,9 @@ enum class language {
 
 struct toolchain_knobs {
     bool is_tty = false;
+    // Directory storing tweaks for the compilation
+    std::optional<fs::path>    tweaks_dir{};
+    std::optional<std::string> cache_buster{};
 };
 
 struct compile_file_spec {

tests/projects/tweaks/include/tweakable.config.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#if __has_include(<tweakable.tweaks.hpp>)
+#include <tweakable.tweaks.hpp>
+#endif
+
+namespace tweakable {
+
+namespace config {
+
+namespace defaults {
+
+const int value = 99;
+
+}  // namespace defaults
+
+using namespace defaults;
+
+}  // namespace config
+
+}  // namespace tweakable

tests/projects/tweaks/include/tweakable.hpp

@@ -0,0 +1,7 @@
+#pragma once
+
+namespace tweakable {
+
+extern int get_value();
+
+}  // namespace tweakable

tests/projects/tweaks/library.jsonc

@@ -0,0 +1,3 @@
+{
+    "name": "foo"
+}

tests/projects/tweaks/package.json5

@@ -0,0 +1,5 @@
+{
+    name: 'tweakable',
+    version: '1.2.3',
+    "namespace": "test",
+}

tests/projects/tweaks/src/tweakable.cpp

@@ -0,0 +1,6 @@
+#include <tweakable.config.hpp>
+#include <tweakable.hpp>
+
+#include <iostream>
+
+int tweakable::get_value() { return tweakable::config::value; }

tests/projects/tweaks/src/tweakable.main.cpp

@@ -0,0 +1,3 @@
+#include <tweakable.hpp>
+
+int main() { return tweakable::get_value(); }

tests/test_tweaks.py

@@ -0,0 +1,29 @@
+from dds_ci.testing.fixtures import ProjectOpener
+from dds_ci import paths, proc
+
+
+def test_lib_with_tweaks(project_opener: ProjectOpener) -> None:
+    pr = project_opener.open('projects/tweaks')
+    pr.build()
+    app = pr.build_root / ('tweakable' + paths.EXE_SUFFIX)
+    res = proc.run([app])
+    # The default value is 99:
+    assert res.returncode == 99
+    # Build again, but with an empty/non-existent tweaks directory
+    pr.build(tweaks_dir=pr.root / 'conf')
+    res = proc.run([app])
+    assert res.returncode == 99
+    # Now write a tweaks header and rebuild:
+    pr.write(
+        'conf/tweakable.tweaks.hpp', r'''
+    #pragma once
+
+    namespace tweakable {
+        namespace config {
+            const int value = 41;
+        }
+    }
+    ''')
+    pr.build(tweaks_dir=pr.root / 'conf')
+    res = proc.run([app])
+    assert res.returncode == 41

tools/dds_ci/dds.py

@@ -94,6 +94,7 @@ class DDSWrapper:
               toolchain: Optional[Path] = None,
               build_root: Optional[Path] = None,
               jobs: Optional[int] = None,
+              tweaks_dir: Optional[Path] = None,
               more_args: Optional[proc.CommandLine] = None,
               timeout: Optional[int] = None) -> None:
         """
@@ -115,6 +116,7 @@ class DDSWrapper:
                 f'--jobs={jobs}',
                 f'{self.project_dir_flag}={root}',
                 f'--out={build_root}',
+                f'--tweaks-dir={tweaks_dir}' if tweaks_dir else (),
                 more_args or (),
             ],
             timeout=timeout,

tools/dds_ci/testing/fixtures.py

@@ -75,7 +75,11 @@ class Project:
         """Argument for --project"""
         return f'--project={self.root}'
 
-    def build(self, *, toolchain: Optional[Pathish] = None, timeout: Optional[int] = None) -> None:
+    def build(self,
+              *,
+              toolchain: Optional[Pathish] = None,
+              timeout: Optional[int] = None,
+              tweaks_dir: Optional[Path] = None) -> None:
         """
         Execute 'dds build' on the project
         """
@@ -84,7 +88,8 @@ class Project:
                            build_root=self.build_root,
                            toolchain=tc,
                            timeout=timeout,
-                           more_args=['-ldebug'])
+                           tweaks_dir=tweaks_dir,
+                           more_args=['-ltrace'])
 
     def compile_file(self, *paths: Pathish, toolchain: Optional[Pathish] = None) -> None:
         with tc_mod.fixup_toolchain(toolchain or tc_mod.get_default_test_toolchain()) as tc: