Merge branch 'feature/repodb' into develop

package.dds

@@ -6,7 +6,7 @@ Depends: spdlog 1.4.2
 Depends: ms-wil 2019.11.10
 Depends: range-v3 0.9.1
 Depends: nlohmann-json 3.7.1
-Depends: neo-sqlite3 0.2.0
-Depends: semver 0.1.0
+Depends: neo-sqlite3 0.2.2
+Depends: semver 0.2.0
 
 Test-Driver: Catch-Main

remote.dds

@@ -10,5 +10,5 @@ Remote-Package: ms-wil 2019.11.10; git url=https://github.com/vector-of-bool/wil
 # XXX: Don't depend on a moving revision!
 Remote-Package: neo-buffer 0.1.0; git url=https://github.com/vector-of-bool/neo-buffer.git ref=develop
 
-Remote-Package: neo-sqlite3 0.2.0; git url=https://github.com/vector-of-bool/neo-sqlite3.git ref=0.2.0
-Remote-Package: semver 0.1.0; git url=https://github.com/vector-of-bool/semver.git ref=0.1.0
+Remote-Package: neo-sqlite3 0.2.2; git url=https://github.com/vector-of-bool/neo-sqlite3.git ref=0.2.2
+Remote-Package: semver 0.2.0; git url=https://github.com/vector-of-bool/semver.git ref=0.2.0

src/dds/catalog/catalog.cpp

@@ -0,0 +1,336 @@
+#include "./catalog.hpp"
+
+#include <neo/sqlite3/exec.hpp>
+#include <neo/sqlite3/iter_tuples.hpp>
+#include <neo/sqlite3/single.hpp>
+#include <nlohmann/json.hpp>
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/join.hpp>
+#include <range/v3/view/transform.hpp>
+
+#include <spdlog/spdlog.h>
+
+using namespace dds;
+
+namespace sqlite3 = neo::sqlite3;
+using namespace sqlite3::literals;
+
+namespace {
+
+void migrate_repodb_1(sqlite3::database& db) {
+    db.exec(R"(
+        CREATE TABLE dds_cat_pkgs (
+            pkg_id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            version TEXT NOT NULL,
+            git_url TEXT,
+            git_ref TEXT,
+            lm_name TEXT,
+            lm_namespace TEXT,
+            UNIQUE(name, version),
+            CONSTRAINT has_remote_info CHECK(
+                (
+                    git_url NOT NULL
+                    AND git_ref NOT NULL
+                )
+            ),
+            CONSTRAINT valid_lm_info CHECK(
+                (
+                    lm_name NOT NULL
+                    AND lm_namespace NOT NULL
+                )
+                +
+                (
+                    lm_name ISNULL
+                    AND lm_namespace ISNULL
+                )
+                = 1
+            )
+        );
+
+        CREATE TABLE dds_cat_pkg_deps (
+            dep_id INTEGER PRIMARY KEY AUTOINCREMENT,
+            pkg_id INTEGER NOT NULL REFERENCES dds_cat_pkgs(pkg_id),
+            dep_name TEXT NOT NULL,
+            low TEXT NOT NULL,
+            high TEXT NOT NULL,
+            UNIQUE(pkg_id, dep_name)
+        );
+    )");
+}
+
+void ensure_migrated(sqlite3::database& db) {
+    sqlite3::transaction_guard tr{db};
+    db.exec(R"(
+        PRAGMA foreign_keys = 1;
+        CREATE TABLE IF NOT EXISTS dds_cat_meta AS
+            WITH init(meta) AS (VALUES ('{"version": 0}'))
+            SELECT * FROM init;
+    )");
+    auto meta_st     = db.prepare("SELECT meta FROM dds_cat_meta");
+    auto [meta_json] = sqlite3::unpack_single<std::string>(meta_st);
+
+    auto meta = nlohmann::json::parse(meta_json);
+    if (!meta.is_object()) {
+        throw std::runtime_error("Corrupted repository database file.");
+    }
+
+    auto version_ = meta["version"];
+    if (!version_.is_number_integer()) {
+        throw std::runtime_error("Corrupted repository database file [bad dds_meta.version]");
+    }
+    int version = version_;
+    if (version < 1) {
+        migrate_repodb_1(db);
+    }
+    meta["version"] = 1;
+    exec(db, "UPDATE dds_cat_meta SET meta=?", std::forward_as_tuple(meta.dump()));
+}
+
+}  // namespace
+
+catalog catalog::open(const std::string& db_path) {
+    auto db = sqlite3::database::open(db_path);
+    try {
+        ensure_migrated(db);
+    } catch (const sqlite3::sqlite3_error& e) {
+        spdlog::critical(
+            "Failed to load the repository databsae. It appears to be invalid/corrupted. The "
+            "exception message is: {}",
+            e.what());
+        throw;
+    }
+    return catalog(std::move(db));
+}
+
+catalog::catalog(sqlite3::database db)
+    : _db(std::move(db)) {}
+
+void catalog::_store_pkg(const package_info& pkg, const git_remote_listing& git) {
+    auto lm_usage = git.auto_lib.value_or(lm::usage{});
+    sqlite3::exec(  //
+        _stmt_cache,
+        R"(
+            INSERT INTO dds_cat_pkgs (
+                name,
+                version,
+                git_url,
+                git_ref,
+                lm_name,
+                lm_namespace
+            ) VALUES (
+                ?1,
+                ?2,
+                ?3,
+                ?4,
+                CASE WHEN ?5 = '' THEN NULL ELSE ?5 END,
+                CASE WHEN ?6 = '' THEN NULL ELSE ?6 END
+            )
+        )"_sql,
+        std::forward_as_tuple(  //
+            pkg.ident.name,
+            pkg.ident.version.to_string(),
+            git.url,
+            git.ref,
+            lm_usage.name,
+            lm_usage.namespace_));
+}
+
+void catalog::store(const package_info& pkg) {
+    sqlite3::transaction_guard tr{_db};
+
+    std::visit([&](auto&& remote) { _store_pkg(pkg, remote); }, pkg.remote);
+
+    auto  db_pkg_id  = _db.last_insert_rowid();
+    auto& new_dep_st = _stmt_cache(R"(
+        INSERT INTO dds_cat_pkg_deps (
+            pkg_id,
+            dep_name,
+            low,
+            high
+        ) VALUES (
+            ?,
+            ?,
+            ?,
+            ?
+        )
+    )"_sql);
+    for (const auto& dep : pkg.deps) {
+        new_dep_st.reset();
+        sqlite3::exec(new_dep_st,
+                      std::forward_as_tuple(db_pkg_id,
+                                            dep.name,
+                                            dep.version.to_string(),
+                                            "[placeholder]"));
+    }
+}
+
+std::optional<package_info> catalog::get(const package_id& pk_id) const noexcept {
+    auto& st = _stmt_cache(R"(
+        SELECT
+            pkg_id,
+            name,
+            version,
+            git_url,
+            git_ref,
+            lm_name,
+            lm_namespace
+        FROM dds_cat_pkgs
+        WHERE name = ? AND version = ?
+    )"_sql);
+    st.reset();
+    st.bindings  = std::forward_as_tuple(pk_id.name, pk_id.version.to_string());
+    auto opt_tup = sqlite3::unpack_single_opt<std::int64_t,
+                                              std::string,
+                                              std::string,
+                                              std::optional<std::string>,
+                                              std::optional<std::string>,
+                                              std::optional<std::string>,
+                                              std::optional<std::string>>(st);
+    if (!opt_tup) {
+        return std::nullopt;
+    }
+    const auto& [pkg_id, name, version, git_url, git_ref, lm_name, lm_namespace] = *opt_tup;
+    assert(pk_id.name == name);
+    assert(pk_id.version == semver::version::parse(version));
+    assert(git_url);
+    assert(git_ref);
+
+    auto deps = sqlite3::exec_iter<std::string, std::string>(  //
+                    _stmt_cache,
+                    R"(
+                        SELECT dep_name, low
+                          FROM dds_cat_pkg_deps
+                         WHERE pkg_id = ?
+                    )"_sql,
+                    std::tie(pkg_id))
+        | ranges::views::transform([](auto&& pair) {
+                    const auto& [name, ver] = pair;
+                    return dependency{name, semver::version::parse(ver)};
+                })  //
+        | ranges::to_vector;
+
+    return package_info{
+        pk_id,
+        deps,
+        git_remote_listing{
+            *git_url,
+            *git_ref,
+            lm_name ? std::make_optional(lm::usage{*lm_name, *lm_namespace}) : std::nullopt,
+        },
+    };
+}
+
+std::vector<package_id> catalog::by_name(std::string_view sv) const noexcept {
+    return sqlite3::exec_iter<std::string, std::string>(  //
+               _stmt_cache,
+               R"(
+                SELECT name, version
+                  FROM dds_cat_pkgs
+                 WHERE name = ?
+                )"_sql,
+               std::tie(sv))  //
+        | ranges::views::transform([](auto& pair) {
+               auto& [name, ver] = pair;
+               return package_id{name, semver::version::parse(ver)};
+           })
+        | ranges::to_vector;
+}
+
+std::vector<dependency> catalog::dependencies_of(const package_id& pkg) const noexcept {
+    return sqlite3::exec_iter<std::string,
+                              std::string>(  //
+               _stmt_cache,
+               R"(
+                WITH this_pkg_id AS (
+                    SELECT pkg_id
+                      FROM dds_cat_pkgs
+                     WHERE name = ? AND version = ?
+                )
+                SELECT dep_name, low
+                  FROM dds_cat_pkg_deps
+                 WHERE pkg_id IN this_pkg_id
+              ORDER BY dep_name
+                )"_sql,
+               std::forward_as_tuple(pkg.name, pkg.version.to_string()))  //
+        | ranges::views::transform([](auto&& pair) {
+               auto& [name, ver] = pair;
+               return dependency{name, semver::version::parse(ver)};
+           })  //
+        | ranges::to_vector;
+}
+
+namespace {
+
+void check_json(bool b, std::string_view what) {
+    if (!b) {
+        throw std::runtime_error("Unable to read repository JSON: " + std::string(what));
+    }
+}
+
+}  // namespace
+
+void catalog::import_json_str(std::string_view content) {
+    using nlohmann::json;
+
+    auto root = json::parse(content);
+    check_json(root.is_object(), "Root of JSON must be an object (key-value mapping)");
+
+    auto version = root["version"];
+    check_json(version.is_number_integer(), "/version must be an integral value");
+    check_json(version <= 1, "/version is too new. We don't know how to parse this.");
+
+    auto packages = root["packages"];
+    check_json(packages.is_object(), "/packages must be an object");
+
+    sqlite3::transaction_guard tr{_db};
+
+    for (const auto& [pkg_name_, versions_map] : packages.items()) {
+        std::string pkg_name = pkg_name_;
+        check_json(versions_map.is_object(),
+                   fmt::format("/packages/{} must be an object", pkg_name));
+
+        for (const auto& [version_, pkg_info] : versions_map.items()) {
+            auto version = semver::version::parse(version_);
+            check_json(pkg_info.is_object(),
+                       fmt::format("/packages/{}/{} must be an object", pkg_name, version_));
+
+            auto deps = pkg_info["depends"];
+            check_json(deps.is_object(),
+                       fmt::format("/packages/{}/{}/depends must be an object",
+                                   pkg_name,
+                                   version_));
+
+            package_info info{{pkg_name, version}, {}, {}};
+            for (const auto& [dep_name, dep_version] : deps.items()) {
+                check_json(dep_version.is_string(),
+                           fmt::format("/packages/{}/{}/depends/{} must be a string",
+                                       pkg_name,
+                                       version_,
+                                       dep_name));
+                info.deps.push_back({
+                    std::string(dep_name),
+                    semver::version::parse(std::string(dep_version)),
+                });
+            }
+
+            auto git_remote = pkg_info["git"];
+            if (!git_remote.is_null()) {
+                check_json(git_remote.is_object(), "`git` must be an object");
+                std::string              url      = git_remote["url"];
+                std::string              ref      = git_remote["ref"];
+                auto                     lm_usage = git_remote["auto-lib"];
+                std::optional<lm::usage> autolib;
+                if (!lm_usage.is_null()) {
+                    autolib = lm::split_usage_string(std::string(lm_usage));
+                }
+                info.remote = git_remote_listing{url, ref, autolib};
+            } else {
+                throw std::runtime_error(
+                    fmt::format("No remote info for /packages/{}/{}", pkg_name, version_));
+            }
+
+            store(info);
+        }
+    }
+}

src/dds/catalog/catalog.hpp

@@ -0,0 +1,54 @@
+#pragma once
+
+#include <dds/deps.hpp>
+#include <dds/package_id.hpp>
+#include <dds/util/fs.hpp>
+#include <dds/catalog/git.hpp>
+
+#include <neo/sqlite3/database.hpp>
+#include <neo/sqlite3/statement.hpp>
+#include <neo/sqlite3/statement_cache.hpp>
+#include <neo/sqlite3/transaction.hpp>
+
+#include <variant>
+#include <vector>
+
+namespace dds {
+
+struct package_info {
+    package_id              ident;
+    std::vector<dependency> deps;
+
+    std::variant<git_remote_listing> remote;
+};
+
+class catalog {
+    neo::sqlite3::database                _db;
+    mutable neo::sqlite3::statement_cache _stmt_cache{_db};
+
+    explicit catalog(neo::sqlite3::database db);
+    catalog(const catalog&) = delete;
+
+    void _store_pkg(const package_info&, const git_remote_listing&);
+
+public:
+    catalog(catalog&&) = default;
+    catalog& operator=(catalog&&) = default;
+
+    static catalog open(const std::string& db_path);
+    static catalog open(path_ref db_path) { return open(db_path.string()); }
+
+    void store(const package_info& info);
+    std::optional<package_info> get(const package_id& id) const noexcept;
+
+    std::vector<package_id> by_name(std::string_view sv) const noexcept;
+    std::vector<dependency> dependencies_of(const package_id& pkg) const noexcept;
+
+    void import_json_str(std::string_view json_str);
+    void import_json_file(path_ref json_path) {
+        auto content = dds::slurp_file(json_path);
+        import_json_str(content);
+    }
+};
+
+}  // namespace dds

src/dds/catalog/catalog.test.cpp

@@ -0,0 +1,76 @@
+#include <dds/catalog/catalog.hpp>
+
+#include <catch2/catch.hpp>
+
+using namespace std::literals;
+
+TEST_CASE("Create a simple database") {
+    // Just create and run migrations on an in-memory database
+    auto repo = dds::catalog::open(":memory:"s);
+}
+
+class catalog_test_case {
+public:
+    dds::catalog db = dds::catalog::open(":memory:"s);
+};
+
+TEST_CASE_METHOD(catalog_test_case, "Store a simple package") {
+    db.store(dds::package_info{
+        dds::package_id("foo", semver::version::parse("1.2.3")),
+        {},
+        dds::git_remote_listing{"http://example.com", "master", std::nullopt},
+    });
+    CHECK_THROWS(db.store(dds::package_info{
+        dds::package_id("foo", semver::version::parse("1.2.3")),
+        {},
+        dds::git_remote_listing{"http://example.com", "master", std::nullopt},
+    }));
+
+    auto pkgs = db.by_name("foo");
+    REQUIRE(pkgs.size() == 1);
+}
+
+TEST_CASE_METHOD(catalog_test_case, "Package requirements") {
+    db.store(dds::package_info{
+        dds::package_id{"foo", semver::version::parse("1.2.3")},
+        {
+            {"bar", semver::version::parse("1.2.5")},
+            {"baz", semver::version::parse("5.3.2")},
+        },
+        dds::git_remote_listing{"http://example.com", "master", std::nullopt},
+    });
+    auto pkgs = db.by_name("foo");
+    REQUIRE(pkgs.size() == 1);
+    CHECK(pkgs[0].name == "foo");
+    auto deps = db.dependencies_of(pkgs[0]);
+    CHECK(deps.size() == 2);
+    CHECK(deps[0].name == "bar");
+    CHECK(deps[1].name == "baz");
+}
+
+TEST_CASE_METHOD(catalog_test_case, "Parse JSON repo") {
+    db.import_json_str(R"({
+        "version": 1,
+        "packages": {
+            "foo": {
+                "1.2.3": {
+                    "depends": {
+                        "bar": "4.2.1"
+                    },
+                    "git": {
+                        "url": "http://example.com",
+                        "ref": "master"
+                    }
+                }
+            }
+        }
+    })");
+    auto pkgs = db.by_name("foo");
+    REQUIRE(pkgs.size() == 1);
+    CHECK(pkgs[0].name == "foo");
+    CHECK(pkgs[0].version == semver::version::parse("1.2.3"));
+    auto deps = db.dependencies_of(pkgs[0]);
+    REQUIRE(deps.size() == 1);
+    CHECK(deps[0].name == "bar");
+    CHECK(deps[0].version == semver::version::parse("4.2.1"));
+}

src/dds/catalog/get.cpp

@@ -0,0 +1,63 @@
+#include "./get.hpp"
+
+#include <dds/catalog/catalog.hpp>
+#include <dds/proc.hpp>
+
+#include <spdlog/spdlog.h>
+
+using namespace dds;
+
+namespace {
+
+temporary_sdist do_pull_sdist(const package_info& listing, const git_remote_listing& git) {
+    auto tmpdir = dds::temporary_dir::create();
+    using namespace std::literals;
+    spdlog::info("Cloning Git repository: {} [{}] ...", git.url, git.ref);
+    auto command = {"git"s,
+                    "clone"s,
+                    "--depth=1"s,
+                    "--branch"s,
+                    git.ref,
+                    git.url,
+                    tmpdir.path().generic_string()};
+    auto git_res = run_proc(command);
+    if (!git_res.okay()) {
+        throw std::runtime_error(
+            fmt::format("Git clone operation failed [Git command: {}] [Exitted {}]:\n{}",
+                        quote_command(command),
+                        git_res.retc,
+                        git_res.output));
+    }
+    spdlog::info("Create sdist from clone ...");
+    if (git.auto_lib.has_value()) {
+        spdlog::info("Generating library data automatically");
+        auto pkg_strm = dds::open(tmpdir.path() / "package.dds", std::ios::binary | std::ios::out);
+        pkg_strm << "Name: " << listing.ident.name << '\n'                    //
+                 << "Version: " << listing.ident.version.to_string() << '\n'  //
+                 << "Namespace: " << git.auto_lib->namespace_;
+        auto lib_strm = dds::open(tmpdir.path() / "library.dds", std::ios::binary | std::ios::out);
+        lib_strm << "Name: " << git.auto_lib->name;
+    }
+
+    sdist_params params;
+    params.project_dir = tmpdir.path();
+    auto sd_tmp_dir    = dds::temporary_dir::create();
+    params.dest_path   = sd_tmp_dir.path();
+    params.force       = true;
+    auto sd            = create_sdist(params);
+    return {sd_tmp_dir, sd};
+}
+
+}  // namespace
+
+temporary_sdist dds::get_package_sdist(const package_info& pkg) {
+    auto tsd = std::visit([&](auto&& remote) { return do_pull_sdist(pkg, remote); }, pkg.remote);
+    if (!(tsd.sdist.manifest.pk_id == pkg.ident)) {
+        throw std::runtime_error(fmt::format(
+            "The package name@version in the generated sdist does not match the name listed in "
+            "the remote listing file (expected '{}', but got '{}')",
+            pkg.ident.to_string(),
+            tsd.sdist.manifest.pk_id.to_string()));
+    }
+    return tsd;
+}

src/dds/catalog/get.hpp

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <dds/sdist.hpp>
+#include <dds/temp.hpp>
+
+namespace dds {
+
+struct package_info;
+
+struct temporary_sdist {
+    temporary_dir tmpdir;
+    dds::sdist    sdist;
+};
+
+temporary_sdist get_package_sdist(const package_info&);
+
+} // namespace dds

src/dds/catalog/git.hpp

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <dds/util/fs.hpp>
+
+#include <libman/package.hpp>
+
+#include <optional>
+#include <string>
+
+namespace dds {
+
+struct git_remote_listing {
+    std::string              url;
+    std::string              ref;
+    std::optional<lm::usage> auto_lib;
+
+    void clone(path_ref path) const;
+};
+
+}  // namespace dds

src/dds/dds.main.cpp

@@ -1,5 +1,6 @@
 #include <dds/build.hpp>
-#include <dds/repo/remote.hpp>
+#include <dds/catalog/catalog.hpp>
+#include <dds/catalog/get.hpp>
 #include <dds/repo/repo.hpp>
 #include <dds/sdist.hpp>
 #include <dds/toolchain/from_dds.hpp>
@@ -45,8 +46,8 @@ struct toolchain_flag : string_flag {
     }
 };
 
-struct repo_where_flag : path_flag {
-    repo_where_flag(args::Group& grp)
+struct repo_path_flag : path_flag {
+    repo_path_flag(args::Group& grp)
         : path_flag{grp,
                     "dir",
                     "Path to the DDS repository directory",
@@ -54,6 +55,17 @@ struct repo_where_flag : path_flag {
                     dds::repository::default_local_path()} {}
 };
 
+struct catalog_path_flag : path_flag {
+    catalog_path_flag(args::Group& cmd)
+        : path_flag(cmd,
+                    "catalog-path",
+                    "Override the path to the catalog database",
+                    {"catalog", 'c'},
+                    dds::dds_data_dir() / "catalog.db") {}
+
+    dds::catalog open() { return dds::catalog::open(Get()); }
+};
+
 /**
  * Base class holds the actual argument parser
  */
@@ -92,6 +104,167 @@ struct common_project_flags {
                    dds::fs::current_path()};
 };
 
+/*
+ ######     ###    ########    ###    ##        #######   ######
+##    ##   ## ##      ##      ## ##   ##       ##     ## ##    ##
+##        ##   ##     ##     ##   ##  ##       ##     ## ##
+##       ##     ##    ##    ##     ## ##       ##     ## ##   ####
+##       #########    ##    ######### ##       ##     ## ##    ##
+##    ## ##     ##    ##    ##     ## ##       ##     ## ##    ##
+ ######  ##     ##    ##    ##     ## ########  #######   ######
+*/
+
+struct cli_catalog {
+    cli_base&     base;
+    args::Command cmd{base.cmd_group, "catalog", "Manage the package catalog"};
+    common_flags  _common{cmd};
+
+    args::Group cat_group{cmd, "Catalog subcommands"};
+
+    struct {
+        cli_catalog&  parent;
+        args::Command cmd{parent.cat_group, "create", "Create a catalog database"};
+        common_flags  _common{cmd};
+
+        catalog_path_flag cat_path{cmd};
+
+        int run() {
+            // Simply opening the DB will initialize the catalog
+            cat_path.open();
+            return 0;
+        }
+    } create{*this};
+
+    struct {
+        cli_catalog&  parent;
+        args::Command cmd{parent.cat_group, "import", "Import entries into a catalog"};
+        common_flags  _common{cmd};
+
+        catalog_path_flag cat_path{cmd};
+        args::ValueFlagList<std::string>
+            json_paths{cmd,
+                       "json",
+                       "Import catalog entries from the given JSON files",
+                       {"json", 'j'}};
+
+        int run() {
+            auto cat = cat_path.open();
+            for (const auto& json_fpath : json_paths.Get()) {
+                cat.import_json_file(json_fpath);
+            }
+            return 0;
+        }
+    } import{*this};
+
+    struct {
+        cli_catalog&  parent;
+        args::Command cmd{parent.cat_group, "get", "Obtain an sdist from a catalog listing"};
+        common_flags  _common{cmd};
+
+        catalog_path_flag cat_path{cmd};
+
+        path_flag out{cmd,
+                      "out",
+                      "The directory where the source distributions will be placed",
+                      {"out-dir", 'o'},
+                      dds::fs::current_path()};
+
+        args::PositionalList<std::string> requirements{cmd,
+                                                       "requirement",
+                                                       "The package IDs to obtain"};
+
+        int run() {
+            auto cat = cat_path.open();
+            for (const auto& req : requirements.Get()) {
+                auto id   = dds::package_id::parse(req);
+                auto info = cat.get(id);
+                if (!info) {
+                    throw std::runtime_error(
+                        fmt::format("No package in the catalog matched the ID '{}'", req));
+                }
+                auto tsd      = dds::get_package_sdist(*info);
+                auto out_path = out.Get();
+                auto dest     = out_path / id.to_string();
+                spdlog::info("Create sdist at {}", dest.string());
+                dds::fs::remove_all(dest);
+                dds::safe_rename(tsd.sdist.path, dest);
+            }
+            return 0;
+        }
+    } get{*this};
+
+    struct {
+        cli_catalog&  parent;
+        args::Command cmd{parent.cat_group, "add", "Manually add an entry to the catalog database"};
+        common_flags  _common{cmd};
+
+        catalog_path_flag cat_path{cmd};
+
+        args::Positional<std::string> pkg_id{cmd,
+                                             "id",
+                                             "The name@version ID of the package to add",
+                                             args::Options::Required};
+
+        string_flag auto_lib{cmd,
+                             "auto-lib",
+                             "Set the auto-library information for this package",
+                             {"auto-lib"}};
+
+        args::ValueFlagList<std::string> deps{cmd,
+                                              "depends",
+                                              "The dependencies of this package",
+                                              {"depends", 'd'}};
+
+        string_flag git_url{cmd, "git-url", "The Git url for the package", {"git-url"}};
+        string_flag git_ref{cmd,
+                            "git-ref",
+                            "The Git ref to from which the source distribution should be created",
+                            {"git-ref"}};
+
+        int run() {
+            auto ident = dds::package_id::parse(pkg_id.Get());
+
+            std::vector<dds::dependency> deps;
+            for (const auto& dep : this->deps.Get()) {
+                auto dep_id = dds::package_id::parse(dep);
+                deps.push_back({dep_id.name, dep_id.version});
+            }
+
+            dds::package_info info{ident, std::move(deps), {}};
+
+            if (git_url) {
+                if (!git_ref) {
+                    throw std::runtime_error(
+                        "`--git-ref` must be specified when using `--git-url`");
+                }
+                auto git = dds::git_remote_listing{git_url.Get(), git_ref.Get(), std::nullopt};
+                if (auto_lib) {
+                    git.auto_lib = lm::split_usage_string(auto_lib.Get());
+                }
+                info.remote = std::move(git);
+            }
+
+            cat_path.open().store(info);
+            return 0;
+        }
+    } add{*this};
+
+    int run() {
+        if (create.cmd) {
+            return create.run();
+        } else if (import.cmd) {
+            return import.run();
+        } else if (get.cmd) {
+            return get.run();
+        } else if (add.cmd) {
+            return add.run();
+        } else {
+            assert(false);
+            std::terminate();
+        }
+    }
+};
+
 /*
 ########  ######## ########   #######
 ##     ## ##       ##     ## ##     ##
@@ -107,7 +280,7 @@ struct cli_repo {
     args::Command cmd{base.cmd_group, "repo", "Manage the package repository"};
     common_flags  _common{cmd};
 
-    repo_where_flag where{cmd};
+    repo_path_flag where{cmd};
 
     args::Group repo_group{cmd, "Repo subcommands"};
 
@@ -118,8 +291,9 @@ struct cli_repo {
 
         int run() {
             auto list_contents = [&](dds::repository repo) {
-                auto same_name
-                    = [](auto&& a, auto&& b) { return a.manifest.pk_id.name == b.manifest.pk_id.name; };
+                auto same_name = [](auto&& a, auto&& b) {
+                    return a.manifest.pk_id.name == b.manifest.pk_id.name;
+                };
 
                 auto all         = repo.iter_sdists();
                 auto grp_by_name = all                             //
@@ -226,8 +400,8 @@ struct cli_sdist {
 
         common_project_flags project{cmd};
 
-        repo_where_flag repo_where{cmd};
-        args::Flag      force{cmd,
+        repo_path_flag repo_where{cmd};
+        args::Flag     force{cmd,
                          "replace-if-exists",
                          "Replace an existing export in the repository",
                          {"replace"}};
@@ -290,6 +464,7 @@ struct cli_build {
     args::Flag     build_apps{cmd, "build_apps", "Build applications", {"apps", 'A'}};
     args::Flag     export_{cmd, "export", "Generate a library export", {"export", 'E'}};
     toolchain_flag tc_filepath{cmd};
+
     args::Flag enable_warnings{cmd,
                                "enable_warnings",
                                "Enable compiler warnings",
@@ -378,18 +553,14 @@ struct cli_deps {
                           "Ensure we have local copies of the project dependencies"};
         common_flags  _common{cmd};
 
-        repo_where_flag repo_where{cmd};
-        path_flag       remote_listing_file{
-            cmd,
-            "remote-listing",
-            "Path to a file containing listing of remote sdists and how to obtain them",
-            {'R', "remote-list"},
-            "remote.dds"};
+        repo_path_flag    repo_where{cmd};
+        catalog_path_flag catalog_path{cmd};
+
 
         int run() {
-            auto man    = parent.load_package_manifest();
-            auto rd     = dds::remote_directory::load_from_file(remote_listing_file.Get());
-            bool failed = false;
+            auto man     = parent.load_package_manifest();
+            auto catalog = catalog_path.open();
+            bool failed  = false;
             dds::repository::with_repository(  //
                 repo_where.Get(),
                 dds::repo_flags::write_lock | dds::repo_flags::create_if_absent,
@@ -397,13 +568,13 @@ struct cli_deps {
                     for (auto& dep : man.dependencies) {
                         auto exists = !!repo.find(dep.name, dep.version);
                         if (!exists) {
-                            spdlog::info("Pull remote: {} {}", dep.name, dep.version.to_string());
-                            auto opt_remote = rd.find(dep.name, dep.version);
-                            if (opt_remote) {
-                                auto tsd = opt_remote->pull_sdist();
+                            spdlog::info("Pull remote: {}@{}", dep.name, dep.version.to_string());
+                            auto opt_pkg = catalog.get(dds::package_id{dep.name, dep.version});
+                            if (opt_pkg) {
+                                auto tsd = dds::get_package_sdist(*opt_pkg);
                                 repo.add_sdist(tsd.sdist, dds::if_exists::ignore);
                             } else {
-                                spdlog::error("No remote listing for {} {}",
+                                spdlog::error("No remote listing for {}@{}",
                                               dep.name,
                                               dep.version.to_string());
                                 failed = true;
@@ -440,7 +611,7 @@ struct cli_deps {
                           "If specified, will not generate an INDEX.lmi",
                           {"skip-lmi"}};
 
-        repo_where_flag repo_where{cmd};
+        repo_path_flag repo_where{cmd};
 
         toolchain_flag tc_filepath{cmd};
 
@@ -502,11 +673,12 @@ int main(int argc, char** argv) {
     spdlog::set_pattern("[%H:%M:%S] [%^%-5l%$] %v");
     args::ArgumentParser parser("DDS - The drop-dead-simple library manager");
 
-    cli_base  cli{parser};
-    cli_build build{cli};
-    cli_sdist sdist{cli};
-    cli_repo  repo{cli};
-    cli_deps  deps{cli};
+    cli_base    cli{parser};
+    cli_build   build{cli};
+    cli_sdist   sdist{cli};
+    cli_repo    repo{cli};
+    cli_deps    deps{cli};
+    cli_catalog catalog{cli};
     try {
         parser.ParseCLI(argc, argv);
     } catch (const args::Help&) {
@@ -532,6 +704,8 @@ int main(int argc, char** argv) {
             return repo.run();
         } else if (deps.cmd) {
             return deps.run();
+        } else if (catalog.cmd) {
+            return catalog.run();
         } else {
             assert(false);
             std::terminate();

src/dds/repo/remote.cpp

@@ -1,135 +0,0 @@
-#include "./remote.hpp"
-
-#include <dds/deps.hpp>
-#include <dds/proc.hpp>
-#include <dds/repo/repo.hpp>
-#include <dds/sdist.hpp>
-#include <dds/temp.hpp>
-#include <dds/toolchain/toolchain.hpp>
-#include <dds/util/shlex.hpp>
-
-#include <spdlog/spdlog.h>
-
-#include <libman/parse.hpp>
-
-#include <algorithm>
-
-using namespace dds;
-
-namespace {
-
-struct read_listing_item {
-    std::string_view                                    _key;
-    std::set<remote_listing, remote_listing_compare_t>& out;
-
-    bool operator()(std::string_view context, std::string_view key, std::string_view value) {
-        if (key != _key) {
-            return false;
-        }
-
-        auto nested = lm::nested_kvlist::parse(value);
-        auto pk_id  = package_id::parse(nested.primary);
-        put_listing(context, std::move(pk_id), nested.pairs);
-        return true;
-    }
-
-    void put_listing(std::string_view context, package_id pk_id, const lm::pair_list& pairs) {
-        if (pairs.find("git")) {
-            std::string              url;
-            std::string              ref;
-            std::optional<lm::usage> auto_id;
-            lm::read(fmt::format("{}: Parsing Git remote listing", context),
-                     pairs,
-                     lm::read_required("url", url),
-                     lm::read_required("ref", ref),
-                     lm::read_check_eq("git", ""),
-                     lm::read_opt("auto", auto_id, &lm::split_usage_string),
-                     lm::reject_unknown());
-            auto did_insert = out.emplace(remote_listing{std::move(pk_id),
-                                                         git_remote_listing{url, ref, auto_id}})
-                                  .second;
-            if (!did_insert) {
-                spdlog::warn("Duplicate remote package defintion for {}", pk_id.to_string());
-            }
-        } else {
-            throw std::runtime_error(
-                fmt::format("Unable to determine remote type of package {}", pk_id.to_string()));
-        }
-    }
-};
-
-temporary_sdist do_pull_sdist(const remote_listing& listing, const git_remote_listing& git) {
-    auto tmpdir = dds::temporary_dir::create();
-    using namespace std::literals;
-    spdlog::info("Cloning repository: {} [{}] ...", git.url, git.ref);
-    auto command = {"git"s,
-                    "clone"s,
-                    "--depth=1"s,
-                    "--branch"s,
-                    git.ref,
-                    git.url,
-                    tmpdir.path().generic_string()};
-    auto git_res = run_proc(command);
-    if (!git_res.okay()) {
-        throw std::runtime_error(
-            fmt::format("Git clone operation failed [Git command: {}] [Exitted {}]:\n{}",
-                        quote_command(command),
-                        git_res.retc,
-                        git_res.output));
-    }
-    spdlog::info("Create sdist from clone ...");
-    if (git.auto_lib.has_value()) {
-        spdlog::info("Generating library data automatically");
-        auto pkg_strm = dds::open(tmpdir.path() / "package.dds", std::ios::binary | std::ios::out);
-        pkg_strm << "Name: " << listing.pk_id.name << '\n'                    //
-                 << "Version: " << listing.pk_id.version.to_string() << '\n'  //
-                 << "Namespace: " << git.auto_lib->namespace_;
-        auto lib_strm = dds::open(tmpdir.path() / "library.dds", std::ios::binary | std::ios::out);
-        lib_strm << "Name: " << git.auto_lib->name;
-    }
-
-    sdist_params params;
-    params.project_dir = tmpdir.path();
-    auto sd_tmp_dir    = dds::temporary_dir::create();
-    params.dest_path   = sd_tmp_dir.path();
-    params.force       = true;
-    auto sd            = create_sdist(params);
-    return {sd_tmp_dir, sd};
-}
-
-}  // namespace
-
-temporary_sdist remote_listing::pull_sdist() const {
-    auto tsd = visit([&](auto&& actual) { return do_pull_sdist(*this, actual); });
-    if (!(tsd.sdist.manifest.pk_id == pk_id)) {
-        throw std::runtime_error(fmt::format(
-            "The package name@version in the generated sdist does not match the name listed in "
-            "the remote listing file (expected '{}', but got '{}')",
-            pk_id.to_string(),
-            tsd.sdist.manifest.pk_id.to_string()));
-    }
-    return tsd;
-}
-
-remote_directory remote_directory::load_from_file(path_ref filepath) {
-    auto        kvs = lm::parse_file(filepath);
-    listing_set listings;
-    lm::read(fmt::format("Loading remote package listing from {}", filepath.string()),
-             kvs,
-             read_listing_item{"Remote-Package", listings},
-             lm::reject_unknown());
-    return {std::move(listings)};
-}
-
-const remote_listing* remote_directory::find(std::string_view name, semver::version ver) const
-    noexcept {
-    auto found = _remotes.find(std::tie(name, ver));
-    if (found == _remotes.end()) {
-        return nullptr;
-    }
-    return &*found;
-}
-
-void remote_directory::ensure_all_local(const repository&) const {
-    spdlog::critical("Dependency download is not fully implemented!");
-}

src/dds/repo/remote.hpp

@@ -1,75 +0,0 @@
-#pragma once
-
-#include <dds/util/fs.hpp>
-
-#include <dds/sdist.hpp>
-#include <dds/temp.hpp>
-#include <libman/library.hpp>
-#include <semver/version.hpp>
-
-#include <set>
-#include <string>
-#include <tuple>
-#include <utility>
-#include <variant>
-
-namespace dds {
-
-struct temporary_sdist {
-    temporary_dir tmpdir;
-    dds::sdist    sdist;
-};
-
-struct git_remote_listing {
-    std::string              url;
-    std::string              ref;
-    std::optional<lm::usage> auto_lib;
-
-    void clone(path_ref path) const;
-};
-
-struct remote_listing {
-    package_id                       pk_id;
-    std::variant<git_remote_listing> remote;
-
-    template <typename Func>
-    decltype(auto) visit(Func&& fn) const {
-        return std::visit(std::forward<Func>(fn), remote);
-    }
-
-    temporary_sdist pull_sdist() const;
-};
-
-inline constexpr struct remote_listing_compare_t {
-    using is_transparent = int;
-    bool operator()(const remote_listing& lhs, const remote_listing& rhs) const {
-        return lhs.pk_id < rhs.pk_id;
-    }
-    template <typename Name, typename Version>
-    bool operator()(const remote_listing& lhs, const std::tuple<Name, Version>& rhs) const {
-        auto&& [name, ver] = rhs;
-        return lhs.pk_id < package_id{name, ver};
-    }
-    template <typename Name, typename Version>
-    bool operator()(const std::tuple<Name, Version>& lhs, const remote_listing& rhs) const {
-        auto&& [name, ver] = lhs;
-        return package_id{name, ver} < rhs.pk_id;
-    }
-} remote_listing_compare;
-
-class remote_directory {
-    using listing_set = std::set<remote_listing, remote_listing_compare_t>;
-    listing_set _remotes;
-
-    remote_directory(listing_set s)
-        : _remotes(std::move(s)) {}
-
-public:
-    static remote_directory load_from_file(path_ref);
-
-    void ensure_all_local(const class repository& repo) const;
-
-    const remote_listing* find(std::string_view name, semver::version ver) const noexcept;
-};
-
-}  // namespace dds

tests/basics/test_app_only.py

@@ -3,8 +3,8 @@ from tests import DDS
 from tests.fileutil import set_contents
 
 
-def test_lib_with_just_app(dds: DDS, scope: ExitStack):
-    scope.enter_context(
+def test_lib_with_just_app(dds: DDS):
+    dds.scope.enter_context(
         set_contents(
             dds.source_root / 'src/foo.main.cpp',
             b'int main() {}',

tests/catalog/create_test.py

@@ -0,0 +1,8 @@
+from tests import dds, DDS
+from tests.fileutil import ensure_dir
+
+
+def test_create_catalog(dds: DDS):
+    dds.scope.enter_context(ensure_dir(dds.build_dir))
+    dds.catalog_create()
+    assert dds.catalog_path.is_file()

tests/catalog/get_test.py

@@ -0,0 +1,34 @@
+import json
+
+from tests import dds, DDS
+from tests.fileutil import ensure_dir
+
+
+def test_get(dds: DDS):
+    dds.scope.enter_context(ensure_dir(dds.build_dir))
+    dds.catalog_create()
+
+    json_path = dds.build_dir / 'catalog.json'
+    import_data = {
+        'version': 1,
+        'packages': {
+            'neo-sqlite3': {
+                '0.2.2': {
+                    'depends': {},
+                    'git': {
+                        'url': 'https://github.com/vector-of-bool/neo-sqlite3.git',
+                        'ref': '0.2.2',
+                    },
+                },
+            },
+        },
+    }
+    dds.scope.enter_context(
+        dds.set_contents(json_path,
+                         json.dumps(import_data).encode()))
+
+    dds.catalog_import(json_path)
+
+    dds.catalog_get('neo-sqlite3@0.2.2')
+    assert (dds.source_root / 'neo-sqlite3@0.2.2').is_dir()
+    assert (dds.source_root / 'neo-sqlite3@0.2.2/package.dds').is_file()

tests/catalog/import_test.py

@@ -0,0 +1,29 @@
+import json
+
+from tests import dds, DDS
+from tests.fileutil import ensure_dir
+
+
+def test_import_json(dds: DDS):
+    dds.scope.enter_context(ensure_dir(dds.build_dir))
+    dds.catalog_create()
+
+    json_fpath = dds.build_dir / 'data.json'
+    import_data = {
+        'version': 1,
+        'packages': {
+            'foo': {
+                '1.2.4': {
+                    'git': {
+                        'url': 'http://example.com',
+                        'ref': 'master',
+                    },
+                    'depends': {},
+                },
+            },
+        },
+    }
+    dds.scope.enter_context(
+        dds.set_contents(json_fpath,
+                         json.dumps(import_data).encode()))
+    dds.catalog_import(json_fpath)

tests/dds.py

@@ -27,6 +27,10 @@ class DDS:
     def repo_dir(self) -> Path:
         return self.scratch_dir / 'repo'
 
+    @property
+    def catalog_path(self) -> Path:
+        return self.scratch_dir / 'catalog.db'
+
     @property
     def deps_build_dir(self) -> Path:
         return self.scratch_dir / 'deps-build'
@@ -51,7 +55,7 @@ class DDS:
     def run(self, cmd: proc.CommandLine, *,
             cwd: Path = None) -> subprocess.CompletedProcess:
         cmdline = list(proc.flatten_cmd(cmd))
-        res = self.run_unchecked(cmd)
+        res = self.run_unchecked(cmd, cwd=cwd)
         if res.returncode != 0:
             raise subprocess.CalledProcessError(
                 res.returncode, [self.dds_exe] + cmdline, res.stdout)
@@ -72,6 +76,7 @@ class DDS:
         return self.run([
             'deps',
             'get',
+            f'--catalog={self.catalog_path}',
             self.repo_dir_arg,
         ])
 
@@ -143,6 +148,29 @@ class DDS:
                 f'We don\'t know the executable suffix for the platform "{os.name}"'
             )
 
+    def catalog_create(self) -> subprocess.CompletedProcess:
+        self.scratch_dir.mkdir(parents=True, exist_ok=True)
+        return self.run(
+            ['catalog', 'create', f'--catalog={self.catalog_path}'],
+            cwd=self.test_dir)
+
+    def catalog_import(self, json_path: Path) -> subprocess.CompletedProcess:
+        self.scratch_dir.mkdir(parents=True, exist_ok=True)
+        return self.run([
+            'catalog',
+            'import',
+            f'--catalog={self.catalog_path}',
+            f'--json={json_path}',
+        ])
+
+    def catalog_get(self, req: str) -> subprocess.CompletedProcess:
+        return self.run([
+            'catalog',
+            'get',
+            f'--catalog={self.catalog_path}',
+            req,
+        ])
+
     def set_contents(self, path: Union[str, Path],
                      content: bytes) -> ContextManager[Path]:
         return fileutil.set_contents(self.source_root / path, content)

tests/deps/do_test.py

@@ -16,6 +16,7 @@ def test_ls(dds: DDS):
 
 @dds_conf
 def test_deps_build(dds: DDS):
+    dds.catalog_import(dds.source_root / 'catalog.json')
     assert not dds.repo_dir.exists()
     dds.deps_get()
     assert dds.repo_dir.exists(), '`deps get` did not generate a repo directory'
@@ -27,6 +28,7 @@ def test_deps_build(dds: DDS):
 
 @dds_fixture_conf_1('use-remote')
 def test_use_nlohmann_json_remote(dds: DDS):
+    dds.catalog_import(dds.source_root / 'catalog.json')
     dds.deps_get()
     dds.deps_build()
     dds.build(apps=True)

tests/deps/git-remote/catalog.json

@@ -0,0 +1,24 @@
+{
+    "version": 1,
+    "packages": {
+        "neo-buffer": {
+            "0.1.0": {
+                "git": {
+                    "url": "https://github.com/vector-of-bool/neo-buffer.git",
+                    "ref": "develop"
+                },
+                "depends": {}
+            }
+        },
+        "range-v3": {
+            "0.9.1": {
+                "git": {
+                    "url": "https://github.com/ericniebler/range-v3.git",
+                    "ref": "0.9.1",
+                    "auto-lib": "Niebler/range-v3"
+                },
+                "depends": {}
+            }
+        }
+    }
+}

tests/deps/git-remote/remote.dds

@@ -1,2 +0,0 @@
-Remote-Package: neo-buffer@0.1.0; git url=https://github.com/vector-of-bool/neo-buffer.git ref=develop
-Remote-Package: range-v3@0.9.1; git url=https://github.com/ericniebler/range-v3.git ref=0.9.1 auto=Niebler/range-v3

tests/deps/no-deps/catalog.json

@@ -0,0 +1,4 @@
+{
+    "version": 1,
+    "packages": {}
+}

tests/deps/use-remote/catalog.json

@@ -0,0 +1,14 @@
+{
+    "version": 1,
+    "packages": {
+        "nlohmann-json": {
+            "3.7.1": {
+                "git": {
+                    "url": "https://github.com/vector-of-bool/json.git",
+                    "ref": "dds/3.7.1"
+                },
+                "depends": {}
+            }
+        }
+    }
+}

tests/deps/use-remote/remote.dds

@@ -1 +0,0 @@
-Remote-Package: nlohmann-json@3.7.1; git url=https://github.com/vector-of-bool/json.git ref=dds/3.7.1

tests/deps/use-spdlog/project/catalog.json

@@ -0,0 +1,15 @@
+{
+    "version": 1,
+    "packages": {
+        "spdlog": {
+            "1.4.2": {
+                "git": {
+                    "url": "https://github.com/gabime/spdlog.git",
+                    "ref": "v1.4.2",
+                    "auto-lib": "spdlog/spdlog"
+                },
+                "depends": {}
+            }
+        }
+    }
+}

tests/deps/use-spdlog/project/remote.dds

@@ -1 +0,0 @@
-Remote-Package: spdlog@1.4.2; git url=https://github.com/gabime/spdlog.git ref=v1.4.2 auto=spdlog/spdlog

tests/deps/use-spdlog/use_spdlog_test.py

@@ -4,6 +4,7 @@ from dds_ci import proc
 
 
 def test_get_build_use_spdlog(dds: DDS):
+    dds.catalog_import(dds.source_root / 'catalog.json')
     dds.deps_get()
     tc_fname = 'gcc.tc.dds' if 'gcc' in dds.default_builtin_toolchain else 'msvc.tc.dds'
     tc = str(dds.test_dir / tc_fname)