'repoman add' to add repositories based on a URL

src/dds/cli/cmd/repoman_add.cpp

@@ -0,0 +1,84 @@
+#include "../options.hpp"
+
+#include <dds/error/errors.hpp>
+#include <dds/pkg/get/get.hpp>
+#include <dds/pkg/info.hpp>
+#include <dds/repoman/repoman.hpp>
+#include <dds/util/http/pool.hpp>
+#include <dds/util/result.hpp>
+
+#include <boost/leaf/handle_exception.hpp>
+#include <fmt/ostream.h>
+#include <neo/sqlite3/error.hpp>
+
+namespace dds::cli::cmd {
+
+static int _repoman_add(const options& opts) {
+    auto          pkg_id  = dds::pkg_id::parse(opts.repoman.add.pkg_id_str);
+    auto          listing = parse_remote_url(opts.repoman.add.url_str);
+    dds::pkg_info add_info{
+        .ident       = pkg_id,
+        .deps        = {},
+        .description = opts.repoman.add.description,
+        .remote      = listing,
+    };
+    auto temp_sdist = get_package_sdist(add_info);
+
+    add_info.deps = temp_sdist.sdist.manifest.dependencies;
+
+    auto repo = repo_manager::open(opts.repoman.repo_dir);
+    repo.add_pkg(add_info, opts.repoman.add.url_str);
+    return 0;
+}
+
+int repoman_add(const options& opts) {
+    return boost::leaf::try_catch(  //
+        [&] {
+            try {
+                return _repoman_add(opts);
+            } catch (...) {
+                dds::capture_exception();
+            }
+        },
+        [](user_error<errc::invalid_pkg_id>,
+           semver::invalid_version   err,
+           dds::e_invalid_pkg_id_str idstr) -> int {
+            dds_log(error,
+                    "Package ID string '{}' is invalid, because '{}' is not a valid semantic "
+                    "version string",
+                    idstr.value,
+                    err.string());
+            write_error_marker("invalid-pkg-id-str-version");
+            throw;
+        },
+        [](user_error<errc::invalid_pkg_id>, dds::e_invalid_pkg_id_str idstr) -> int {
+            dds_log(error, "Invalid package ID string '{}'", idstr.value);
+            write_error_marker("invalid-pkg-id-str");
+            throw;
+        },
+        [](dds::e_sqlite3_error_exc,
+           boost::leaf::match<neo::sqlite3::errc, neo::sqlite3::errc::constraint_unique>,
+           dds::pkg_id pkid) {
+            dds_log(error, "Package {} is already present in the repository", pkid.to_string());
+            write_error_marker("dup-pkg-add");
+            return 1;
+        },
+        [](http_status_error, http_response_info resp, neo::url url) {
+            dds_log(error,
+                    "Error resulted from HTTP request [{}]: {} {}",
+                    url.to_string(),
+                    resp.status,
+                    resp.status_message);
+            return 1;
+        },
+        [](dds::e_sqlite3_error_exc e, dds::e_repo_import_targz tgz) {
+            dds_log(error, "Database error while importing tar file {}: {}", tgz.path, e.message);
+            return 1;
+        },
+        [](dds::e_system_error_exc e, dds::e_open_repo_db db) {
+            dds_log(error, "Error while opening repository database {}: {}", db.path, e.message);
+            return 1;
+        });
+}
+
+}  // namespace dds::cli::cmd

src/dds/cli/dispatch_main.cpp

@@ -21,6 +21,7 @@ command pkg_import;
 command pkg_ls;
 command pkg_repo_add;
 command pkg_repo_update;
+command repoman_add;
 command repoman_import;
 command repoman_init;
 command repoman_ls;
@@ -66,6 +67,8 @@ int dispatch_main(const options& opts) noexcept {
             switch (opts.repoman.subcommand) {
             case repoman_subcommand::import:
                 return cmd::repoman_import(opts);
+            case repoman_subcommand::add:
+                return cmd::repoman_add(opts);
             case repoman_subcommand::init:
                 return cmd::repoman_init(opts);
             case repoman_subcommand::remove:

src/dds/cli/error_handler.cpp

@@ -55,25 +55,5 @@ auto handlers = std::tuple(  //
 }  // namespace
 
 int dds::handle_cli_errors(std::function<int()> fn) noexcept {
-    return boost::leaf::try_catch(
-        [&] {
-            boost::leaf::context<dds::e_error_marker> marker_ctx;
-            marker_ctx.activate();
-            neo_defer {
-                marker_ctx.deactivate();
-                marker_ctx.handle_error<void>(
-                    boost::leaf::current_error(),
-                    [](dds::e_error_marker mark) {
-                        dds_log(trace, "[error marker {}]", mark.value);
-                        auto efile_path = std::getenv("DDS_WRITE_ERROR_MARKER");
-                        if (efile_path) {
-                            std::ofstream outfile{efile_path, std::ios::binary};
-                            fmt::print(outfile, "{}", mark.value);
-                        }
-                    },
-                    [] {});
-            };
-            return fn();
-        },
-        handlers);
+    return boost::leaf::try_catch(fn, handlers);
 }

src/dds/cli/options.cpp

@@ -344,15 +344,19 @@ struct setup {
             .name = "init",
             .help = "Initialize a directory as a new repository",
         }));
-        setup_repoman_import_cmd(grp.add_parser({
-            .name = "import",
-            .help = "Import a source distribution into the repository",
-        }));
         auto& ls_cmd = grp.add_parser({
             .name = "ls",
             .help = "List the contents of a package repository directory",
         });
         ls_cmd.add_argument(repoman_repo_dir_arg.dup());
+        setup_repoman_add_cmd(grp.add_parser({
+            .name = "add",
+            .help = "Add a package listing to the repository by URL",
+        }));
+        setup_repoman_import_cmd(grp.add_parser({
+            .name = "import",
+            .help = "Import a source distribution into the repository",
+        }));
         setup_repoman_remove_cmd(grp.add_parser({
             .name = "remove",
             .help = "Remove packages from a package repository",
@@ -382,6 +386,27 @@ struct setup {
         });
     }
 
+    void setup_repoman_add_cmd(argument_parser& repoman_add_cmd) {
+        repoman_add_cmd.add_argument(repoman_repo_dir_arg.dup());
+        repoman_add_cmd.add_argument({
+            .help     = "The package ID of the package to add",
+            .valname  = "<pkg-id>",
+            .required = true,
+            .action   = put_into(opts.repoman.add.pkg_id_str),
+        });
+        repoman_add_cmd.add_argument({
+            .help     = "URL to add to the repository",
+            .valname  = "<url>",
+            .required = true,
+            .action   = put_into(opts.repoman.add.url_str),
+        });
+        repoman_add_cmd.add_argument({
+            .long_spellings  = {"description"},
+            .short_spellings = {"d"},
+            .action          = put_into(opts.repoman.add.description),
+        });
+    }
+
     void setup_repoman_remove_cmd(argument_parser& repoman_remove_cmd) {
         repoman_remove_cmd.add_argument(repoman_repo_dir_arg.dup());
         repoman_remove_cmd.add_argument({

src/dds/cli/options.hpp

@@ -65,6 +65,7 @@ enum class repoman_subcommand {
     _none_,
     init,
     import,
+    add,
     remove,
     ls,
 };
@@ -224,6 +225,13 @@ struct options {
             std::vector<fs::path> files;
         } import;
 
+        /// Options for 'dds repoman add'
+        struct {
+            std::string pkg_id_str;
+            std::string url_str;
+            std::string description;
+        } add;
+
         /// Options for 'dds repoman remove'
         struct {
             /// Package IDs of packages to remove

src/dds/pkg/id.cpp

@@ -1,6 +1,7 @@
 #include <dds/pkg/id.hpp>
 
 #include <dds/error/errors.hpp>
+#include <dds/util/result.hpp>
 
 #include <fmt/core.h>
 
@@ -8,7 +9,8 @@
 
 using namespace dds;
 
-pkg_id pkg_id::parse(std::string_view s) {
+pkg_id pkg_id::parse(const std::string_view s) {
+    DDS_E_SCOPE(e_invalid_pkg_id_str{std::string(s)});
     auto at_pos = s.find('@');
     if (at_pos == s.npos) {
         throw_user_error<errc::invalid_pkg_id>("Invalid package ID '{}'", s);
@@ -17,7 +19,12 @@ pkg_id pkg_id::parse(std::string_view s) {
     auto name    = s.substr(0, at_pos);
     auto ver_str = s.substr(at_pos + 1);
 
-    return {std::string(name), semver::version::parse(ver_str)};
+    try {
+        return {std::string(name), semver::version::parse(ver_str)};
+    } catch (const semver::invalid_version& err) {
+        BOOST_LEAF_THROW_EXCEPTION(user_error<errc::invalid_pkg_id>("Package ID string is invalid"),
+                                   err);
+    }
 }
 
 pkg_id::pkg_id(std::string_view n, semver::version v)

src/dds/pkg/id.hpp

@@ -8,6 +8,10 @@
 
 namespace dds {
 
+struct e_invalid_pkg_id_str {
+    std::string value;
+};
+
 /**
  * Represents a unique package ID. We store this as a simple name-version pair.
  *

src/dds/repoman/repoman.cpp

@@ -1,5 +1,6 @@
 #include "./repoman.hpp"
 
+#include <dds/pkg/info.hpp>
 #include <dds/sdist/package.hpp>
 #include <dds/util/log.hpp>
 #include <dds/util/result.hpp>
@@ -16,6 +17,8 @@
 #include <neo/utility.hpp>
 #include <nlohmann/json.hpp>
 
+#include <fstream>
+
 using namespace dds;
 
 namespace nsql = neo::sqlite3;
@@ -148,35 +151,12 @@ void repo_manager::import_targz(path_ref tgz_file) {
     neo::sqlite3::transaction_guard tr{_db};
 
     dds_log(debug, "Recording package {}@{}", man->id.name, man->id.version.to_string());
-    nsql::exec(  //
-        _stmts(R"(
-            INSERT INTO dds_repo_packages (name, version, description, url)
-                VALUES (
-                    ?1,
-                    ?2,
-                    'No description',
-                    printf('dds:%s@%s', ?1, ?2)
-                )
-        )"_sql),
-        man->id.name,
-        man->id.version.to_string());
-
-    auto package_id = _db.last_insert_rowid();
-
-    auto& insert_dep_st = _stmts(R"(
-        INSERT INTO dds_repo_package_deps(package_id, dep_name, low, high)
-            VALUES (?, ?, ?, ?)
-    )"_sql);
-    for (auto& dep : man->dependencies) {
-        assert(dep.versions.num_intervals() == 1);
-        auto iv_1 = *dep.versions.iter_intervals().begin();
-        dds_log(trace, "  Depends on: {}", dep.to_string());
-        nsql::exec(insert_dep_st,
-                   package_id,
-                   dep.name,
-                   iv_1.low.to_string(),
-                   iv_1.high.to_string());
-    }
+    dds::pkg_info info{.ident       = man->id,
+                       .deps        = man->dependencies,
+                       .description = "[No description]",
+                       .remote      = {}};
+    auto          rel_url = fmt::format("dds:{}", man->id.to_string());
+    add_pkg(info, rel_url);
 
     auto dest_path = pkg_dir() / man->id.name / man->id.version.to_string() / "sdist.tar.gz";
     fs::create_directories(dest_path.parent_path());
@@ -220,3 +200,41 @@ void repo_manager::delete_package(pkg_id pkg_id) {
         throw std::system_error(ec, "Failed to delete package name directory");
     }
 }
+
+void repo_manager::add_pkg(const pkg_info& info, std::string_view url) {
+    dds_log(info, "Directly add an entry for {}", info.ident.to_string());
+    DDS_E_SCOPE(info.ident);
+    nsql::recursive_transaction_guard tr{_db};
+    nsql::exec(  //
+        _stmts(R"(
+            INSERT INTO dds_repo_packages (name, version, description, url)
+            VALUES (?, ?, ?, ?)
+        )"_sql),
+        info.ident.name,
+        info.ident.version.to_string(),
+        info.description,
+        url);
+
+    auto package_rowid = _db.last_insert_rowid();
+
+    auto& insert_dep_st = _stmts(R"(
+        INSERT INTO dds_repo_package_deps(package_id, dep_name, low, high)
+        VALUES (?, ?, ?, ?)
+    )"_sql);
+    for (auto& dep : info.deps) {
+        assert(dep.versions.num_intervals() == 1);
+        auto iv_1 = *dep.versions.iter_intervals().begin();
+        dds_log(trace, "  Depends on: {}", dep.to_string());
+        nsql::exec(insert_dep_st,
+                   package_rowid,
+                   dep.name,
+                   iv_1.low.to_string(),
+                   iv_1.high.to_string());
+    }
+
+    auto dest_dir   = pkg_dir() / info.ident.name / info.ident.version.to_string();
+    auto stamp_path = dest_dir / "url.txt";
+    fs::create_directories(dest_dir);
+    std::ofstream stamp_file{stamp_path, std::ios::binary};
+    stamp_file << url;
+}

src/dds/repoman/repoman.hpp

@@ -10,6 +10,8 @@
 
 namespace dds {
 
+struct pkg_info;
+
 struct e_init_repo {
     fs::path path;
 };
@@ -55,6 +57,7 @@ public:
 
     void import_targz(path_ref tgz_path);
     void delete_package(pkg_id id);
+    void add_pkg(const pkg_info& info, std::string_view url);
 
     auto all_packages() const noexcept {
         using namespace neo::sqlite3::literals;

src/dds/repoman/repoman.test.cpp

@@ -1,6 +1,8 @@
 #include <dds/repoman/repoman.hpp>
 
+#include <dds/pkg/info.hpp>
 #include <dds/temp.hpp>
+
 #include <neo/sqlite3/error.hpp>
 
 #include <catch2/catch.hpp>
@@ -12,11 +14,14 @@ const auto THIS_DIR  = THIS_FILE.parent_path();
 const auto REPO_ROOT = (THIS_DIR / "../../../").lexically_normal();
 const auto DATA_DIR  = REPO_ROOT / "data";
 
+struct tmp_repo {
+    dds::temporary_dir tempdir = dds::temporary_dir::create();
+    dds::repo_manager  repo    = dds::repo_manager::create(tempdir.path(), "test-repo");
+};
+
 }  // namespace
 
-TEST_CASE("Open and import into a repository") {
-    auto tdir        = dds::temporary_dir::create();
-    auto repo        = dds::repo_manager::create(tdir.path(), "test-repo");
+TEST_CASE_METHOD(tmp_repo, "Open and import into a repository") {
     auto neo_url_tgz = DATA_DIR / "neo-url@0.2.1.tar.gz";
     repo.import_targz(neo_url_tgz);
     CHECK(dds::fs::is_directory(repo.pkg_dir() / "neo-url/"));
@@ -28,3 +33,16 @@ TEST_CASE("Open and import into a repository") {
     CHECK_THROWS_AS(repo.delete_package(dds::pkg_id::parse("neo-url@0.2.1")), std::system_error);
     CHECK_NOTHROW(repo.import_targz(neo_url_tgz));
 }
+
+TEST_CASE_METHOD(tmp_repo, "Add a package directly") {
+    dds::pkg_info info{
+        .ident       = dds::pkg_id::parse("foo@1.2.3"),
+        .deps        = {},
+        .description = "Something",
+        .remote      = {},
+    };
+    repo.add_pkg(info, "http://example.com");
+    CHECK_THROWS_AS(repo.add_pkg(info, "https://example.com"),
+                    neo::sqlite3::constraint_unique_error);
+    repo.delete_package(dds::pkg_id::parse("foo@1.2.3"));
+}

src/dds/util/result.cpp

@@ -1,7 +1,12 @@
 #include "./result.hpp"
 
+#include <dds/util/log.hpp>
+
+#include <fmt/ostream.h>
 #include <neo/sqlite3/error.hpp>
 
+#include <fstream>
+
 void dds::capture_exception() {
     try {
         throw;
@@ -15,3 +20,12 @@ void dds::capture_exception() {
     // Re-throw as a bare exception.
     throw std::exception();
 }
+
+void dds::write_error_marker(std::string_view error) noexcept {
+    dds_log(trace, "[error marker {}]", error);
+    auto efile_path = std::getenv("DDS_WRITE_ERROR_MARKER");
+    if (efile_path) {
+        std::ofstream outfile{efile_path, std::ios::binary};
+        fmt::print(outfile, "{}", error);
+    }
+}

src/dds/util/result.hpp

@@ -9,6 +9,7 @@
 #include <exception>
 #include <filesystem>
 #include <string>
+#include <string_view>
 
 namespace dds {
 
@@ -68,6 +69,8 @@ struct e_parse_error {
 
 #define DDS_ERROR_MARKER(Value) DDS_E_ARG(::dds::e_error_marker{Value})
 
+void write_error_marker(std::string_view error) noexcept;
+
 /**
  * @brief Generate a leaf::on_error object that loads the given expression into the currently
  * in-flight error if the current scope is exitted via exception or a bad result<>

tests/test_repoman.py

@@ -0,0 +1,33 @@
+import pytest
+
+from dds_ci import dds
+from dds_ci.testing.fixtures import DDSWrapper, Project
+from dds_ci.testing.error import expect_error_marker
+from pathlib import Path
+
+
+@pytest.fixture()
+def tmp_repo(tmp_path: Path, dds: DDSWrapper) -> Path:
+    dds.run(['repoman', 'init', tmp_path])
+    return tmp_path
+
+
+def test_bad_pkg_id(dds: DDSWrapper, tmp_repo: Path) -> None:
+    with expect_error_marker('invalid-pkg-id-str-version'):
+        dds.run(['repoman', 'add', tmp_repo, 'foo@bar', 'http://example.com'])
+
+    with expect_error_marker('invalid-pkg-id-str'):
+        dds.run(['repoman', 'add', tmp_repo, 'foo', 'http://example.com'])
+
+
+def test_add_simple(dds: DDSWrapper, tmp_repo: Path) -> None:
+    dds.run(['repoman', 'add', tmp_repo, 'neo-fun@0.6.0', 'git+https://github.com/vector-of-bool/neo-fun.git#0.6.0'])
+    with expect_error_marker('dup-pkg-add'):
+        dds.run(
+            ['repoman', 'add', tmp_repo, 'neo-fun@0.6.0', 'git+https://github.com/vector-of-bool/neo-fun.git#0.6.0'])
+
+
+def test_add_github(dds: DDSWrapper, tmp_repo: Path) -> None:
+    dds.run(['repoman', 'add', tmp_repo, 'neo-fun@0.6.0', 'github:vector-of-bool/neo-fun#0.6.0'])
+    with expect_error_marker('dup-pkg-add'):
+        dds.run(['repoman', 'add', tmp_repo, 'neo-fun@0.6.0', 'github:vector-of-bool/neo-fun#0.6.0'])