We can pull basic catalog info via HTTP

src/dds.main.cpp

@@ -3,6 +3,8 @@
 #include <dds/catalog/get.hpp>
 #include <dds/dym.hpp>
 #include <dds/error/errors.hpp>
+#include <dds/http/session.hpp>
+#include <dds/remote/remote.hpp>
 #include <dds/repo/repo.hpp>
 #include <dds/repoman/repoman.hpp>
 #include <dds/source/dist.hpp>
@@ -697,6 +699,80 @@ struct cli_repo {
         }
     } import_{*this};
 
+    struct {
+        cli_repo&     parent;
+        args::Command cmd{parent.repo_group, "add", "Add a remote repository"};
+        common_flags  _flags{cmd};
+
+        catalog_path_flag cat_path{cmd};
+
+        args::Positional<std::string> url{cmd,
+                                          "<url>",
+                                          "URL of a repository to add",
+                                          args::Options::Required};
+
+        args::Flag update{cmd, "update", "Update catalog contents immediately", {"update", 'U'}};
+
+        int run() {
+            return boost::leaf::try_handle_all(  //
+                [&]() -> dds::result<int> {
+                    try {
+                        auto cat  = cat_path.open();
+                        auto repo = dds::remote_repository::connect(url.Get());
+                        repo.store(cat.database());
+                        if (update) {
+                            repo.update_catalog(cat.database());
+                        }
+                    } catch (...) {
+                        return dds::capture_exception();
+                    }
+                    return 0;
+                },
+                [&](neo::url_validation_error url_err, dds::e_url_string bad_url) {
+                    dds_log(error, "Invalid URL [{}]: {}", bad_url.value, url_err.what());
+                    return 1;
+                },
+                [&](const json5::parse_error& e, dds::e_http_url bad_url) {
+                    dds_log(error,
+                            "Error parsing JSON downloaded from URL [{}]: {}",
+                            bad_url.value,
+                            e.what());
+                    return 1;
+                },
+                [](dds::e_sqlite3_error_exc e, dds::e_url_string url) {
+                    dds_log(error,
+                            "Error accessing remote database (From {}): {}",
+                            url.value,
+                            e.message);
+                    return 1;
+                },
+                [](dds::e_sqlite3_error_exc e) {
+                    dds_log(error, "Unexpected database error: {}", e.message);
+                    return 1;
+                },
+                [&](dds::e_system_error_exc e, dds::e_http_connect conn) {
+                    dds_log(error,
+                            "Error opening connection to [{}:{}]: {}",
+                            conn.host,
+                            conn.port,
+                            e.message);
+                    return 1;
+                },
+                [](const std::exception& e) {
+                    dds_log(error, "An unknown unhandled exception occurred: {}", e.what());
+                    return 1;
+                },
+                [](dds::e_system_error_exc e) {
+                    dds_log(error, "An unknown system_error occurred: {}", e.message);
+                    return 42;
+                },
+                [](boost::leaf::diagnostic_info const& info) {
+                    dds_log(error, "An unnknown error occurred? {}", info);
+                    return 42;
+                });
+        }
+    } add{*this};
+
     struct {
         cli_repo&     parent;
         args::Command cmd{parent.repo_group, "init", "Initialize a directory as a repository"};
@@ -720,6 +796,8 @@ struct cli_repo {
             return init.run();
         } else if (import_.cmd) {
             return import_.run();
+        } else if (add.cmd) {
+            return add.run();
         } else {
             assert(false);
             std::terminate();

src/dds/catalog/catalog.cpp

@@ -81,7 +81,7 @@ void migrate_repodb_3(nsql::database& db) {
     db.exec(R"(
         CREATE TABLE dds_cat_remotes (
             remote_id INTEGER PRIMARY KEY AUTOINCREMENT,
-            ident TEXT NOT NULL UNIQUE,
+            name TEXT NOT NULL UNIQUE,
             gen_ident TEXT NOT NULL,
             remote_url TEXT NOT NULL
         );

src/dds/remote/remote.cpp

@@ -0,0 +1,111 @@
+#include "./remote.hpp"
+
+#include <dds/error/errors.hpp>
+#include <dds/http/session.hpp>
+#include <dds/temp.hpp>
+#include <dds/util/result.hpp>
+
+#include <neo/sqlite3/exec.hpp>
+#include <neo/sqlite3/single.hpp>
+#include <neo/sqlite3/transaction.hpp>
+#include <neo/url.hpp>
+#include <neo/utility.hpp>
+
+using namespace dds;
+namespace nsql = neo::sqlite3;
+
+namespace {
+
+struct remote_db {
+    temporary_dir  _tempdir;
+    nsql::database db;
+
+    static remote_db download_and_open(neo::url const& url) {
+        neo_assert(expects,
+                   url.host.has_value(),
+                   "URL does not have a hostname??",
+                   url.to_string());
+        auto sess = url.scheme == "https"
+            ? http_session::connect_ssl(*url.host, url.port_or_default_port_or(443))
+            : http_session::connect(*url.host, url.port_or_default_port_or(80));
+
+        auto tempdir    = temporary_dir::create();
+        auto repo_db_dl = tempdir.path() / "repo.db";
+        fs::create_directories(tempdir.path());
+        sess.download_file(
+            {
+                .method = "GET",
+                .path   = url.path,
+            },
+            repo_db_dl);
+
+        auto db = nsql::open(repo_db_dl.string());
+        return {tempdir, std::move(db)};
+    }
+
+    static remote_db download_and_open_for_base(neo::url url) {
+        auto repo_url = url;
+        repo_url.path = fs::path(url.path).append("repo.db").string();
+        return download_and_open(repo_url);
+    }
+
+    static remote_db download_and_open_for_base(std::string_view url_str) {
+        return download_and_open_for_base(neo::url::parse(url_str));
+    }
+};
+
+}  // namespace
+
+remote_repository remote_repository::connect(std::string_view url_str) {
+    DDS_E_SCOPE(e_url_string{std::string(url_str)});
+    const auto url = neo::url::parse(url_str);
+
+    auto db      = remote_db::download_and_open_for_base(url);
+    auto name_st = db.db.prepare("SELECT name FROM dds_repo_meta");
+    auto [name]  = nsql::unpack_single<std::string>(name_st);
+
+    remote_repository ret;
+    ret._base_url = url;
+    ret._name     = name;
+    return ret;
+}
+
+void remote_repository::store(nsql::database_ref db) {
+    auto st = db.prepare(R"(
+        INSERT INTO dds_cat_remotes (name, gen_ident, remote_url)
+            VALUES (?, ?, ?)
+    )");
+    nsql::exec(st, _name, "[placeholder]", _base_url.to_string());
+}
+
+void remote_repository::update_catalog(nsql::database_ref db) {
+    auto rdb = remote_db::download_and_open_for_base(_base_url);
+
+    auto db_path = rdb._tempdir.path() / "repo.db";
+
+    auto rid_st          = db.prepare("SELECT remote_id FROM dds_cat_remotes WHERE name = ?");
+    rid_st.bindings()[1] = _name;
+    auto [remote_id]     = nsql::unpack_single<std::int64_t>(rid_st);
+
+    nsql::transaction_guard tr{db};
+    nsql::exec(db.prepare("ATTACH DATABASE ? AS remote"), db_path.string());
+    nsql::exec(  //
+        db.prepare(R"(
+            DELETE FROM dds_cat_pkgs
+            WHERE remote_id = ?
+        )"),
+        remote_id);
+    nsql::exec(  //
+        db.prepare(R"(
+            INSERT INTO dds_cat_pkgs
+                (name, version, description, remote_url, remote_id)
+            SELECT
+                name,
+                version,
+                description,
+                printf('dds:%s/%s', name, version),
+                ?1
+            FROM remote.dds_repo_packages
+        )"),
+        remote_id);
+}

src/dds/remote/remote.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <dds/util/fs.hpp>
+#include <dds/util/result.hpp>
+
+#include <neo/concepts.hpp>
+#include <neo/sqlite3/database.hpp>
+#include <neo/url.hpp>
+
+#include <string_view>
+#include <variant>
+
+namespace dds {
+
+class remote_repository {
+    std::string _name;
+    neo::url    _base_url;
+
+    remote_repository() = default;
+
+public:
+    static remote_repository connect(std::string_view url);
+
+    // const repository_manifest& manifest() const noexcept;
+
+    void store(neo::sqlite3::database_ref);
+    void update_catalog(neo::sqlite3::database_ref);
+};
+
+}  // namespace dds

src/dds/util/result.hpp

@@ -30,6 +30,10 @@ struct e_sqlite3_error_exc {
     std::error_code code;
 };
 
+struct e_url_string {
+    std::string value;
+};
+
 /**
  * @brief Capture currently in-flight special exceptions as new error object. Works around a bug in
  * Boost.LEAF when catching std::system error.

tests/catalog/import_test.py

@@ -67,6 +67,21 @@ def http_import_server():
             httpd.shutdown()
 
 
+@pytest.yield_fixture
+def http_repo_server():
+    handler = partial(
+        DirectoryServingHTTPRequestHandler,
+        dir=Path.cwd() / 'data/test-repo-1')
+    addr = ('0.0.0.0', 4646)
+    pool = ThreadPoolExecutor()
+    with HTTPServer(addr, handler) as httpd:
+        pool.submit(lambda: httpd.serve_forever(poll_interval=0.1))
+        try:
+            yield
+        finally:
+            httpd.shutdown()
+
+
 def test_import_http(dds: DDS, http_import_server):
     dds.repo_dir.mkdir(parents=True, exist_ok=True)
     dds.run(
@@ -74,8 +89,21 @@ def test_import_http(dds: DDS, http_import_server):
             'repo',
             dds.repo_dir_arg,
             'import',
-            'https://github.com/vector-of-bool/neo-buffer/archive/0.4.2.tar.gz?dds_strpcmp=1',
+            'http://localhost:8000/neo-buffer-0.4.2.tar.gz',
         ],
         cwd=dds.repo_dir,
     )
     assert dds.repo_dir.joinpath('neo-buffer@0.4.2').is_dir()
+
+
+def test_repo_add(dds: DDS, http_repo_server):
+    dds.repo_dir.mkdir(parents=True, exist_ok=True)
+    dds.run([
+        'repo',
+        dds.repo_dir_arg,
+        'add',
+        dds.catalog_path_arg,
+        'http://localhost:4646',
+        '--update',
+    ])
+    # dds.build_deps(['neo-url@0.2.1'])

tests/dds.py

@@ -69,13 +69,17 @@ class DDS:
     def project_dir_arg(self) -> str:
         return f'--project-dir={self.source_root}'
 
+    @property
+    def catalog_path_arg(self) -> str:
+        return f'--catalog={self.catalog_path}'
+
     def build_deps(self, args: proc.CommandLine, *,
                    toolchain: str = None) -> subprocess.CompletedProcess:
         return self.run([
             'build-deps',
             f'--toolchain={toolchain or self.default_builtin_toolchain}',
-            f'--catalog={self.catalog_path}',
-            f'--repo-dir={self.repo_dir}',
+            self.catalog_path_arg,
+            self.repo_dir_arg,
             f'--out={self.deps_build_dir}',
             f'--lmi-path={self.lmi_path}',
             args,