Packages can now be imported over HTTP

src/dds.main.cpp

@@ -459,8 +459,10 @@ struct cli_repo {
                           "Import a source distribution archive file into the repository"};
         common_flags  _common{cmd};
 
-        args::PositionalList<dds::fs::path>
-            sdist_paths{cmd, "sdist-path", "Path to one or more source distribution archive"};
+        args::PositionalList<std::string>
+            sdist_paths{cmd,
+                        "sdist-path-or-url",
+                        "Path/URL to one or more source distribution archives"};
 
         args::Flag force{cmd,
                          "replace-if-exists",
@@ -476,8 +478,11 @@ struct cli_repo {
             auto import_sdists = [&](dds::repository repo) {
                 auto if_exists_action
                     = force.Get() ? dds::if_exists::replace : dds::if_exists::throw_exc;
-                for (auto& tgz_path : sdist_paths.Get()) {
-                    auto tmp_sd = dds::expand_sdist_targz(tgz_path);
+                for (std::string_view tgz_where : sdist_paths.Get()) {
+                    auto tmp_sd
+                        = (tgz_where.starts_with("http://") || tgz_where.starts_with("https://"))
+                        ? dds::download_expand_sdist_targz(tgz_where)
+                        : dds::expand_sdist_targz(tgz_where);
                     repo.add_sdist(tmp_sd.sdist, if_exists_action);
                 }
                 if (import_stdin) {

src/dds/catalog/remote/http.cpp

@@ -0,0 +1,167 @@
+#include "./http.hpp"
+
+#include <dds/error/errors.hpp>
+#include <dds/http/session.hpp>
+#include <dds/temp.hpp>
+#include <dds/util/log.hpp>
+
+#include <neo/tar/util.hpp>
+#include <neo/url.hpp>
+#include <neo/url/query.hpp>
+
+using namespace dds;
+
+namespace {
+
+void http_download_with_redir(neo::url url, path_ref dest) {
+    for (auto redir_count = 0;; ++redir_count) {
+        auto sess = url.scheme == "https"
+            ? http_session::connect_ssl(*url.host, url.port_or_default_port_or(443))
+            : http_session::connect(*url.host, url.port_or_default_port_or(80));
+
+        sess.send_head({.method = "GET", .path = url.path_string()});
+
+        auto res_head = sess.recv_head();
+        if (res_head.is_error()) {
+            dds_log(error,
+                    "Received an HTTP {} {} for [{}]",
+                    res_head.status,
+                    res_head.status_message,
+                    url.to_string());
+            throw_external_error<errc::http_download_failure>(
+                "HTTP error while downloading resource [{}]. Got: HTTP {} '{}'",
+                url.to_string(),
+                res_head.status,
+                res_head.status_message);
+        }
+
+        if (res_head.is_redirect()) {
+            dds_log(trace,
+                    "Received HTTP redirect for [{}]: {} {}",
+                    url.to_string(),
+                    res_head.status,
+                    res_head.status_message);
+            if (redir_count == 100) {
+                throw_external_error<errc::http_download_failure>("Too many redirects on URL");
+            }
+            auto loc = res_head.headers.find("Location");
+            if (!loc) {
+                throw_external_error<errc::http_download_failure>(
+                    "HTTP endpoint told us to redirect without sending a 'Location' header "
+                    "(Received "
+                    "HTTP {} '{}')",
+                    res_head.status,
+                    res_head.status_message);
+            }
+            dds_log(debug,
+                    "Redirect [{}]: {} {} to [{}]",
+                    url.to_string(),
+                    res_head.status,
+                    res_head.status_message,
+                    loc->value);
+            auto new_url = neo::url::try_parse(loc->value);
+            auto err     = std::get_if<neo::url_validation_error>(&new_url);
+            if (err) {
+                throw_external_error<errc::http_download_failure>(
+                    "Server returned an invalid URL for HTTP redirection [{}]", loc->value);
+            }
+            url = std::move(std::get<neo::url>(new_url));
+            continue;
+        }
+
+        // Not a redirect nor an error: Download the body
+        dds_log(trace,
+                "HTTP {} {} [{}]: Saving to [{}]",
+                res_head.status,
+                res_head.status_message,
+                url.to_string(),
+                dest.string());
+        sess.recv_body_to_file(res_head, dest);
+        break;
+    }
+}
+
+}  // namespace
+
+void http_remote_listing::pull_to(path_ref dest) const {
+    neo::url url;
+    try {
+        url = neo::url::parse(this->url);
+    } catch (const neo::url_validation_error& e) {
+        throw_user_error<errc::invalid_remote_url>("Failed to parse the string '{}' as a URL: {}",
+                                                   this->url,
+                                                   e.what());
+    }
+    dds_log(trace, "Downloading HTTP remote from [{}]", url.to_string());
+
+    if (url.scheme != "http" && url.scheme != "https") {
+        dds_log(error, "Unsupported URL scheme '{}' (in [{}])", url.scheme, url.to_string());
+        throw_user_error<errc::invalid_remote_url>(
+            "The given URL download is not supported. (Only 'http' URLs are supported, "
+            "got '{}')",
+            this->url);
+    }
+
+    neo_assert(invariant,
+               !!url.host,
+               "The given URL did not have a host part. This shouldn't be possible... Please file "
+               "a bug report.",
+               this->url);
+
+    auto tdir     = dds::temporary_dir::create();
+    auto url_path = fs::path(url.path_string());
+    auto fname    = url_path.filename();
+    if (fname.empty()) {
+        fname = "dds-download.tmp";
+    }
+    auto dl_path = tdir.path() / fname;
+    fs::create_directory(dl_path.parent_path());
+
+    http_download_with_redir(url, dl_path);
+
+    neo_assert(invariant,
+               fs::is_regular_file(dl_path),
+               "HTTP client did not properly download the file??",
+               this->url,
+               dl_path);
+
+    fs::create_directories(dest);
+    dds_log(debug, "Expanding downloaded source distribution into {}", dest.string());
+    std::ifstream infile{dl_path, std::ios::binary};
+    neo::expand_directory_targz(
+        neo::expand_options{
+            .destination_directory = dest,
+            .input_name            = dl_path.string(),
+            .strip_components      = this->strip_components,
+        },
+        infile);
+}
+
+http_remote_listing http_remote_listing::from_url(std::string_view sv) {
+    auto url = neo::url::parse(sv);
+    dds_log(trace, "Create HTTP remote listing from URL [{}]", sv);
+
+    auto q = url.query;
+
+    unsigned                 strip_components = 0;
+    std::optional<lm::usage> auto_lib;
+
+    if (q) {
+        neo::basic_query_string_view qsv{*q};
+        for (auto qstr : qsv) {
+            if (qstr.key_raw() == "dds_lm") {
+                auto_lib = lm::split_usage_string(qstr.value_decoded());
+            } else if (qstr.key_raw() == "dds_strpcmp") {
+                strip_components = static_cast<unsigned>(std::stoul(qstr.value_decoded()));
+            } else {
+                dds_log(warn, "Unknown query string parameter in package url: '{}'", qstr.string());
+            }
+        }
+    }
+
+    return http_remote_listing{
+        .url              = url.to_string(),
+        .strip_components = strip_components,
+        .auto_lib         = auto_lib,
+    };
+}

src/dds/catalog/remote/http.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <dds/package/id.hpp>
+#include <dds/util/fs.hpp>
+
+#include <libman/package.hpp>
+
+#include <string>
+#include <string_view>
+
+namespace dds {
+
+struct http_remote_listing {
+    std::string              url;
+    unsigned                 strip_components = 0;
+    std::optional<lm::usage> auto_lib{};
+
+    void pull_to(path_ref path) const;
+
+    static http_remote_listing from_url(std::string_view sv);
+};
+
+}  // namespace dds

src/dds/catalog/remote/http.test.cpp

@@ -0,0 +1,12 @@
+#include <dds/catalog/remote/http.hpp>
+
+#include <dds/error/errors.hpp>
+#include <dds/source/dist.hpp>
+#include <dds/util/log.hpp>
+
+#include <catch2/catch.hpp>
+
+TEST_CASE("Convert URL to an HTTP remote listing") {
+    auto remote = dds::http_remote_listing::from_url(
+        "http://localhost:8000/neo-buffer-0.4.2.tar.gz?dds_strpcmp=1");
+}

src/dds/http/session.cpp

@@ -2,6 +2,8 @@
 
 #include <dds/error/errors.hpp>
 #include <dds/util/fs.hpp>
+#include <dds/util/log.hpp>
+#include <dds/util/result.hpp>
 
 #include <fmt/format.h>
 #include <fmt/ostream.h>
@@ -45,6 +47,8 @@ void download_into(Out&& out, In&& in, http_response_info resp) {
 }  // namespace
 
 http_session http_session::connect(const std::string& host, int port) {
+    DDS_E_SCOPE(e_http_connect{host, port});
+
     auto addr = neo::address::resolve(host, std::to_string(port));
     auto sock = neo::socket::open_connected(addr, neo::socket::type::stream);
 
@@ -52,6 +56,8 @@ http_session http_session::connect(const std::string& host, int port) {
 }
 
 http_session http_session::connect_ssl(const std::string& host, int port) {
+    DDS_E_SCOPE(e_http_connect{host, port});
+
     auto addr = neo::address::resolve(host, std::to_string(port));
     auto sock = neo::socket::open_connected(addr, neo::socket::type::stream);
 
@@ -85,6 +91,8 @@ void http_session::send_head(http_request_params params) {
         .parse_tail = neo::const_buffer(),
     };
 
+    dds_log(trace, "Send: HTTP {} to {}{}", params.method, host_string(), params.path);
+
     auto cl_str = std::to_string(params.content_length);
 
     std::pair<std::string_view, std::string_view> headers[] = {
@@ -105,6 +113,7 @@ http_response_info http_session::recv_head() {
                       _state);
     auto r
         = _do_io([&](auto&& io) { return neo::http::read_response_head<http_response_info>(io); });
+    dds_log(trace, "Recv: HTTP {} {}", r.status, r.status_message);
     _state = _state_t::recvd_head;
     return r;
 }

src/dds/http/session.hpp

@@ -12,6 +12,15 @@
 
 namespace dds {
 
+struct e_http_url {
+    std::string value;
+};
+
+struct e_http_connect {
+    std::string host;
+    int         port;
+};
+
 struct http_request_params {
     std::string_view method;
     std::string_view path;

src/dds/solve/solve.cpp

@@ -92,7 +92,7 @@ struct solver_provider {
             dds_log(debug, "No candidate for requirement {}", req.dep.to_string());
             return std::nullopt;
         }
-        dds_log(debug, "Select candidate {}@{}", cand->to_string());
+        dds_log(debug, "Select candidate {}", cand->to_string());
         return req_type{dependency{cand->name, {cand->version, cand->version.next_after()}}};
     }
 

src/dds/source/dist.cpp

@@ -1,5 +1,6 @@
 #include "./dist.hpp"
 
+#include <dds/catalog/remote/http.hpp>
 #include <dds/error/errors.hpp>
 #include <dds/library/root.hpp>
 #include <dds/temp.hpp>
@@ -122,26 +123,35 @@ sdist dds::create_sdist_in_dir(path_ref out, const sdist_params& params) {
 sdist sdist::from_directory(path_ref where) {
     auto pkg_man = package_manifest::load_from_directory(where);
     // Code paths should only call here if they *know* that the sdist is valid
-    neo_assert(invariant,
-               pkg_man.has_value(),
-               "All dirs in the repo should be proper source distributions. If you see this, it "
-               "means one of the directories in the repository is not a valid sdist.",
-               where.string());
+    if (!pkg_man.has_value()) {
+        throw_user_error<errc::invalid_pkg_filesystem>(
+            "The given directory [{}] does not contain a package manifest file. All source "
+            "distribution directories are required to contain a package manifest.",
+            where.string());
+    }
     return sdist{pkg_man.value(), where};
 }
 
 temporary_sdist dds::expand_sdist_targz(path_ref targz_path) {
+    auto infile = open(targz_path, std::ios::binary | std::ios::in);
+    return expand_sdist_from_istream(infile, targz_path.string());
+}
+
+temporary_sdist dds::expand_sdist_from_istream(std::istream& is, std::string_view input_name) {
     auto tempdir = temporary_dir::create();
-    dds_log(debug, "Expanding source ditsribution content into {}", tempdir.path().string());
+    dds_log(debug,
+            "Expanding source distribution content from [{}] into [{}]",
+            input_name,
+            tempdir.path().string());
     fs::create_directories(tempdir.path());
-    neo::expand_directory_targz(tempdir.path(), targz_path);
+    neo::expand_directory_targz({.destination_directory = tempdir.path(), .input_name = input_name},
+                                is);
     return {tempdir, sdist::from_directory(tempdir.path())};
 }
 
-temporary_sdist dds::expand_sdist_from_istream(std::istream& is, std::string_view input_name) {
+temporary_sdist dds::download_expand_sdist_targz(std::string_view url_str) {
+    auto remote  = http_remote_listing::from_url(url_str);
     auto tempdir = temporary_dir::create();
-    dds_log(debug, "Expanding source ditsribution content into {}", tempdir.path().string());
-    fs::create_directories(tempdir.path());
-    neo::expand_directory_targz(tempdir.path(), is, input_name);
+    remote.pull_to(tempdir.path());
     return {tempdir, sdist::from_directory(tempdir.path())};
 }

src/dds/source/dist.hpp

@@ -51,5 +51,6 @@ void  create_sdist_targz(path_ref, const sdist_params&);
 
 temporary_sdist expand_sdist_targz(path_ref targz);
 temporary_sdist expand_sdist_from_istream(std::istream&, std::string_view input_name);
+temporary_sdist download_expand_sdist_targz(std::string_view);
 
 }  // namespace dds

tests/catalog/import_test.py

@@ -1,9 +1,27 @@
 import json
+from pathlib import Path
+from functools import partial
+from concurrent.futures import ThreadPoolExecutor
+from http.server import SimpleHTTPRequestHandler, HTTPServer
+import time
+
+import pytest
 
 from tests import dds, DDS
 from tests.fileutil import ensure_dir
 
 
+class DirectoryServingHTTPRequestHandler(SimpleHTTPRequestHandler):
+    def __init__(self, *args, **kwargs) -> None:
+        self.dir = kwargs.pop('dir')
+        super().__init__(*args, **kwargs)
+
+    def translate_path(self, path) -> str:
+        abspath = Path(super().translate_path(path))
+        relpath = abspath.relative_to(Path.cwd())
+        return self.dir / relpath
+
+
 def test_import_json(dds: DDS):
     dds.scope.enter_context(ensure_dir(dds.build_dir))
     dds.catalog_create()
@@ -27,3 +45,32 @@ def test_import_json(dds: DDS):
         dds.set_contents(json_fpath,
                          json.dumps(import_data).encode()))
     dds.catalog_import(json_fpath)
+
+
+@pytest.yield_fixture
+def http_import_server():
+    handler = partial(
+        DirectoryServingHTTPRequestHandler,
+        dir=Path.cwd() / 'data/http-test-1')
+    addr = ('0.0.0.0', 8000)
+    pool = ThreadPoolExecutor()
+    with HTTPServer(addr, handler) as httpd:
+        pool.submit(lambda: httpd.serve_forever(poll_interval=0.1))
+        try:
+            yield
+        finally:
+            httpd.shutdown()
+
+
+def test_import_http(dds: DDS, http_import_server):
+    dds.repo_dir.mkdir(parents=True, exist_ok=True)
+    dds.run(
+        [
+            'repo',
+            dds.repo_dir_arg,
+            'import',
+            'https://github.com/vector-of-bool/neo-buffer/archive/0.4.2.tar.gz?dds_strpcmp=1',
+        ],
+        cwd=dds.repo_dir,
+    )
+    assert dds.repo_dir.joinpath('neo-buffer@0.4.2').is_dir()

tests/dds.py

@@ -49,7 +49,7 @@ class DDS:
 
     def run_unchecked(self, cmd: proc.CommandLine, *,
                       cwd: Path = None) -> subprocess.CompletedProcess:
-        full_cmd = itertools.chain([self.dds_exe], cmd)
+        full_cmd = itertools.chain([self.dds_exe, '-ltrace'], cmd)
         return proc.run(full_cmd, cwd=cwd or self.source_root)
 
     def run(self, cmd: proc.CommandLine, *, cwd: Path = None,