HTTP session utility

Pull neo-io and neo-http
fix: failbit will cause exceptions on EOF

catalog.json

@@ -1841,6 +1841,16 @@
         "description": "Compression, archiving, etc. for C++20",
         "transform": [],
         "url": "git+https://github.com/vector-of-bool/neo-compress.git#0.1.0"
+      },
+      "0.1.1": {
+        "depends": [
+          "neo-buffer^0.4.1",
+          "neo-fun^0.5.0",
+          "zlib^1.2.9"
+        ],
+        "description": "Compression, archiving, etc. for C++20",
+        "transform": [],
+        "url": "git+https://github.com/vector-of-bool/neo-compress.git#0.1.1"
       }
     },
     "neo-concepts": {
@@ -1953,6 +1963,34 @@
         "description": "Some library components that didn't quite fit anywhere else...",
         "transform": [],
         "url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.3"
+      },
+      "0.5.4": {
+        "depends": [],
+        "description": "Some library components that didn't quite fit anywhere else...",
+        "transform": [],
+        "url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.4"
+      }
+    },
+    "neo-http": {
+      "0.1.0": {
+        "depends": [
+          "neo-buffer^0.4.2",
+          "neo-fun^0.5.4"
+        ],
+        "description": "A modern HTTP library",
+        "transform": [],
+        "url": "git+https://github.com/vector-of-bool/neo-http.git#0.1.0"
+      }
+    },
+    "neo-io": {
+      "0.1.0": {
+        "depends": [
+          "neo-fun~0.5.4",
+          "neo-buffer~0.4.2"
+        ],
+        "description": "A modern IO library",
+        "transform": [],
+        "url": "git+https://github.com/vector-of-bool/neo-io.git#0.1.0"
       }
     },
     "neo-sqlite3": {

catalog.old.json

@@ -2228,6 +2228,19 @@
           "transform": [],
           "url": "https://github.com/vector-of-bool/neo-compress.git"
         }
+      },
+      "0.1.1": {
+        "depends": [
+          "neo-buffer^0.4.1",
+          "neo-fun^0.5.0",
+          "zlib^1.2.9"
+        ],
+        "description": "Compression, archiving, etc. for C++20",
+        "git": {
+          "ref": "0.1.1",
+          "transform": [],
+          "url": "https://github.com/vector-of-bool/neo-compress.git"
+        }
       }
     },
     "neo-concepts": {
@@ -2394,6 +2407,43 @@
           "transform": [],
           "url": "https://github.com/vector-of-bool/neo-fun.git"
         }
+      },
+      "0.5.4": {
+        "depends": [],
+        "description": "Some library components that didn't quite fit anywhere else...",
+        "git": {
+          "ref": "0.5.4",
+          "transform": [],
+          "url": "https://github.com/vector-of-bool/neo-fun.git"
+        }
+      }
+    },
+    "neo-http": {
+      "0.1.0": {
+        "depends": [
+          "neo-buffer^0.4.2",
+          "neo-fun^0.5.4"
+        ],
+        "description": "A modern HTTP library",
+        "git": {
+          "ref": "0.1.0",
+          "transform": [],
+          "url": "https://github.com/vector-of-bool/neo-http.git"
+        }
+      }
+    },
+    "neo-io": {
+      "0.1.0": {
+        "depends": [
+          "neo-fun~0.5.4",
+          "neo-buffer~0.4.2"
+        ],
+        "description": "A modern IO library",
+        "git": {
+          "ref": "0.1.0",
+          "transform": [],
+          "url": "https://github.com/vector-of-bool/neo-io.git"
+        }
       }
     },
     "neo-sqlite3": {

library.jsonc

@@ -13,7 +13,8 @@
         "vob/json5",
         "vob/semester",
         "hanickadot/ctre",
-        // "neo/io",
+        "neo/io",
+        "neo/http",
         "neo/url",
         // Explicit zlib link is required due to linker input order bug.
         // Can be removed after alpha.5

package.jsonc

@@ -9,15 +9,17 @@
         "range-v3@0.11.0",
         "nlohmann-json@3.7.1",
         "neo-sqlite3@0.4.1",
-        "neo-fun~0.5.3",
-        "neo-compress^0.1.0",
+        "neo-fun~0.5.4",
+        "neo-compress~0.1.1",
         "neo-url~0.1.2",
         "semver@0.2.2",
         "pubgrub@0.2.1",
         "vob-json5@0.1.5",
         "vob-semester@0.2.2",
         "ctre@2.8.1",
-        "fmt^7.0.3"
+        "fmt^7.0.3",
+        "neo-http^0.1.0",
+        "neo-io^0.1.0",
     ],
     "test_driver": "Catch-Main"
 }

src/dds/error/errors.cpp

@@ -39,6 +39,8 @@ std::string error_url_suffix(dds::errc ec) noexcept {
         return "git-clone-failure.html";
     case errc::invalid_remote_url:
         return "invalid-remote-url.html";
+    case errc::http_download_failure:
+        return "http-failure.html";
     case errc::invalid_repo_transform:
         return "invalid-repo-transform.html";
     case errc::sdist_ident_mismatch:
@@ -176,6 +178,12 @@ Git in diagnosing this failure.
 )";
     case errc::invalid_remote_url:
         return R"(The given package/remote URL is invalid)";
+    case errc::http_download_failure:
+        return R"(
+There was a problem when trying to download data from an HTTP server. HTTP 40x
+errors indicate problems on the client-side, and HTTP 50x errors indicate that
+the server itself encountered an error.
+)";
     case errc::invalid_repo_transform:
         return R"(
 A 'transform' property in a catalog entry contains an invalid transformation.
@@ -290,6 +298,8 @@ std::string_view dds::default_error_string(dds::errc ec) noexcept {
         return "A git-clone operation failed.";
     case errc::invalid_remote_url:
         return "The given package/remote URL is not valid";
+    case errc::http_download_failure:
+        return "There was an error downloading data from an HTTP server.";
     case errc::invalid_repo_transform:
         return "A repository filesystem transformation is invalid";
     case errc::sdist_ident_mismatch:

src/dds/error/errors.hpp

@@ -25,6 +25,7 @@ enum class errc {
 
     git_clone_failure,
     invalid_remote_url,
+    http_download_failure,
     invalid_repo_transform,
     sdist_ident_mismatch,
     sdist_exists,

src/dds/http/session.cpp

@@ -0,0 +1,160 @@
+#include "./session.hpp"
+
+#include <dds/error/errors.hpp>
+#include <dds/util/fs.hpp>
+
+#include <fmt/format.h>
+#include <fmt/ostream.h>
+#include <neo/http/parse/chunked.hpp>
+#include <neo/http/request.hpp>
+#include <neo/http/response.hpp>
+#include <neo/io/openssl/init.hpp>
+#include <neo/io/stream/buffers.hpp>
+#include <neo/io/stream/file.hpp>
+#include <neo/string_io.hpp>
+
+using namespace dds;
+
+namespace {
+
+struct simple_request {
+    std::string method;
+
+    std::vector<std::pair<std::string, std::string>> headers;
+};
+
+template <typename Out, typename In>
+void download_into(Out&& out, In&& in, http_response_info resp) {
+    auto resp_te = resp.headers.find(neo::http::standard_headers::transfer_encoding);
+    if (resp_te) {
+        if (resp_te->value != "chunked") {
+            throw std::runtime_error(fmt::format(
+                "We can't yet handle HTTP responses that set Transfer-Encoding [Transfer "
+                "encoding is '{}']",
+                resp_te->value));
+        }
+        neo::http::chunked_buffers chunked_in{in};
+        buffer_copy(out, chunked_in);
+    } else {
+        auto clen = resp.headers.find(neo::http::standard_headers::content_length);
+        neo_assert(invariant, !!clen, "HTTP response has no Content-Length header??");
+        buffer_copy(out, in, std::stoull(clen->value));
+    }
+}
+
+}  // namespace
+
+http_session http_session::connect(const std::string& host, int port) {
+    auto addr = neo::address::resolve(host, std::to_string(port));
+    auto sock = neo::socket::open_connected(addr, neo::socket::type::stream);
+
+    return http_session{std::move(sock), fmt::format("{}:{}", host, port)};
+}
+
+http_session http_session::connect_ssl(const std::string& host, int port) {
+    auto addr = neo::address::resolve(host, std::to_string(port));
+    auto sock = neo::socket::open_connected(addr, neo::socket::type::stream);
+
+    static neo::ssl::openssl_app_init ssl_init;
+    static neo::ssl::context          ssl_ctx{neo::ssl::protocol::tls_any, neo::ssl::role::client};
+
+    neo::stream_io_buffers sock_in{sock};
+    ssl_engine             ssl_eng{ssl_ctx, sock_in, neo::stream_io_buffers{sock}};
+    ssl_eng.connect();
+
+    return http_session(std::move(sock), fmt::format("{}:{}", host, port), std::move(ssl_eng));
+}
+
+void http_session::send_head(http_request_params params) {
+    neo_assert_always(invariant,
+                      _state == _state_t::ready,
+                      "Invalid state for HTTP session to send a request head",
+                      _state,
+                      params.method,
+                      params.path,
+                      params.query);
+    neo::http::request_line start_line{
+        .method_view = params.method,
+        .target = neo::http::origin_form_target{
+            .path_view = params.path,
+            .query_view = params.query,
+            .has_query = !params.query.empty(),
+            .parse_tail = neo::const_buffer(),
+        },
+        .http_version = neo::http::version::v1_1,
+        .parse_tail = neo::const_buffer(),
+    };
+
+    auto cl_str = std::to_string(params.content_length);
+
+    std::pair<std::string_view, std::string_view> headers[] = {
+        {"Host", host_string()},
+        {"Accept", "*/*"},
+        {"Content-Length", cl_str},
+    };
+
+    _do_io(
+        [&](auto&& io) { neo::http::write_request(io, start_line, headers, neo::const_buffer()); });
+    _state = _state_t::sent_request;
+}
+
+http_response_info http_session::recv_head() {
+    neo_assert_always(invariant,
+                      _state == _state_t::sent_request,
+                      "Invalid state to receive response head",
+                      _state);
+    auto r
+        = _do_io([&](auto&& io) { return neo::http::read_response_head<http_response_info>(io); });
+    _state = _state_t::recvd_head;
+    return r;
+}
+
+std::string http_session::request(http_request_params params) {
+    send_head(params);
+
+    auto resp_head = recv_head();
+
+    neo::string_dynbuf_io resp_body;
+    _do_io([&](auto&& io) { download_into(resp_body, io, resp_head); });
+    auto body_size = resp_body.available();
+    auto str       = std::move(resp_body.string());
+    str.resize(body_size);
+    return str;
+}
+
+void http_session::recv_body_to_file(http_response_info const&    resp_head,
+                                     const std::filesystem::path& dest) {
+    neo_assert_always(invariant,
+                      _state == _state_t::recvd_head,
+                      "Invalid state to receive request body",
+                      _state,
+                      dest);
+    auto ofile = neo::file_stream::open(dest, neo::open_mode::write | neo::open_mode::create);
+    neo::stream_io_buffers file_out{ofile};
+    _do_io([&](auto&& io) { download_into(file_out, io, resp_head); });
+    _state = _state_t::ready;
+}
+
+void http_session::download_file(http_request_params params, const std::filesystem::path& dest) {
+    send_head(params);
+    auto resp_head = recv_head();
+    if (resp_head.is_error()) {
+        throw_external_error<
+            errc::http_download_failure>("Failed to download file from {}{} to {}: HTTP {} {}",
+                                         host_string(),
+                                         params.path,
+                                         dest,
+                                         resp_head.status,
+                                         resp_head.status_message);
+    }
+
+    if (resp_head.is_redirect()) {
+        throw_external_error<errc::http_download_failure>(
+            "dds does not yet support HTTP redirects when downloading data. An HTTP redirect "
+            "was encountered when accessing {}{}: It wants to redirect to {}",
+            host_string(),
+            params.path,
+            resp_head.headers["Location"].value);
+    }
+    recv_body_to_file(resp_head, dest);
+}

src/dds/http/session.hpp

@@ -0,0 +1,121 @@
+#pragma once
+
+#include <neo/http/headers.hpp>
+#include <neo/http/version.hpp>
+#include <neo/io/openssl/engine.hpp>
+#include <neo/io/stream/buffers.hpp>
+#include <neo/io/stream/socket.hpp>
+#include <neo/string_io.hpp>
+
+#include <filesystem>
+#include <string>
+
+namespace dds {
+
+struct http_request_params {
+    std::string_view method;
+    std::string_view path;
+    std::string_view query          = "";
+    std::size_t      content_length = 0;
+};
+
+struct http_response_info {
+    int                status;
+    std::string        status_message;
+    neo::http::version version;
+    neo::http::headers headers;
+
+    std::size_t head_byte_size = 0;
+
+    void throw_for_status() const;
+
+    bool is_client_error() const noexcept { return status >= 400 && status < 500; }
+    bool is_server_error() const noexcept { return status >= 500 && status < 600; }
+    bool is_error() const noexcept { return is_client_error() || is_server_error(); }
+    bool is_redirect() const noexcept { return status >= 300 && status < 400; }
+};
+
+enum class http_kind {
+    plain,
+    ssl,
+};
+
+class http_session {
+    neo::socket _conn;
+
+    std::string _host_string;
+
+    using sock_buffers = neo::stream_io_buffers<neo::socket&>;
+    sock_buffers _sock_in{_conn};
+
+    using ssl_engine  = neo::ssl::engine<sock_buffers&, sock_buffers>;
+    using ssl_buffers = neo::stream_io_buffers<ssl_engine>;
+    std::optional<ssl_buffers> _ssl_in;
+
+    enum _state_t { ready, sent_request, recvd_head } _state = ready;
+
+    template <typename F>
+    decltype(auto) _do_io(F&& fn) {
+        if (_ssl_in) {
+            return fn(*_ssl_in);
+        } else {
+            return fn(_sock_in);
+        }
+    }
+
+    void _rebind_refs() {
+        if (_ssl_in) {
+            _ssl_in->stream().rebind_input(_sock_in);
+            _ssl_in->stream().output().rebind_stream(_conn);
+        }
+    }
+
+public:
+    explicit http_session(neo::socket s, std::string host_header)
+        : _conn(std::move(s))
+        , _host_string(std::move(host_header)) {}
+
+    explicit http_session(neo::socket s, std::string host_header, ssl_engine&& eng)
+        : _conn(std::move(s))
+        , _host_string(std::move(host_header))
+        , _sock_in(_conn, std::move(eng.input().io_buffers()))
+        , _ssl_in(std::move(eng)) {
+        _rebind_refs();
+    }
+
+    http_session(http_session&& other) noexcept
+        : _conn(std::move(other._conn))
+        , _host_string(std::move(other._host_string))
+        , _sock_in(_conn, std::move(other._sock_in.io_buffers()))
+        , _ssl_in(std::move(other._ssl_in)) {
+        _rebind_refs();
+    }
+
+    http_session& operator=(http_session&& other) noexcept {
+        _conn                 = std::move(other._conn);
+        _host_string          = std::move(other._host_string);
+        _sock_in.io_buffers() = std::move(other._sock_in.io_buffers());
+        _ssl_in               = std::move(other._ssl_in);
+        _rebind_refs();
+        return *this;
+    }
+
+    void               send_head(http_request_params);
+    http_response_info recv_head();
+    void recv_body_to_file(http_response_info const& res_head, const std::filesystem::path& dest);
+
+    std::string_view host_string() const noexcept { return _host_string; }
+
+    static http_session connect(const std::string& host, int port);
+    static http_session connect_ssl(const std::string& host, int port);
+
+    std::string request(http_request_params);
+
+    std::string request_get(std::string_view path) {
+        return request({.method = "GET", .path = path});
+    }
+
+    void download_file(http_request_params, const std::filesystem::path& dest);
+};
+
+}  // namespace dds

src/dds/http/session.test.cpp

@@ -0,0 +1,8 @@
+#include <dds/http/session.hpp>
+
+#include <catch2/catch.hpp>
+
+TEST_CASE("Create an HTTP session") {
+    auto sess = dds::http_session::connect("google.com", 80);
+    auto resp = sess.request_get("/");
+}

src/dds/util/fs.cpp

@@ -8,7 +8,7 @@ using namespace dds;
 
 std::fstream dds::open(const fs::path& filepath, std::ios::openmode mode, std::error_code& ec) {
     std::fstream ret;
-    auto         mask = ret.exceptions() | std::ios::failbit;
+    auto         mask = ret.exceptions() | std::ios::badbit;
     ret.exceptions(mask);
 
     try {

tools/gen-catalog-json.py

@@ -291,14 +291,16 @@ def many_versions(name: str,
 PACKAGES = [
     github_package('neo-buffer', 'vector-of-bool/neo-buffer',
                    ['0.2.1', '0.3.0', '0.4.0', '0.4.1', '0.4.2']),
-    github_package('neo-compress', 'vector-of-bool/neo-compress', ['0.1.0']),
+    github_package('neo-compress', 'vector-of-bool/neo-compress', ['0.1.0', '0.1.1']),
     github_package('neo-url', 'vector-of-bool/neo-url', ['0.1.0', '0.1.1', '0.1.2']),
     github_package('neo-sqlite3', 'vector-of-bool/neo-sqlite3',
                    ['0.2.3', '0.3.0', '0.4.0', '0.4.1']),
     github_package('neo-fun', 'vector-of-bool/neo-fun', [
         '0.1.1', '0.2.0', '0.2.1', '0.3.0', '0.3.1', '0.3.2', '0.4.0', '0.4.1',
-        '0.4.2', '0.5.0', '0.5.1', '0.5.2', '0.5.3',
+        '0.4.2', '0.5.0', '0.5.1', '0.5.2', '0.5.3', '0.5.4',
     ]),
+    github_package('neo-io', 'vector-of-bool/neo-io', ['0.1.0']),
+    github_package('neo-http', 'vector-of-bool/neo-http', ['0.1.0']),
     github_package('neo-concepts', 'vector-of-bool/neo-concepts', (
         '0.2.2',
         '0.3.0',