OpenSSL support

- CI step cleanups, and get openssl for windows
- WORKAROUND: Bug in libgcc in certain conditions with partial pthread
  linked statically.

.gitignore

@@ -5,4 +5,6 @@ __pycache__/
 .mypy_cache/
 _prebuilt/
 .pytest_cache/
-.vagrant/
+.vagrant/
+
+external/OpenSSL

Makefile

@@ -41,7 +41,7 @@ docs-sync-server:
 macos-ci:
 	python3 -u tools/ci.py \
 		-B download \
-		-T tools/gcc-9-rel.jsonc
+		-T tools/gcc-9-rel-macos.jsonc
 	mv _build/dds _build/dds-macos-x64
 
 linux-ci:

azure-pipelines.yml

@@ -17,12 +17,15 @@ stages:
         pool:
           vmImage: windows-2019
         steps:
+          - pwsh: tools\get-win-openssl.ps1
+            displayName: Get OpenSSL for Windows
+          - script: python -m pip install pytest pytest-xdist
+            displayName: Install Python deps
           - script: |
               echo Loading VS environment
               call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\vsdevcmd" -arch=x64 || exit 1
               echo Executing Build and Tests
               reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1
-              python -m pip install pytest pytest-xdist || exit 1
               python -u tools/ci.py -B download -T tools\msvc.jsonc || exit 1
               move _build\dds.exe _build\dds-win-x64.exe || exit 1
             displayName: Build and Test
@@ -35,12 +38,14 @@ stages:
         pool:
           vmImage: ubuntu-18.04
         steps:
-          - script: |
-              set -eu
-              sudo apt update -y
-              sudo apt install -y python3-minimal python3-setuptools g++-9 ccache
-              python3 -m pip install pytest pytest-xdist
-            displayName: Prepare System
+          - script: sudo apt-get -y update
+            displayName: APT Update
+          - script: sudo apt-get -y install g++-9 ccache
+            displayName: Get GCC 9
+          - script: sudo apt-get -y install python3-minimal python3-setuptools
+            displayName: Get Python 3
+          - script: python3 -m pip install pytest pytest-xdist
+            displayName: Get Python Dependencies
           - script: make linux-ci
             displayName: Build and Test
           - publish: _build/dds-linux-x64
@@ -50,14 +55,15 @@ stages:
       - job: macos_gcc9
         displayName: macOS - GCC 9
         pool:
-          vmImage: macOS-10.14
+          vmImage: macOS-10.15
         steps:
           - script: brew install gcc@9 ccache
-            displayName: Prepare System
-          - script: |
-              set -eu
-              python3 -m pip install pytest pytest-xdist
-              make macos-ci
+            displayName: Get GCC 9
+          - script: brew install openssl@1.1
+            displayName: Install OpenSSL
+          - script: python3 -m pip install pytest pytest-xdist
+            displayName: Get Python Dependencies
+          - script: make macos-ci
             displayName: Build and Test
           - publish: _build/dds-macos-x64
             displayName: Publish

catalog.json

@@ -1969,6 +1969,12 @@
         "description": "Some library components that didn't quite fit anywhere else...",
         "transform": [],
         "url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.4"
+      },
+      "0.5.5": {
+        "depends": [],
+        "description": "Some library components that didn't quite fit anywhere else...",
+        "transform": [],
+        "url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.5"
       }
     },
     "neo-http": {
@@ -2047,6 +2053,22 @@
         "description": "URL parsing and handling library.",
         "transform": [],
         "url": "git+https://github.com/vector-of-bool/neo-url.git#0.1.2"
+      },
+      "0.2.0": {
+        "depends": [
+          "neo-fun^0.5.5"
+        ],
+        "description": "URL parsing and handling library.",
+        "transform": [],
+        "url": "git+https://github.com/vector-of-bool/neo-url.git#0.2.0"
+      },
+      "0.2.1": {
+        "depends": [
+          "neo-fun^0.5.5"
+        ],
+        "description": "URL parsing and handling library.",
+        "transform": [],
+        "url": "git+https://github.com/vector-of-bool/neo-url.git#0.2.1"
       }
     },
     "nlohmann-json": {

catalog.old.json

@@ -2416,6 +2416,15 @@
           "transform": [],
           "url": "https://github.com/vector-of-bool/neo-fun.git"
         }
+      },
+      "0.5.5": {
+        "depends": [],
+        "description": "Some library components that didn't quite fit anywhere else...",
+        "git": {
+          "ref": "0.5.5",
+          "transform": [],
+          "url": "https://github.com/vector-of-bool/neo-fun.git"
+        }
       }
     },
     "neo-http": {
@@ -2521,6 +2530,28 @@
           "transform": [],
           "url": "https://github.com/vector-of-bool/neo-url.git"
         }
+      },
+      "0.2.0": {
+        "depends": [
+          "neo-fun^0.5.5"
+        ],
+        "description": "URL parsing and handling library.",
+        "git": {
+          "ref": "0.2.0",
+          "transform": [],
+          "url": "https://github.com/vector-of-bool/neo-url.git"
+        }
+      },
+      "0.2.1": {
+        "depends": [
+          "neo-fun^0.5.5"
+        ],
+        "description": "URL parsing and handling library.",
+        "git": {
+          "ref": "0.2.1",
+          "transform": [],
+          "url": "https://github.com/vector-of-bool/neo-url.git"
+        }
       }
     },
     "nlohmann-json": {

package.jsonc

@@ -11,7 +11,7 @@
         "neo-sqlite3@0.4.1",
         "neo-fun~0.5.4",
         "neo-compress~0.1.1",
-        "neo-url~0.1.2",
+        "neo-url~0.2.1",
         "semver@0.2.2",
         "pubgrub@0.2.1",
         "vob-json5@0.1.5",

src/dds.main.cpp

@@ -262,26 +262,24 @@ struct cli_catalog {
         catalog_path_flag cat_path{cmd};
 
         args::Positional<std::string> pkg_id{cmd,
-                                             "id",
+                                             "<id>",
                                              "The name@version ID of the package to add",
                                              args::Options::Required};
 
-        string_flag auto_lib{cmd,
-                             "auto-lib",
-                             "Set the auto-library information for this package",
-                             {"auto-lib"}};
+        args::Positional<std::string> uri{cmd,
+                                          "<uri>",
+                                          "The URI of the package",
+                                          args::Options::Required};
+        // string_flag auto_lib{cmd,
+        //                      "auto-lib",
+        //                      "Set the auto-library information for this package",
+        //                      {"auto-lib"}};
 
         args::ValueFlagList<std::string> deps{cmd,
                                               "depends",
                                               "The dependencies of this package",
                                               {"depends", 'd'}};
 
-        string_flag git_url{cmd, "git-url", "The Git url for the package", {"git-url"}};
-        string_flag git_ref{cmd,
-                            "git-ref",
-                            "The Git ref to from which the source distribution should be created",
-                            {"git-ref"}};
-
         string_flag description{cmd, "description", "A description of the package", {"desc"}};
 
         int run() {
@@ -294,21 +292,14 @@ struct cli_catalog {
                 // deps.push_back({dep_id.name, dep_id.version});
             }
 
-            dds::package_info info{ident, std::move(deps), description.Get(), {}};
+            auto remote = dds::parse_remote_url(uri.Get());
 
-            if (git_url) {
-                if (!git_ref) {
-                    dds::throw_user_error<dds::errc::git_url_ref_mutual_req>();
-                }
-                auto git = dds::git_remote_listing{git_url.Get(), git_ref.Get(), std::nullopt, {}};
-                if (auto_lib) {
-                    git.auto_lib = lm::split_usage_string(auto_lib.Get());
-                }
-                info.remote = std::move(git);
-            } else if (git_ref) {
-                dds::throw_user_error<dds::errc::git_url_ref_mutual_req>();
-            }
+            neo_assertion_breadcrumbs("Running 'catalog add'",
+                                      uri.Get(),
+                                      description.Get(),
+                                      pkg_id.Get());
 
+            dds::package_info info{ident, std::move(deps), description.Get(), remote};
             cat_path.open().store(info);
             return 0;
         }
@@ -352,6 +343,13 @@ struct cli_catalog {
             }
         }
 
+        void print_remote_info(const dds::http_remote_listing& http) {
+            fmt::print("HTTP/S URL: {}", http.url);
+            if (http.auto_lib) {
+                fmt::print("Auto-lib: {}/{}", http.auto_lib->name, http.auto_lib->namespace_);
+            }
+        }
+
         void print_remote_info(std::monostate) {
             std::cout << "THIS ENTRY IS MISSING REMOTE INFORMATION!\n";
         }

src/dds/catalog/catalog.cpp

@@ -158,9 +158,27 @@ void store_with_remote(const neo::sqlite3::statement_cache&,
 
 void store_with_remote(neo::sqlite3::statement_cache& stmts,
                        const package_info&            pkg,
-                       const git_remote_listing&      git) {
-    auto lm_usage = git.auto_lib.value_or(lm::usage{});
+                       const http_remote_listing&     http) {
+    nsql::exec(  //
+        stmts(R"(
+            INSERT OR REPLACE INTO dds_cat_pkgs (
+                name,
+                version,
+                remote_url,
+                description,
+                repo_transform
+            ) VALUES (?1, ?2, ?3, ?4, ?5)
+        )"_sql),
+        pkg.ident.name,
+        pkg.ident.version.to_string(),
+        http.url,
+        pkg.description,
+        transforms_to_json(http.transforms));
+}
 
+void store_with_remote(neo::sqlite3::statement_cache& stmts,
+                       const package_info&            pkg,
+                       const git_remote_listing&      git) {
     std::string url = git.url;
     if (url.starts_with("https://") || url.starts_with("http://")) {
         url = "git+" + url;

src/dds/catalog/catalog.test.cpp

@@ -30,7 +30,7 @@ TEST_CASE_METHOD(catalog_test_case, "Store a simple package") {
         dds::package_id("foo", semver::version::parse("1.2.3")),
         {},
         "example",
-        dds::git_remote_listing{"git+http://example.com", "master", std::nullopt, {}},
+        dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "master"},
     });
 
     auto pkgs = db.by_name("foo");
@@ -49,7 +49,7 @@ TEST_CASE_METHOD(catalog_test_case, "Store a simple package") {
         dds::package_id("foo", semver::version::parse("1.2.3")),
         {},
         "example",
-        dds::git_remote_listing{"git+http://example.com", "develop", std::nullopt, {}},
+        dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "develop"},
     }));
     // The previous pkg_id is still a valid lookup key
     info = db.get(pkgs[0]);
@@ -65,7 +65,7 @@ TEST_CASE_METHOD(catalog_test_case, "Package requirements") {
             {"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}},
         },
         "example",
-        dds::git_remote_listing{"git+http://example.com", "master", std::nullopt, {}},
+        dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "master"},
     });
     auto pkgs = db.by_name("foo");
     REQUIRE(pkgs.size() == 1);

src/dds/catalog/get.cpp

@@ -24,12 +24,15 @@ temporary_sdist do_pull_sdist(const package_info& listing, std::monostate) {
         listing.ident.to_string());
 }
 
-temporary_sdist do_pull_sdist(const package_info& listing, const git_remote_listing& git) {
+template <remote_listing R>
+temporary_sdist do_pull_sdist(const package_info& listing, const R& remote) {
     auto tmpdir = dds::temporary_dir::create();
 
-    git.pull_to(listing.ident, tmpdir.path());
+    remote.pull_source(tmpdir.path());
+    remote.apply_transforms(tmpdir.path());
+    remote.generate_auto_lib_files(listing.ident, tmpdir.path());
 
-    dds_log(info, "Create sdist from clone ...");
+    dds_log(info, "Create sdist ...");
     sdist_params params;
     params.project_dir = tmpdir.path();
     auto sd_tmp_dir    = dds::temporary_dir::create();

src/dds/catalog/import.cpp

@@ -69,7 +69,7 @@ auto parse_remote = [](const std::string& str) {
     try {
         return parse_remote_url(str);
     } catch (const neo::url_validation_error& e) {
-        import_error("{}: Invalid URL: {}", walk.path(), str);
+        import_error("{}: Invalid URL [{}]: {}", walk.path(), str, e.what());
     } catch (const user_error<errc::invalid_remote_url>& e) {
         import_error("{}: Invalid URL: {}", walk.path(), e.what());
     }

src/dds/catalog/package_info.cpp

@@ -3,14 +3,17 @@
 #include <dds/error/errors.hpp>
 
 #include <neo/url.hpp>
+#include <neo/utility.hpp>
 
 using namespace dds;
 
 dds::remote_listing_var dds::parse_remote_url(std::string_view sv) {
+    neo_assertion_breadcrumbs("Loading package remote from URI string", sv);
     auto url = neo::url::parse(sv);
-    if (url.scheme == "git+https" || url.scheme == "git+http" || url.scheme == "https+git"
-        || url.scheme == "http+git" || url.scheme == "git") {
+    if (url.scheme == neo::oper::any_of("git+https", "git+http", "http+git", "https+git", "git")) {
         return git_remote_listing::from_url(sv);
+    } else if (url.scheme == neo::oper::any_of("http", "https")) {
+        return http_remote_listing::from_url(sv);
     } else {
         throw_user_error<
             errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'",

src/dds/catalog/package_info.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "./remote/git.hpp"
+#include "./remote/http.hpp"
 
 #include <dds/deps.hpp>
 #include <dds/package/id.hpp>
@@ -14,7 +15,7 @@
 
 namespace dds {
 
-using remote_listing_var = std::variant<std::monostate, git_remote_listing>;
+using remote_listing_var = std::variant<std::monostate, git_remote_listing, http_remote_listing>;
 
 remote_listing_var parse_remote_url(std::string_view url);
 

src/dds/catalog/remote/base.cpp

@@ -0,0 +1,32 @@
+#include "./base.hpp"
+
+#include <dds/package/id.hpp>
+#include <dds/util/log.hpp>
+
+#include <nlohmann/json.hpp>
+
+using namespace dds;
+
+void remote_listing_base::apply_transforms(path_ref root) const {
+    for (const auto& tr : transforms) {
+        tr.apply_to(root);
+    }
+}
+
+void remote_listing_base::generate_auto_lib_files(const package_id& pid, path_ref root) const {
+    if (auto_lib.has_value()) {
+        dds_log(info, "Generating library data automatically");
+
+        auto pkg_strm         = open(root / "package.json5", std::ios::binary | std::ios::out);
+        auto man_json         = nlohmann::json::object();
+        man_json["name"]      = pid.name;
+        man_json["version"]   = pid.version.to_string();
+        man_json["namespace"] = auto_lib->namespace_;
+        pkg_strm << nlohmann::to_string(man_json);
+
+        auto lib_strm    = open(root / "library.json5", std::ios::binary | std::ios::out);
+        auto lib_json    = nlohmann::json::object();
+        lib_json["name"] = auto_lib->name;
+        lib_strm << nlohmann::to_string(lib_json);
+    }
+}

src/dds/catalog/remote/base.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <dds/util/fs_transform.hpp>
+
+#include <libman/package.hpp>
+#include <neo/concepts.hpp>
+
+#include <optional>
+#include <vector>
+
+namespace dds {
+
+struct package_id;
+
+struct remote_listing_base {
+    std::optional<lm::usage>       auto_lib{};
+    std::vector<fs_transformation> transforms{};
+
+    void apply_transforms(path_ref root) const;
+    void generate_auto_lib_files(const package_id& pid, path_ref root) const;
+};
+
+template <typename T>
+concept remote_listing = neo::derived_from<std::remove_cvref_t<T>, remote_listing_base>;
+
+}  // namespace dds

src/dds/catalog/remote/git.cpp

@@ -6,11 +6,10 @@
 
 #include <neo/url.hpp>
 #include <neo/url/query.hpp>
-#include <nlohmann/json.hpp>
 
 using namespace dds;
 
-void git_remote_listing::pull_to(const package_id& pid, path_ref dest) const {
+void git_remote_listing::pull_source(path_ref dest) const {
     fs::remove_all(dest);
     using namespace std::literals;
     dds_log(info, "Clone Git repository [{}] (at {}) to [{}]", url, ref, dest.string());
@@ -23,26 +22,6 @@ void git_remote_listing::pull_to(const package_id& pid, path_ref dest) const {
             git_res.retc,
             git_res.output);
     }
-
-    for (const auto& tr : transforms) {
-        tr.apply_to(dest);
-    }
-
-    if (auto_lib.has_value()) {
-        dds_log(info, "Generating library data automatically");
-
-        auto pkg_strm         = open(dest / "package.json5", std::ios::binary | std::ios::out);
-        auto man_json         = nlohmann::json::object();
-        man_json["name"]      = pid.name;
-        man_json["version"]   = pid.version.to_string();
-        man_json["namespace"] = auto_lib->namespace_;
-        pkg_strm << nlohmann::to_string(man_json);
-
-        auto lib_strm    = open(dest / "library.json5", std::ios::binary | std::ios::out);
-        auto lib_json    = nlohmann::json::object();
-        lib_json["name"] = auto_lib->name;
-        lib_strm << nlohmann::to_string(lib_json);
-    }
 }
 
 git_remote_listing git_remote_listing::from_url(std::string_view sv) {
@@ -79,5 +58,9 @@ git_remote_listing git_remote_listing::from_url(std::string_view sv) {
         throw_user_error<errc::invalid_remote_url>(
             "Git URL requires a fragment specifying the Git ref to clone");
     }
-    return {.url = url.to_string(), .ref = *ref, .auto_lib = auto_lib, .transforms = {}};
+    return git_remote_listing{
+        {.auto_lib = auto_lib},
+        url.to_string(),
+        *ref,
+    };
 }

src/dds/catalog/remote/git.hpp

@@ -1,24 +1,17 @@
 #pragma once
 
-#include <dds/catalog/get.hpp>
-#include <dds/util/fs.hpp>
-#include <dds/util/fs_transform.hpp>
+#include "./base.hpp"
 
-#include <libman/package.hpp>
-
-#include <optional>
 #include <string>
+#include <string_view>
 
 namespace dds {
 
-struct git_remote_listing {
-    std::string              url;
-    std::string              ref;
-    std::optional<lm::usage> auto_lib;
-
-    std::vector<fs_transformation> transforms;
+struct git_remote_listing : remote_listing_base {
+    std::string url;
+    std::string ref;
 
-    void pull_to(const package_id& pid, path_ref path) const;
+    void pull_source(path_ref path) const;
 
     static git_remote_listing from_url(std::string_view sv);
 };

src/dds/catalog/remote/http.cpp

@@ -60,7 +60,7 @@ void http_download_with_redir(neo::url url, path_ref dest) {
                     res_head.status_message,
                     loc->value);
             auto new_url = neo::url::try_parse(loc->value);
-            auto err     = std::get_if<neo::url_validation_error>(&new_url);
+            auto err     = std::get_if<neo::url_parse_error>(&new_url);
             if (err) {
                 throw_external_error<errc::http_download_failure>(
                     "Server returned an invalid URL for HTTP redirection [{}]", loc->value);
@@ -83,7 +83,7 @@ void http_download_with_redir(neo::url url, path_ref dest) {
 
 }  // namespace
 
-void http_remote_listing::pull_to(path_ref dest) const {
+void http_remote_listing::pull_source(path_ref dest) const {
     neo::url url;
     try {
         url = neo::url::parse(this->url);
@@ -128,26 +128,33 @@ void http_remote_listing::pull_to(path_ref dest) const {
     fs::create_directories(dest);
     dds_log(debug, "Expanding downloaded source distribution into {}", dest.string());
     std::ifstream infile{dl_path, std::ios::binary};
-    neo::expand_directory_targz(
-        neo::expand_options{
-            .destination_directory = dest,
-            .input_name            = dl_path.string(),
-            .strip_components      = this->strip_components,
-        },
-        infile);
+    try {
+        neo::expand_directory_targz(
+            neo::expand_options{
+                .destination_directory = dest,
+                .input_name            = dl_path.string(),
+                .strip_components      = this->strip_components,
+            },
+            infile);
+    } catch (const std::runtime_error& err) {
+        throw_external_error<errc::invalid_remote_url>(
+            "The file downloaded from [{}] failed to extract (Inner error: {})",
+            this->url,
+            err.what());
+    }
 }
 
 http_remote_listing http_remote_listing::from_url(std::string_view sv) {
     auto url = neo::url::parse(sv);
     dds_log(trace, "Create HTTP remote listing from URL [{}]", sv);
 
-    auto q = url.query;
-
-    unsigned                 strip_components = 0;
+    // Because archives most often have one top-level directory, the default strip-components
+    // setting is 'one'
+    unsigned int             strip_components = 1;
     std::optional<lm::usage> auto_lib;
 
-    if (q) {
-        neo::basic_query_string_view qsv{*q};
+    if (url.query) {
+        neo::basic_query_string_view qsv{*url.query};
         for (auto qstr : qsv) {
             if (qstr.key_raw() == "dds_lm") {
                 auto_lib = lm::split_usage_string(qstr.value_decoded());
@@ -160,8 +167,8 @@ http_remote_listing http_remote_listing::from_url(std::string_view sv) {
     }
 
     return http_remote_listing{
-        .url              = url.to_string(),
-        .strip_components = strip_components,
-        .auto_lib         = auto_lib,
+        {.auto_lib = auto_lib},
+        url.to_string(),
+        strip_components,
     };
 }

src/dds/catalog/remote/http.hpp

@@ -1,21 +1,17 @@
 #pragma once
 
-#include <dds/package/id.hpp>
-#include <dds/util/fs.hpp>
-
-#include <libman/package.hpp>
+#include "./base.hpp"
 
 #include <string>
 #include <string_view>
 
 namespace dds {
 
-struct http_remote_listing {
-    std::string              url;
-    unsigned                 strip_components = 0;
-    std::optional<lm::usage> auto_lib{};
+struct http_remote_listing : remote_listing_base {
+    std::string url;
+    unsigned    strip_components = 0;
 
-    void pull_to(path_ref path) const;
+    void pull_source(path_ref path) const;
 
     static http_remote_listing from_url(std::string_view sv);
 };

src/dds/source/dist.cpp

@@ -152,6 +152,6 @@ temporary_sdist dds::expand_sdist_from_istream(std::istream& is, std::string_vie
 temporary_sdist dds::download_expand_sdist_targz(std::string_view url_str) {
     auto remote  = http_remote_listing::from_url(url_str);
     auto tempdir = temporary_dir::create();
-    remote.pull_to(tempdir.path());
+    remote.pull_source(tempdir.path());
     return {tempdir, sdist::from_directory(tempdir.path())};
 }

tests/catalog/get_test.py

@@ -1,4 +1,5 @@
 import json
+from contextlib import contextmanager
 
 from tests import dds, DDS
 from tests.fileutil import ensure_dir
@@ -6,28 +7,48 @@ from tests.fileutil import ensure_dir
 import pytest
 
 
-def test_get(dds: DDS):
+def load_catalog(dds: DDS, data):
     dds.scope.enter_context(ensure_dir(dds.build_dir))
     dds.catalog_create()
 
     json_path = dds.build_dir / 'catalog.json'
-    import_data = {
-        'version': 2,
-        'packages': {
-            'neo-sqlite3': {
-                '0.3.0': {
-                    'url':
-                    'git+https://github.com/vector-of-bool/neo-sqlite3.git#0.3.0',
-                },
-            },
-        },
-    }
     dds.scope.enter_context(
         dds.set_contents(json_path,
-                         json.dumps(import_data).encode()))
-
+                         json.dumps(data).encode()))
     dds.catalog_import(json_path)
 
+
+def test_get(dds: DDS):
+    load_catalog(
+        dds, {
+            'version': 2,
+            'packages': {
+                'neo-sqlite3': {
+                    '0.3.0': {
+                        'url':
+                        'git+https://github.com/vector-of-bool/neo-sqlite3.git#0.3.0',
+                    },
+                },
+            },
+        })
+
     dds.catalog_get('neo-sqlite3@0.3.0')
-    assert (dds.source_root / 'neo-sqlite3@0.3.0').is_dir()
-    assert (dds.source_root / 'neo-sqlite3@0.3.0/package.jsonc').is_file()
+    assert (dds.scratch_dir / 'neo-sqlite3@0.3.0').is_dir()
+    assert (dds.scratch_dir / 'neo-sqlite3@0.3.0/package.jsonc').is_file()
+
+
+def test_get_http(dds: DDS):
+    load_catalog(
+        dds, {
+            'version': 2,
+            'packages': {
+                'cmcstl2': {
+                    '2020.2.24': {
+                        'url':
+                        'https://github.com/CaseyCarter/cmcstl2/archive/684a96d527e4dc733897255c0177b784dc280980.tar.gz?dds_lm=cmc/stl2;',
+                    },
+                },
+            },
+        })
+    dds.catalog_get('cmcstl2@2020.2.24')
+    assert dds.scratch_dir.joinpath('cmcstl2@2020.2.24/include').is_dir()

tests/catalog/import_test.py

@@ -39,6 +39,11 @@ def test_import_json(dds: DDS):
                     'url': 'git+http://example.com#master',
                 },
             },
+            'bar': {
+                '1.5.1': {
+                    'url': 'http://example.com/bar-1.5.2.tgz'
+                },
+            }
         },
     }
     dds.scope.enter_context(

tests/dds.py

@@ -161,6 +161,7 @@ class DDS:
             'catalog',
             'get',
             f'--catalog={self.catalog_path}',
+            f'--out-dir={self.scratch_dir}',
             req,
         ])
 

tools/gcc-9-dev.jsonc

@@ -15,7 +15,10 @@
     ],
     "link_flags": [
         "-fuse-ld=lld",
-        "-fsanitize=address,undefined"
+        "-fsanitize=address,undefined",
+        "-l:libssl.a",
+        "-l:libcrypto.a",
+        "-ldl",
     ],
     "debug": true,
     "compiler_launcher": "ccache"

tools/gcc-9-rel-macos.jsonc

@@ -0,0 +1,23 @@
+{
+    "$schema": "../res/toolchain-schema.json",
+    "compiler_id": "gnu",
+    "c_compiler": "gcc-9",
+    "cxx_compiler": "g++-9",
+    "warning_flags": [
+        "-Werror",
+    ],
+    "flags": [
+        "-I/usr/local/opt/openssl@1.1/include",
+    ],
+    "cxx_flags": [
+        "-fconcepts",
+        "-std=c++2a",
+    ],
+    "link_flags": [
+        "-static-libgcc",
+        "-static-libstdc++",
+        "/usr/local/opt/openssl@1.1/lib/libssl.a",
+        "/usr/local/opt/openssl@1.1/lib/libcrypto.a",
+    ],
+    "optimize": true
+}

tools/gcc-9-rel.jsonc

@@ -12,7 +12,10 @@
     ],
     "link_flags": [
         "-static-libgcc",
-        "-static-libstdc++"
+        "-static-libstdc++",
+        "-l:libssl.a",
+        "-l:libcrypto.a",
+        "-ldl",
     ],
     "optimize": true
 }

tools/gcc-9-static-rel.jsonc

@@ -11,7 +11,12 @@
         "-std=c++2a",
     ],
     "link_flags": [
-        "-static"
+        "-static",
+        "-l:libssl.a",
+        "-l:libcrypto.a",
+        "-ldl",
+        // WORKAROUND: https://sourceware.org/legacy-ml/glibc-bugs/2018-09/msg00009.html
+        "-Wl,-u,pthread_mutex_lock,-u,pthread_mutex_unlock,-u,pthread_self",
     ],
     "optimize": true
 }

tools/gen-catalog-json.py

@@ -292,12 +292,12 @@ PACKAGES = [
     github_package('neo-buffer', 'vector-of-bool/neo-buffer',
                    ['0.2.1', '0.3.0', '0.4.0', '0.4.1', '0.4.2']),
     github_package('neo-compress', 'vector-of-bool/neo-compress', ['0.1.0', '0.1.1']),
-    github_package('neo-url', 'vector-of-bool/neo-url', ['0.1.0', '0.1.1', '0.1.2']),
+    github_package('neo-url', 'vector-of-bool/neo-url', ['0.1.0', '0.1.1', '0.1.2', '0.2.0', '0.2.1']),
     github_package('neo-sqlite3', 'vector-of-bool/neo-sqlite3',
                    ['0.2.3', '0.3.0', '0.4.0', '0.4.1']),
     github_package('neo-fun', 'vector-of-bool/neo-fun', [
         '0.1.1', '0.2.0', '0.2.1', '0.3.0', '0.3.1', '0.3.2', '0.4.0', '0.4.1',
-        '0.4.2', '0.5.0', '0.5.1', '0.5.2', '0.5.3', '0.5.4',
+        '0.4.2', '0.5.0', '0.5.1', '0.5.2', '0.5.3', '0.5.4', '0.5.5',
     ]),
     github_package('neo-io', 'vector-of-bool/neo-io', ['0.1.0']),
     github_package('neo-http', 'vector-of-bool/neo-http', ['0.1.0']),

tools/get-win-openssl.ps1

@@ -0,0 +1,30 @@
+[CmdletBinding()]
+param ()
+
+$ErrorActionPreference = "Stop"
+
+$this_script = $MyInvocation.MyCommand.Definition
+$tools_dir = Split-Path -Parent $this_script
+$root_dir = Split-Path -Parent $tools_dir
+$build_dir = Join-Path $root_dir "_build"
+New-Item -ItemType Container $build_dir -ErrorAction Ignore
+
+$local_tgz = Join-Path $build_dir "openssl.tgz"
+
+# This is the path to the release static vs2019 x64 build of OpenSSL in bintray
+$conan_ssl_path = "_/openssl/1.1.1h/_/7098aea4e4f2247cc9b5dcaaa1ebddbe/package/a79a557254fabcb77849dd623fed97c9c5ab7651/141ef2c6711a254707ba1f7f4fd07ad4"
+$openssl_url = "https://dl.bintray.com/conan/conan-center/$conan_ssl_path/conan_package.tgz"
+
+Write-Host "Downloading OpenSSL for Windows"
+Invoke-WebRequest `
+    -Uri $openssl_url `
+    -OutFile $local_tgz
+
+$openssl_tree = Join-Path $root_dir "external/OpenSSL"
+Write-Host "Expanding OpenSSL archive..."
+Remove-Item $openssl_tree -Recurse -Force -ErrorAction Ignore
+New-Item $openssl_tree -ItemType Container | Out-Null
+& cmake -E chdir $openssl_tree cmake -E tar xf $local_tgz
+if ($LASTEXITCODE) {
+    throw "Archive expansion failed"
+}

tools/msvc.jsonc

@@ -5,9 +5,22 @@
         "/Zc:preprocessor",
         "/std:c++latest",
         "/DNOMINMAX",
+        // OpenSSL headers:
+        "/Iexternal/OpenSSL/include",
     ],
     "link_flags": [
         "rpcrt4.lib",
+        // Networking:
+        "Ws2_32.lib",
+        // Deps for OpenSSL:
+        "AdvApi32.lib",
+        "Crypt32.lib",
+        "User32.lib",
+        // Link in our external OpenSSL:
+        "/link",
+        "/LibPath:external/OpenSSL/lib",
+        "libssl.lib",
+        "libcrypto.lib",
     ],
     // "debug": true,
     "optimize": true