- CI step cleanups, and get openssl for windows - WORKAROUND: Bug in libgcc in certain conditions with partial pthread linked statically.default_compile_flags
| .mypy_cache/ | .mypy_cache/ | ||||
| _prebuilt/ | _prebuilt/ | ||||
| .pytest_cache/ | .pytest_cache/ | ||||
| .vagrant/ | |||||
| .vagrant/ | |||||
| external/OpenSSL |
| macos-ci: | macos-ci: | ||||
| python3 -u tools/ci.py \ | python3 -u tools/ci.py \ | ||||
| -B download \ | -B download \ | ||||
| -T tools/gcc-9-rel.jsonc | |||||
| -T tools/gcc-9-rel-macos.jsonc | |||||
| mv _build/dds _build/dds-macos-x64 | mv _build/dds _build/dds-macos-x64 | ||||
| linux-ci: | linux-ci: |
| pool: | pool: | ||||
| vmImage: windows-2019 | vmImage: windows-2019 | ||||
| steps: | steps: | ||||
| - pwsh: tools\get-win-openssl.ps1 | |||||
| displayName: Get OpenSSL for Windows | |||||
| - script: python -m pip install pytest pytest-xdist | |||||
| displayName: Install Python deps | |||||
| - script: | | - script: | | ||||
| echo Loading VS environment | echo Loading VS environment | ||||
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\vsdevcmd" -arch=x64 || exit 1 | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\vsdevcmd" -arch=x64 || exit 1 | ||||
| echo Executing Build and Tests | echo Executing Build and Tests | ||||
| reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1 | reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1 | ||||
| python -m pip install pytest pytest-xdist || exit 1 | |||||
| python -u tools/ci.py -B download -T tools\msvc.jsonc || exit 1 | python -u tools/ci.py -B download -T tools\msvc.jsonc || exit 1 | ||||
| move _build\dds.exe _build\dds-win-x64.exe || exit 1 | move _build\dds.exe _build\dds-win-x64.exe || exit 1 | ||||
| displayName: Build and Test | displayName: Build and Test | ||||
| pool: | pool: | ||||
| vmImage: ubuntu-18.04 | vmImage: ubuntu-18.04 | ||||
| steps: | steps: | ||||
| - script: | | |||||
| set -eu | |||||
| sudo apt update -y | |||||
| sudo apt install -y python3-minimal python3-setuptools g++-9 ccache | |||||
| python3 -m pip install pytest pytest-xdist | |||||
| displayName: Prepare System | |||||
| - script: sudo apt-get -y update | |||||
| displayName: APT Update | |||||
| - script: sudo apt-get -y install g++-9 ccache | |||||
| displayName: Get GCC 9 | |||||
| - script: sudo apt-get -y install python3-minimal python3-setuptools | |||||
| displayName: Get Python 3 | |||||
| - script: python3 -m pip install pytest pytest-xdist | |||||
| displayName: Get Python Dependencies | |||||
| - script: make linux-ci | - script: make linux-ci | ||||
| displayName: Build and Test | displayName: Build and Test | ||||
| - publish: _build/dds-linux-x64 | - publish: _build/dds-linux-x64 | ||||
| - job: macos_gcc9 | - job: macos_gcc9 | ||||
| displayName: macOS - GCC 9 | displayName: macOS - GCC 9 | ||||
| pool: | pool: | ||||
| vmImage: macOS-10.14 | |||||
| vmImage: macOS-10.15 | |||||
| steps: | steps: | ||||
| - script: brew install gcc@9 ccache | - script: brew install gcc@9 ccache | ||||
| displayName: Prepare System | |||||
| - script: | | |||||
| set -eu | |||||
| python3 -m pip install pytest pytest-xdist | |||||
| make macos-ci | |||||
| displayName: Get GCC 9 | |||||
| - script: brew install openssl@1.1 | |||||
| displayName: Install OpenSSL | |||||
| - script: python3 -m pip install pytest pytest-xdist | |||||
| displayName: Get Python Dependencies | |||||
| - script: make macos-ci | |||||
| displayName: Build and Test | displayName: Build and Test | ||||
| - publish: _build/dds-macos-x64 | - publish: _build/dds-macos-x64 | ||||
| displayName: Publish | displayName: Publish |
| "description": "Some library components that didn't quite fit anywhere else...", | "description": "Some library components that didn't quite fit anywhere else...", | ||||
| "transform": [], | "transform": [], | ||||
| "url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.4" | "url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.4" | ||||
| }, | |||||
| "0.5.5": { | |||||
| "depends": [], | |||||
| "description": "Some library components that didn't quite fit anywhere else...", | |||||
| "transform": [], | |||||
| "url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.5" | |||||
| } | } | ||||
| }, | }, | ||||
| "neo-http": { | "neo-http": { | ||||
| "description": "URL parsing and handling library.", | "description": "URL parsing and handling library.", | ||||
| "transform": [], | "transform": [], | ||||
| "url": "git+https://github.com/vector-of-bool/neo-url.git#0.1.2" | "url": "git+https://github.com/vector-of-bool/neo-url.git#0.1.2" | ||||
| }, | |||||
| "0.2.0": { | |||||
| "depends": [ | |||||
| "neo-fun^0.5.5" | |||||
| ], | |||||
| "description": "URL parsing and handling library.", | |||||
| "transform": [], | |||||
| "url": "git+https://github.com/vector-of-bool/neo-url.git#0.2.0" | |||||
| }, | |||||
| "0.2.1": { | |||||
| "depends": [ | |||||
| "neo-fun^0.5.5" | |||||
| ], | |||||
| "description": "URL parsing and handling library.", | |||||
| "transform": [], | |||||
| "url": "git+https://github.com/vector-of-bool/neo-url.git#0.2.1" | |||||
| } | } | ||||
| }, | }, | ||||
| "nlohmann-json": { | "nlohmann-json": { |
| "transform": [], | "transform": [], | ||||
| "url": "https://github.com/vector-of-bool/neo-fun.git" | "url": "https://github.com/vector-of-bool/neo-fun.git" | ||||
| } | } | ||||
| }, | |||||
| "0.5.5": { | |||||
| "depends": [], | |||||
| "description": "Some library components that didn't quite fit anywhere else...", | |||||
| "git": { | |||||
| "ref": "0.5.5", | |||||
| "transform": [], | |||||
| "url": "https://github.com/vector-of-bool/neo-fun.git" | |||||
| } | |||||
| } | } | ||||
| }, | }, | ||||
| "neo-http": { | "neo-http": { | ||||
| "transform": [], | "transform": [], | ||||
| "url": "https://github.com/vector-of-bool/neo-url.git" | "url": "https://github.com/vector-of-bool/neo-url.git" | ||||
| } | } | ||||
| }, | |||||
| "0.2.0": { | |||||
| "depends": [ | |||||
| "neo-fun^0.5.5" | |||||
| ], | |||||
| "description": "URL parsing and handling library.", | |||||
| "git": { | |||||
| "ref": "0.2.0", | |||||
| "transform": [], | |||||
| "url": "https://github.com/vector-of-bool/neo-url.git" | |||||
| } | |||||
| }, | |||||
| "0.2.1": { | |||||
| "depends": [ | |||||
| "neo-fun^0.5.5" | |||||
| ], | |||||
| "description": "URL parsing and handling library.", | |||||
| "git": { | |||||
| "ref": "0.2.1", | |||||
| "transform": [], | |||||
| "url": "https://github.com/vector-of-bool/neo-url.git" | |||||
| } | |||||
| } | } | ||||
| }, | }, | ||||
| "nlohmann-json": { | "nlohmann-json": { |
| "neo-sqlite3@0.4.1", | "neo-sqlite3@0.4.1", | ||||
| "neo-fun~0.5.4", | "neo-fun~0.5.4", | ||||
| "neo-compress~0.1.1", | "neo-compress~0.1.1", | ||||
| "neo-url~0.1.2", | |||||
| "neo-url~0.2.1", | |||||
| "semver@0.2.2", | "semver@0.2.2", | ||||
| "pubgrub@0.2.1", | "pubgrub@0.2.1", | ||||
| "vob-json5@0.1.5", | "vob-json5@0.1.5", |
| catalog_path_flag cat_path{cmd}; | catalog_path_flag cat_path{cmd}; | ||||
| args::Positional<std::string> pkg_id{cmd, | args::Positional<std::string> pkg_id{cmd, | ||||
| "id", | |||||
| "<id>", | |||||
| "The name@version ID of the package to add", | "The name@version ID of the package to add", | ||||
| args::Options::Required}; | args::Options::Required}; | ||||
| string_flag auto_lib{cmd, | |||||
| "auto-lib", | |||||
| "Set the auto-library information for this package", | |||||
| {"auto-lib"}}; | |||||
| args::Positional<std::string> uri{cmd, | |||||
| "<uri>", | |||||
| "The URI of the package", | |||||
| args::Options::Required}; | |||||
| // string_flag auto_lib{cmd, | |||||
| // "auto-lib", | |||||
| // "Set the auto-library information for this package", | |||||
| // {"auto-lib"}}; | |||||
| args::ValueFlagList<std::string> deps{cmd, | args::ValueFlagList<std::string> deps{cmd, | ||||
| "depends", | "depends", | ||||
| "The dependencies of this package", | "The dependencies of this package", | ||||
| {"depends", 'd'}}; | {"depends", 'd'}}; | ||||
| string_flag git_url{cmd, "git-url", "The Git url for the package", {"git-url"}}; | |||||
| string_flag git_ref{cmd, | |||||
| "git-ref", | |||||
| "The Git ref to from which the source distribution should be created", | |||||
| {"git-ref"}}; | |||||
| string_flag description{cmd, "description", "A description of the package", {"desc"}}; | string_flag description{cmd, "description", "A description of the package", {"desc"}}; | ||||
| int run() { | int run() { | ||||
| // deps.push_back({dep_id.name, dep_id.version}); | // deps.push_back({dep_id.name, dep_id.version}); | ||||
| } | } | ||||
| dds::package_info info{ident, std::move(deps), description.Get(), {}}; | |||||
| auto remote = dds::parse_remote_url(uri.Get()); | |||||
| if (git_url) { | |||||
| if (!git_ref) { | |||||
| dds::throw_user_error<dds::errc::git_url_ref_mutual_req>(); | |||||
| } | |||||
| auto git = dds::git_remote_listing{git_url.Get(), git_ref.Get(), std::nullopt, {}}; | |||||
| if (auto_lib) { | |||||
| git.auto_lib = lm::split_usage_string(auto_lib.Get()); | |||||
| } | |||||
| info.remote = std::move(git); | |||||
| } else if (git_ref) { | |||||
| dds::throw_user_error<dds::errc::git_url_ref_mutual_req>(); | |||||
| } | |||||
| neo_assertion_breadcrumbs("Running 'catalog add'", | |||||
| uri.Get(), | |||||
| description.Get(), | |||||
| pkg_id.Get()); | |||||
| dds::package_info info{ident, std::move(deps), description.Get(), remote}; | |||||
| cat_path.open().store(info); | cat_path.open().store(info); | ||||
| return 0; | return 0; | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| void print_remote_info(const dds::http_remote_listing& http) { | |||||
| fmt::print("HTTP/S URL: {}", http.url); | |||||
| if (http.auto_lib) { | |||||
| fmt::print("Auto-lib: {}/{}", http.auto_lib->name, http.auto_lib->namespace_); | |||||
| } | |||||
| } | |||||
| void print_remote_info(std::monostate) { | void print_remote_info(std::monostate) { | ||||
| std::cout << "THIS ENTRY IS MISSING REMOTE INFORMATION!\n"; | std::cout << "THIS ENTRY IS MISSING REMOTE INFORMATION!\n"; | ||||
| } | } |
| void store_with_remote(neo::sqlite3::statement_cache& stmts, | void store_with_remote(neo::sqlite3::statement_cache& stmts, | ||||
| const package_info& pkg, | const package_info& pkg, | ||||
| const git_remote_listing& git) { | |||||
| auto lm_usage = git.auto_lib.value_or(lm::usage{}); | |||||
| const http_remote_listing& http) { | |||||
| nsql::exec( // | |||||
| stmts(R"( | |||||
| INSERT OR REPLACE INTO dds_cat_pkgs ( | |||||
| name, | |||||
| version, | |||||
| remote_url, | |||||
| description, | |||||
| repo_transform | |||||
| ) VALUES (?1, ?2, ?3, ?4, ?5) | |||||
| )"_sql), | |||||
| pkg.ident.name, | |||||
| pkg.ident.version.to_string(), | |||||
| http.url, | |||||
| pkg.description, | |||||
| transforms_to_json(http.transforms)); | |||||
| } | |||||
| void store_with_remote(neo::sqlite3::statement_cache& stmts, | |||||
| const package_info& pkg, | |||||
| const git_remote_listing& git) { | |||||
| std::string url = git.url; | std::string url = git.url; | ||||
| if (url.starts_with("https://") || url.starts_with("http://")) { | if (url.starts_with("https://") || url.starts_with("http://")) { | ||||
| url = "git+" + url; | url = "git+" + url; |
| dds::package_id("foo", semver::version::parse("1.2.3")), | dds::package_id("foo", semver::version::parse("1.2.3")), | ||||
| {}, | {}, | ||||
| "example", | "example", | ||||
| dds::git_remote_listing{"git+http://example.com", "master", std::nullopt, {}}, | |||||
| dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "master"}, | |||||
| }); | }); | ||||
| auto pkgs = db.by_name("foo"); | auto pkgs = db.by_name("foo"); | ||||
| dds::package_id("foo", semver::version::parse("1.2.3")), | dds::package_id("foo", semver::version::parse("1.2.3")), | ||||
| {}, | {}, | ||||
| "example", | "example", | ||||
| dds::git_remote_listing{"git+http://example.com", "develop", std::nullopt, {}}, | |||||
| dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "develop"}, | |||||
| })); | })); | ||||
| // The previous pkg_id is still a valid lookup key | // The previous pkg_id is still a valid lookup key | ||||
| info = db.get(pkgs[0]); | info = db.get(pkgs[0]); | ||||
| {"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}}, | {"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}}, | ||||
| }, | }, | ||||
| "example", | "example", | ||||
| dds::git_remote_listing{"git+http://example.com", "master", std::nullopt, {}}, | |||||
| dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "master"}, | |||||
| }); | }); | ||||
| auto pkgs = db.by_name("foo"); | auto pkgs = db.by_name("foo"); | ||||
| REQUIRE(pkgs.size() == 1); | REQUIRE(pkgs.size() == 1); |
| listing.ident.to_string()); | listing.ident.to_string()); | ||||
| } | } | ||||
| temporary_sdist do_pull_sdist(const package_info& listing, const git_remote_listing& git) { | |||||
| template <remote_listing R> | |||||
| temporary_sdist do_pull_sdist(const package_info& listing, const R& remote) { | |||||
| auto tmpdir = dds::temporary_dir::create(); | auto tmpdir = dds::temporary_dir::create(); | ||||
| git.pull_to(listing.ident, tmpdir.path()); | |||||
| remote.pull_source(tmpdir.path()); | |||||
| remote.apply_transforms(tmpdir.path()); | |||||
| remote.generate_auto_lib_files(listing.ident, tmpdir.path()); | |||||
| dds_log(info, "Create sdist from clone ..."); | |||||
| dds_log(info, "Create sdist ..."); | |||||
| sdist_params params; | sdist_params params; | ||||
| params.project_dir = tmpdir.path(); | params.project_dir = tmpdir.path(); | ||||
| auto sd_tmp_dir = dds::temporary_dir::create(); | auto sd_tmp_dir = dds::temporary_dir::create(); |
| try { | try { | ||||
| return parse_remote_url(str); | return parse_remote_url(str); | ||||
| } catch (const neo::url_validation_error& e) { | } catch (const neo::url_validation_error& e) { | ||||
| import_error("{}: Invalid URL: {}", walk.path(), str); | |||||
| import_error("{}: Invalid URL [{}]: {}", walk.path(), str, e.what()); | |||||
| } catch (const user_error<errc::invalid_remote_url>& e) { | } catch (const user_error<errc::invalid_remote_url>& e) { | ||||
| import_error("{}: Invalid URL: {}", walk.path(), e.what()); | import_error("{}: Invalid URL: {}", walk.path(), e.what()); | ||||
| } | } |
| #include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
| #include <neo/url.hpp> | #include <neo/url.hpp> | ||||
| #include <neo/utility.hpp> | |||||
| using namespace dds; | using namespace dds; | ||||
| dds::remote_listing_var dds::parse_remote_url(std::string_view sv) { | dds::remote_listing_var dds::parse_remote_url(std::string_view sv) { | ||||
| neo_assertion_breadcrumbs("Loading package remote from URI string", sv); | |||||
| auto url = neo::url::parse(sv); | auto url = neo::url::parse(sv); | ||||
| if (url.scheme == "git+https" || url.scheme == "git+http" || url.scheme == "https+git" | |||||
| || url.scheme == "http+git" || url.scheme == "git") { | |||||
| if (url.scheme == neo::oper::any_of("git+https", "git+http", "http+git", "https+git", "git")) { | |||||
| return git_remote_listing::from_url(sv); | return git_remote_listing::from_url(sv); | ||||
| } else if (url.scheme == neo::oper::any_of("http", "https")) { | |||||
| return http_remote_listing::from_url(sv); | |||||
| } else { | } else { | ||||
| throw_user_error< | throw_user_error< | ||||
| errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'", | errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'", |
| #pragma once | #pragma once | ||||
| #include "./remote/git.hpp" | #include "./remote/git.hpp" | ||||
| #include "./remote/http.hpp" | |||||
| #include <dds/deps.hpp> | #include <dds/deps.hpp> | ||||
| #include <dds/package/id.hpp> | #include <dds/package/id.hpp> | ||||
| namespace dds { | namespace dds { | ||||
| using remote_listing_var = std::variant<std::monostate, git_remote_listing>; | |||||
| using remote_listing_var = std::variant<std::monostate, git_remote_listing, http_remote_listing>; | |||||
| remote_listing_var parse_remote_url(std::string_view url); | remote_listing_var parse_remote_url(std::string_view url); | ||||
| #include "./base.hpp" | |||||
| #include <dds/package/id.hpp> | |||||
| #include <dds/util/log.hpp> | |||||
| #include <nlohmann/json.hpp> | |||||
| using namespace dds; | |||||
| void remote_listing_base::apply_transforms(path_ref root) const { | |||||
| for (const auto& tr : transforms) { | |||||
| tr.apply_to(root); | |||||
| } | |||||
| } | |||||
| void remote_listing_base::generate_auto_lib_files(const package_id& pid, path_ref root) const { | |||||
| if (auto_lib.has_value()) { | |||||
| dds_log(info, "Generating library data automatically"); | |||||
| auto pkg_strm = open(root / "package.json5", std::ios::binary | std::ios::out); | |||||
| auto man_json = nlohmann::json::object(); | |||||
| man_json["name"] = pid.name; | |||||
| man_json["version"] = pid.version.to_string(); | |||||
| man_json["namespace"] = auto_lib->namespace_; | |||||
| pkg_strm << nlohmann::to_string(man_json); | |||||
| auto lib_strm = open(root / "library.json5", std::ios::binary | std::ios::out); | |||||
| auto lib_json = nlohmann::json::object(); | |||||
| lib_json["name"] = auto_lib->name; | |||||
| lib_strm << nlohmann::to_string(lib_json); | |||||
| } | |||||
| } |
| #pragma once | |||||
| #include <dds/util/fs_transform.hpp> | |||||
| #include <libman/package.hpp> | |||||
| #include <neo/concepts.hpp> | |||||
| #include <optional> | |||||
| #include <vector> | |||||
| namespace dds { | |||||
| struct package_id; | |||||
| struct remote_listing_base { | |||||
| std::optional<lm::usage> auto_lib{}; | |||||
| std::vector<fs_transformation> transforms{}; | |||||
| void apply_transforms(path_ref root) const; | |||||
| void generate_auto_lib_files(const package_id& pid, path_ref root) const; | |||||
| }; | |||||
| template <typename T> | |||||
| concept remote_listing = neo::derived_from<std::remove_cvref_t<T>, remote_listing_base>; | |||||
| } // namespace dds |
| #include <neo/url.hpp> | #include <neo/url.hpp> | ||||
| #include <neo/url/query.hpp> | #include <neo/url/query.hpp> | ||||
| #include <nlohmann/json.hpp> | |||||
| using namespace dds; | using namespace dds; | ||||
| void git_remote_listing::pull_to(const package_id& pid, path_ref dest) const { | |||||
| void git_remote_listing::pull_source(path_ref dest) const { | |||||
| fs::remove_all(dest); | fs::remove_all(dest); | ||||
| using namespace std::literals; | using namespace std::literals; | ||||
| dds_log(info, "Clone Git repository [{}] (at {}) to [{}]", url, ref, dest.string()); | dds_log(info, "Clone Git repository [{}] (at {}) to [{}]", url, ref, dest.string()); | ||||
| git_res.retc, | git_res.retc, | ||||
| git_res.output); | git_res.output); | ||||
| } | } | ||||
| for (const auto& tr : transforms) { | |||||
| tr.apply_to(dest); | |||||
| } | |||||
| if (auto_lib.has_value()) { | |||||
| dds_log(info, "Generating library data automatically"); | |||||
| auto pkg_strm = open(dest / "package.json5", std::ios::binary | std::ios::out); | |||||
| auto man_json = nlohmann::json::object(); | |||||
| man_json["name"] = pid.name; | |||||
| man_json["version"] = pid.version.to_string(); | |||||
| man_json["namespace"] = auto_lib->namespace_; | |||||
| pkg_strm << nlohmann::to_string(man_json); | |||||
| auto lib_strm = open(dest / "library.json5", std::ios::binary | std::ios::out); | |||||
| auto lib_json = nlohmann::json::object(); | |||||
| lib_json["name"] = auto_lib->name; | |||||
| lib_strm << nlohmann::to_string(lib_json); | |||||
| } | |||||
| } | } | ||||
| git_remote_listing git_remote_listing::from_url(std::string_view sv) { | git_remote_listing git_remote_listing::from_url(std::string_view sv) { | ||||
| throw_user_error<errc::invalid_remote_url>( | throw_user_error<errc::invalid_remote_url>( | ||||
| "Git URL requires a fragment specifying the Git ref to clone"); | "Git URL requires a fragment specifying the Git ref to clone"); | ||||
| } | } | ||||
| return {.url = url.to_string(), .ref = *ref, .auto_lib = auto_lib, .transforms = {}}; | |||||
| return git_remote_listing{ | |||||
| {.auto_lib = auto_lib}, | |||||
| url.to_string(), | |||||
| *ref, | |||||
| }; | |||||
| } | } |
| #pragma once | #pragma once | ||||
| #include <dds/catalog/get.hpp> | |||||
| #include <dds/util/fs.hpp> | |||||
| #include <dds/util/fs_transform.hpp> | |||||
| #include "./base.hpp" | |||||
| #include <libman/package.hpp> | |||||
| #include <optional> | |||||
| #include <string> | #include <string> | ||||
| #include <string_view> | |||||
| namespace dds { | namespace dds { | ||||
| struct git_remote_listing { | |||||
| std::string url; | |||||
| std::string ref; | |||||
| std::optional<lm::usage> auto_lib; | |||||
| std::vector<fs_transformation> transforms; | |||||
| struct git_remote_listing : remote_listing_base { | |||||
| std::string url; | |||||
| std::string ref; | |||||
| void pull_to(const package_id& pid, path_ref path) const; | |||||
| void pull_source(path_ref path) const; | |||||
| static git_remote_listing from_url(std::string_view sv); | static git_remote_listing from_url(std::string_view sv); | ||||
| }; | }; |
| res_head.status_message, | res_head.status_message, | ||||
| loc->value); | loc->value); | ||||
| auto new_url = neo::url::try_parse(loc->value); | auto new_url = neo::url::try_parse(loc->value); | ||||
| auto err = std::get_if<neo::url_validation_error>(&new_url); | |||||
| auto err = std::get_if<neo::url_parse_error>(&new_url); | |||||
| if (err) { | if (err) { | ||||
| throw_external_error<errc::http_download_failure>( | throw_external_error<errc::http_download_failure>( | ||||
| "Server returned an invalid URL for HTTP redirection [{}]", loc->value); | "Server returned an invalid URL for HTTP redirection [{}]", loc->value); | ||||
| } // namespace | } // namespace | ||||
| void http_remote_listing::pull_to(path_ref dest) const { | |||||
| void http_remote_listing::pull_source(path_ref dest) const { | |||||
| neo::url url; | neo::url url; | ||||
| try { | try { | ||||
| url = neo::url::parse(this->url); | url = neo::url::parse(this->url); | ||||
| fs::create_directories(dest); | fs::create_directories(dest); | ||||
| dds_log(debug, "Expanding downloaded source distribution into {}", dest.string()); | dds_log(debug, "Expanding downloaded source distribution into {}", dest.string()); | ||||
| std::ifstream infile{dl_path, std::ios::binary}; | std::ifstream infile{dl_path, std::ios::binary}; | ||||
| neo::expand_directory_targz( | |||||
| neo::expand_options{ | |||||
| .destination_directory = dest, | |||||
| .input_name = dl_path.string(), | |||||
| .strip_components = this->strip_components, | |||||
| }, | |||||
| infile); | |||||
| try { | |||||
| neo::expand_directory_targz( | |||||
| neo::expand_options{ | |||||
| .destination_directory = dest, | |||||
| .input_name = dl_path.string(), | |||||
| .strip_components = this->strip_components, | |||||
| }, | |||||
| infile); | |||||
| } catch (const std::runtime_error& err) { | |||||
| throw_external_error<errc::invalid_remote_url>( | |||||
| "The file downloaded from [{}] failed to extract (Inner error: {})", | |||||
| this->url, | |||||
| err.what()); | |||||
| } | |||||
| } | } | ||||
| http_remote_listing http_remote_listing::from_url(std::string_view sv) { | http_remote_listing http_remote_listing::from_url(std::string_view sv) { | ||||
| auto url = neo::url::parse(sv); | auto url = neo::url::parse(sv); | ||||
| dds_log(trace, "Create HTTP remote listing from URL [{}]", sv); | dds_log(trace, "Create HTTP remote listing from URL [{}]", sv); | ||||
| auto q = url.query; | |||||
| unsigned strip_components = 0; | |||||
| // Because archives most often have one top-level directory, the default strip-components | |||||
| // setting is 'one' | |||||
| unsigned int strip_components = 1; | |||||
| std::optional<lm::usage> auto_lib; | std::optional<lm::usage> auto_lib; | ||||
| if (q) { | |||||
| neo::basic_query_string_view qsv{*q}; | |||||
| if (url.query) { | |||||
| neo::basic_query_string_view qsv{*url.query}; | |||||
| for (auto qstr : qsv) { | for (auto qstr : qsv) { | ||||
| if (qstr.key_raw() == "dds_lm") { | if (qstr.key_raw() == "dds_lm") { | ||||
| auto_lib = lm::split_usage_string(qstr.value_decoded()); | auto_lib = lm::split_usage_string(qstr.value_decoded()); | ||||
| } | } | ||||
| return http_remote_listing{ | return http_remote_listing{ | ||||
| .url = url.to_string(), | |||||
| .strip_components = strip_components, | |||||
| .auto_lib = auto_lib, | |||||
| {.auto_lib = auto_lib}, | |||||
| url.to_string(), | |||||
| strip_components, | |||||
| }; | }; | ||||
| } | } |
| #pragma once | #pragma once | ||||
| #include <dds/package/id.hpp> | |||||
| #include <dds/util/fs.hpp> | |||||
| #include <libman/package.hpp> | |||||
| #include "./base.hpp" | |||||
| #include <string> | #include <string> | ||||
| #include <string_view> | #include <string_view> | ||||
| namespace dds { | namespace dds { | ||||
| struct http_remote_listing { | |||||
| std::string url; | |||||
| unsigned strip_components = 0; | |||||
| std::optional<lm::usage> auto_lib{}; | |||||
| struct http_remote_listing : remote_listing_base { | |||||
| std::string url; | |||||
| unsigned strip_components = 0; | |||||
| void pull_to(path_ref path) const; | |||||
| void pull_source(path_ref path) const; | |||||
| static http_remote_listing from_url(std::string_view sv); | static http_remote_listing from_url(std::string_view sv); | ||||
| }; | }; |
| temporary_sdist dds::download_expand_sdist_targz(std::string_view url_str) { | temporary_sdist dds::download_expand_sdist_targz(std::string_view url_str) { | ||||
| auto remote = http_remote_listing::from_url(url_str); | auto remote = http_remote_listing::from_url(url_str); | ||||
| auto tempdir = temporary_dir::create(); | auto tempdir = temporary_dir::create(); | ||||
| remote.pull_to(tempdir.path()); | |||||
| remote.pull_source(tempdir.path()); | |||||
| return {tempdir, sdist::from_directory(tempdir.path())}; | return {tempdir, sdist::from_directory(tempdir.path())}; | ||||
| } | } |
| import json | import json | ||||
| from contextlib import contextmanager | |||||
| from tests import dds, DDS | from tests import dds, DDS | ||||
| from tests.fileutil import ensure_dir | from tests.fileutil import ensure_dir | ||||
| import pytest | import pytest | ||||
| def test_get(dds: DDS): | |||||
| def load_catalog(dds: DDS, data): | |||||
| dds.scope.enter_context(ensure_dir(dds.build_dir)) | dds.scope.enter_context(ensure_dir(dds.build_dir)) | ||||
| dds.catalog_create() | dds.catalog_create() | ||||
| json_path = dds.build_dir / 'catalog.json' | json_path = dds.build_dir / 'catalog.json' | ||||
| import_data = { | |||||
| 'version': 2, | |||||
| 'packages': { | |||||
| 'neo-sqlite3': { | |||||
| '0.3.0': { | |||||
| 'url': | |||||
| 'git+https://github.com/vector-of-bool/neo-sqlite3.git#0.3.0', | |||||
| }, | |||||
| }, | |||||
| }, | |||||
| } | |||||
| dds.scope.enter_context( | dds.scope.enter_context( | ||||
| dds.set_contents(json_path, | dds.set_contents(json_path, | ||||
| json.dumps(import_data).encode())) | |||||
| json.dumps(data).encode())) | |||||
| dds.catalog_import(json_path) | dds.catalog_import(json_path) | ||||
| def test_get(dds: DDS): | |||||
| load_catalog( | |||||
| dds, { | |||||
| 'version': 2, | |||||
| 'packages': { | |||||
| 'neo-sqlite3': { | |||||
| '0.3.0': { | |||||
| 'url': | |||||
| 'git+https://github.com/vector-of-bool/neo-sqlite3.git#0.3.0', | |||||
| }, | |||||
| }, | |||||
| }, | |||||
| }) | |||||
| dds.catalog_get('neo-sqlite3@0.3.0') | dds.catalog_get('neo-sqlite3@0.3.0') | ||||
| assert (dds.source_root / 'neo-sqlite3@0.3.0').is_dir() | |||||
| assert (dds.source_root / 'neo-sqlite3@0.3.0/package.jsonc').is_file() | |||||
| assert (dds.scratch_dir / 'neo-sqlite3@0.3.0').is_dir() | |||||
| assert (dds.scratch_dir / 'neo-sqlite3@0.3.0/package.jsonc').is_file() | |||||
| def test_get_http(dds: DDS): | |||||
| load_catalog( | |||||
| dds, { | |||||
| 'version': 2, | |||||
| 'packages': { | |||||
| 'cmcstl2': { | |||||
| '2020.2.24': { | |||||
| 'url': | |||||
| 'https://github.com/CaseyCarter/cmcstl2/archive/684a96d527e4dc733897255c0177b784dc280980.tar.gz?dds_lm=cmc/stl2;', | |||||
| }, | |||||
| }, | |||||
| }, | |||||
| }) | |||||
| dds.catalog_get('cmcstl2@2020.2.24') | |||||
| assert dds.scratch_dir.joinpath('cmcstl2@2020.2.24/include').is_dir() |
| 'url': 'git+http://example.com#master', | 'url': 'git+http://example.com#master', | ||||
| }, | }, | ||||
| }, | }, | ||||
| 'bar': { | |||||
| '1.5.1': { | |||||
| 'url': 'http://example.com/bar-1.5.2.tgz' | |||||
| }, | |||||
| } | |||||
| }, | }, | ||||
| } | } | ||||
| dds.scope.enter_context( | dds.scope.enter_context( |
| 'catalog', | 'catalog', | ||||
| 'get', | 'get', | ||||
| f'--catalog={self.catalog_path}', | f'--catalog={self.catalog_path}', | ||||
| f'--out-dir={self.scratch_dir}', | |||||
| req, | req, | ||||
| ]) | ]) | ||||
| ], | ], | ||||
| "link_flags": [ | "link_flags": [ | ||||
| "-fuse-ld=lld", | "-fuse-ld=lld", | ||||
| "-fsanitize=address,undefined" | |||||
| "-fsanitize=address,undefined", | |||||
| "-l:libssl.a", | |||||
| "-l:libcrypto.a", | |||||
| "-ldl", | |||||
| ], | ], | ||||
| "debug": true, | "debug": true, | ||||
| "compiler_launcher": "ccache" | "compiler_launcher": "ccache" |
| { | |||||
| "$schema": "../res/toolchain-schema.json", | |||||
| "compiler_id": "gnu", | |||||
| "c_compiler": "gcc-9", | |||||
| "cxx_compiler": "g++-9", | |||||
| "warning_flags": [ | |||||
| "-Werror", | |||||
| ], | |||||
| "flags": [ | |||||
| "-I/usr/local/opt/openssl@1.1/include", | |||||
| ], | |||||
| "cxx_flags": [ | |||||
| "-fconcepts", | |||||
| "-std=c++2a", | |||||
| ], | |||||
| "link_flags": [ | |||||
| "-static-libgcc", | |||||
| "-static-libstdc++", | |||||
| "/usr/local/opt/openssl@1.1/lib/libssl.a", | |||||
| "/usr/local/opt/openssl@1.1/lib/libcrypto.a", | |||||
| ], | |||||
| "optimize": true | |||||
| } |
| ], | ], | ||||
| "link_flags": [ | "link_flags": [ | ||||
| "-static-libgcc", | "-static-libgcc", | ||||
| "-static-libstdc++" | |||||
| "-static-libstdc++", | |||||
| "-l:libssl.a", | |||||
| "-l:libcrypto.a", | |||||
| "-ldl", | |||||
| ], | ], | ||||
| "optimize": true | "optimize": true | ||||
| } | } |
| "-std=c++2a", | "-std=c++2a", | ||||
| ], | ], | ||||
| "link_flags": [ | "link_flags": [ | ||||
| "-static" | |||||
| "-static", | |||||
| "-l:libssl.a", | |||||
| "-l:libcrypto.a", | |||||
| "-ldl", | |||||
| // WORKAROUND: https://sourceware.org/legacy-ml/glibc-bugs/2018-09/msg00009.html | |||||
| "-Wl,-u,pthread_mutex_lock,-u,pthread_mutex_unlock,-u,pthread_self", | |||||
| ], | ], | ||||
| "optimize": true | "optimize": true | ||||
| } | } |
| github_package('neo-buffer', 'vector-of-bool/neo-buffer', | github_package('neo-buffer', 'vector-of-bool/neo-buffer', | ||||
| ['0.2.1', '0.3.0', '0.4.0', '0.4.1', '0.4.2']), | ['0.2.1', '0.3.0', '0.4.0', '0.4.1', '0.4.2']), | ||||
| github_package('neo-compress', 'vector-of-bool/neo-compress', ['0.1.0', '0.1.1']), | github_package('neo-compress', 'vector-of-bool/neo-compress', ['0.1.0', '0.1.1']), | ||||
| github_package('neo-url', 'vector-of-bool/neo-url', ['0.1.0', '0.1.1', '0.1.2']), | |||||
| github_package('neo-url', 'vector-of-bool/neo-url', ['0.1.0', '0.1.1', '0.1.2', '0.2.0', '0.2.1']), | |||||
| github_package('neo-sqlite3', 'vector-of-bool/neo-sqlite3', | github_package('neo-sqlite3', 'vector-of-bool/neo-sqlite3', | ||||
| ['0.2.3', '0.3.0', '0.4.0', '0.4.1']), | ['0.2.3', '0.3.0', '0.4.0', '0.4.1']), | ||||
| github_package('neo-fun', 'vector-of-bool/neo-fun', [ | github_package('neo-fun', 'vector-of-bool/neo-fun', [ | ||||
| '0.1.1', '0.2.0', '0.2.1', '0.3.0', '0.3.1', '0.3.2', '0.4.0', '0.4.1', | '0.1.1', '0.2.0', '0.2.1', '0.3.0', '0.3.1', '0.3.2', '0.4.0', '0.4.1', | ||||
| '0.4.2', '0.5.0', '0.5.1', '0.5.2', '0.5.3', '0.5.4', | |||||
| '0.4.2', '0.5.0', '0.5.1', '0.5.2', '0.5.3', '0.5.4', '0.5.5', | |||||
| ]), | ]), | ||||
| github_package('neo-io', 'vector-of-bool/neo-io', ['0.1.0']), | github_package('neo-io', 'vector-of-bool/neo-io', ['0.1.0']), | ||||
| github_package('neo-http', 'vector-of-bool/neo-http', ['0.1.0']), | github_package('neo-http', 'vector-of-bool/neo-http', ['0.1.0']), |
| [CmdletBinding()] | |||||
| param () | |||||
| $ErrorActionPreference = "Stop" | |||||
| $this_script = $MyInvocation.MyCommand.Definition | |||||
| $tools_dir = Split-Path -Parent $this_script | |||||
| $root_dir = Split-Path -Parent $tools_dir | |||||
| $build_dir = Join-Path $root_dir "_build" | |||||
| New-Item -ItemType Container $build_dir -ErrorAction Ignore | |||||
| $local_tgz = Join-Path $build_dir "openssl.tgz" | |||||
| # This is the path to the release static vs2019 x64 build of OpenSSL in bintray | |||||
| $conan_ssl_path = "_/openssl/1.1.1h/_/7098aea4e4f2247cc9b5dcaaa1ebddbe/package/a79a557254fabcb77849dd623fed97c9c5ab7651/141ef2c6711a254707ba1f7f4fd07ad4" | |||||
| $openssl_url = "https://dl.bintray.com/conan/conan-center/$conan_ssl_path/conan_package.tgz" | |||||
| Write-Host "Downloading OpenSSL for Windows" | |||||
| Invoke-WebRequest ` | |||||
| -Uri $openssl_url ` | |||||
| -OutFile $local_tgz | |||||
| $openssl_tree = Join-Path $root_dir "external/OpenSSL" | |||||
| Write-Host "Expanding OpenSSL archive..." | |||||
| Remove-Item $openssl_tree -Recurse -Force -ErrorAction Ignore | |||||
| New-Item $openssl_tree -ItemType Container | Out-Null | |||||
| & cmake -E chdir $openssl_tree cmake -E tar xf $local_tgz | |||||
| if ($LASTEXITCODE) { | |||||
| throw "Archive expansion failed" | |||||
| } |
| "/Zc:preprocessor", | "/Zc:preprocessor", | ||||
| "/std:c++latest", | "/std:c++latest", | ||||
| "/DNOMINMAX", | "/DNOMINMAX", | ||||
| // OpenSSL headers: | |||||
| "/Iexternal/OpenSSL/include", | |||||
| ], | ], | ||||
| "link_flags": [ | "link_flags": [ | ||||
| "rpcrt4.lib", | "rpcrt4.lib", | ||||
| // Networking: | |||||
| "Ws2_32.lib", | |||||
| // Deps for OpenSSL: | |||||
| "AdvApi32.lib", | |||||
| "Crypt32.lib", | |||||
| "User32.lib", | |||||
| // Link in our external OpenSSL: | |||||
| "/link", | |||||
| "/LibPath:external/OpenSSL/lib", | |||||
| "libssl.lib", | |||||
| "libcrypto.lib", | |||||
| ], | ], | ||||
| // "debug": true, | // "debug": true, | ||||
| "optimize": true | "optimize": true |