- CI step cleanups, and get openssl for windows - WORKAROUND: Bug in libgcc in certain conditions with partial pthread linked statically.default_compile_flags
.mypy_cache/ | .mypy_cache/ | ||||
_prebuilt/ | _prebuilt/ | ||||
.pytest_cache/ | .pytest_cache/ | ||||
.vagrant/ | |||||
.vagrant/ | |||||
external/OpenSSL |
macos-ci: | macos-ci: | ||||
python3 -u tools/ci.py \ | python3 -u tools/ci.py \ | ||||
-B download \ | -B download \ | ||||
-T tools/gcc-9-rel.jsonc | |||||
-T tools/gcc-9-rel-macos.jsonc | |||||
mv _build/dds _build/dds-macos-x64 | mv _build/dds _build/dds-macos-x64 | ||||
linux-ci: | linux-ci: |
pool: | pool: | ||||
vmImage: windows-2019 | vmImage: windows-2019 | ||||
steps: | steps: | ||||
- pwsh: tools\get-win-openssl.ps1 | |||||
displayName: Get OpenSSL for Windows | |||||
- script: python -m pip install pytest pytest-xdist | |||||
displayName: Install Python deps | |||||
- script: | | - script: | | ||||
echo Loading VS environment | echo Loading VS environment | ||||
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\vsdevcmd" -arch=x64 || exit 1 | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\vsdevcmd" -arch=x64 || exit 1 | ||||
echo Executing Build and Tests | echo Executing Build and Tests | ||||
reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1 | reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f || exit 1 | ||||
python -m pip install pytest pytest-xdist || exit 1 | |||||
python -u tools/ci.py -B download -T tools\msvc.jsonc || exit 1 | python -u tools/ci.py -B download -T tools\msvc.jsonc || exit 1 | ||||
move _build\dds.exe _build\dds-win-x64.exe || exit 1 | move _build\dds.exe _build\dds-win-x64.exe || exit 1 | ||||
displayName: Build and Test | displayName: Build and Test | ||||
pool: | pool: | ||||
vmImage: ubuntu-18.04 | vmImage: ubuntu-18.04 | ||||
steps: | steps: | ||||
- script: | | |||||
set -eu | |||||
sudo apt update -y | |||||
sudo apt install -y python3-minimal python3-setuptools g++-9 ccache | |||||
python3 -m pip install pytest pytest-xdist | |||||
displayName: Prepare System | |||||
- script: sudo apt-get -y update | |||||
displayName: APT Update | |||||
- script: sudo apt-get -y install g++-9 ccache | |||||
displayName: Get GCC 9 | |||||
- script: sudo apt-get -y install python3-minimal python3-setuptools | |||||
displayName: Get Python 3 | |||||
- script: python3 -m pip install pytest pytest-xdist | |||||
displayName: Get Python Dependencies | |||||
- script: make linux-ci | - script: make linux-ci | ||||
displayName: Build and Test | displayName: Build and Test | ||||
- publish: _build/dds-linux-x64 | - publish: _build/dds-linux-x64 | ||||
- job: macos_gcc9 | - job: macos_gcc9 | ||||
displayName: macOS - GCC 9 | displayName: macOS - GCC 9 | ||||
pool: | pool: | ||||
vmImage: macOS-10.14 | |||||
vmImage: macOS-10.15 | |||||
steps: | steps: | ||||
- script: brew install gcc@9 ccache | - script: brew install gcc@9 ccache | ||||
displayName: Prepare System | |||||
- script: | | |||||
set -eu | |||||
python3 -m pip install pytest pytest-xdist | |||||
make macos-ci | |||||
displayName: Get GCC 9 | |||||
- script: brew install openssl@1.1 | |||||
displayName: Install OpenSSL | |||||
- script: python3 -m pip install pytest pytest-xdist | |||||
displayName: Get Python Dependencies | |||||
- script: make macos-ci | |||||
displayName: Build and Test | displayName: Build and Test | ||||
- publish: _build/dds-macos-x64 | - publish: _build/dds-macos-x64 | ||||
displayName: Publish | displayName: Publish |
"description": "Some library components that didn't quite fit anywhere else...", | "description": "Some library components that didn't quite fit anywhere else...", | ||||
"transform": [], | "transform": [], | ||||
"url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.4" | "url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.4" | ||||
}, | |||||
"0.5.5": { | |||||
"depends": [], | |||||
"description": "Some library components that didn't quite fit anywhere else...", | |||||
"transform": [], | |||||
"url": "git+https://github.com/vector-of-bool/neo-fun.git#0.5.5" | |||||
} | } | ||||
}, | }, | ||||
"neo-http": { | "neo-http": { | ||||
"description": "URL parsing and handling library.", | "description": "URL parsing and handling library.", | ||||
"transform": [], | "transform": [], | ||||
"url": "git+https://github.com/vector-of-bool/neo-url.git#0.1.2" | "url": "git+https://github.com/vector-of-bool/neo-url.git#0.1.2" | ||||
}, | |||||
"0.2.0": { | |||||
"depends": [ | |||||
"neo-fun^0.5.5" | |||||
], | |||||
"description": "URL parsing and handling library.", | |||||
"transform": [], | |||||
"url": "git+https://github.com/vector-of-bool/neo-url.git#0.2.0" | |||||
}, | |||||
"0.2.1": { | |||||
"depends": [ | |||||
"neo-fun^0.5.5" | |||||
], | |||||
"description": "URL parsing and handling library.", | |||||
"transform": [], | |||||
"url": "git+https://github.com/vector-of-bool/neo-url.git#0.2.1" | |||||
} | } | ||||
}, | }, | ||||
"nlohmann-json": { | "nlohmann-json": { |
"transform": [], | "transform": [], | ||||
"url": "https://github.com/vector-of-bool/neo-fun.git" | "url": "https://github.com/vector-of-bool/neo-fun.git" | ||||
} | } | ||||
}, | |||||
"0.5.5": { | |||||
"depends": [], | |||||
"description": "Some library components that didn't quite fit anywhere else...", | |||||
"git": { | |||||
"ref": "0.5.5", | |||||
"transform": [], | |||||
"url": "https://github.com/vector-of-bool/neo-fun.git" | |||||
} | |||||
} | } | ||||
}, | }, | ||||
"neo-http": { | "neo-http": { | ||||
"transform": [], | "transform": [], | ||||
"url": "https://github.com/vector-of-bool/neo-url.git" | "url": "https://github.com/vector-of-bool/neo-url.git" | ||||
} | } | ||||
}, | |||||
"0.2.0": { | |||||
"depends": [ | |||||
"neo-fun^0.5.5" | |||||
], | |||||
"description": "URL parsing and handling library.", | |||||
"git": { | |||||
"ref": "0.2.0", | |||||
"transform": [], | |||||
"url": "https://github.com/vector-of-bool/neo-url.git" | |||||
} | |||||
}, | |||||
"0.2.1": { | |||||
"depends": [ | |||||
"neo-fun^0.5.5" | |||||
], | |||||
"description": "URL parsing and handling library.", | |||||
"git": { | |||||
"ref": "0.2.1", | |||||
"transform": [], | |||||
"url": "https://github.com/vector-of-bool/neo-url.git" | |||||
} | |||||
} | } | ||||
}, | }, | ||||
"nlohmann-json": { | "nlohmann-json": { |
"neo-sqlite3@0.4.1", | "neo-sqlite3@0.4.1", | ||||
"neo-fun~0.5.4", | "neo-fun~0.5.4", | ||||
"neo-compress~0.1.1", | "neo-compress~0.1.1", | ||||
"neo-url~0.1.2", | |||||
"neo-url~0.2.1", | |||||
"semver@0.2.2", | "semver@0.2.2", | ||||
"pubgrub@0.2.1", | "pubgrub@0.2.1", | ||||
"vob-json5@0.1.5", | "vob-json5@0.1.5", |
catalog_path_flag cat_path{cmd}; | catalog_path_flag cat_path{cmd}; | ||||
args::Positional<std::string> pkg_id{cmd, | args::Positional<std::string> pkg_id{cmd, | ||||
"id", | |||||
"<id>", | |||||
"The name@version ID of the package to add", | "The name@version ID of the package to add", | ||||
args::Options::Required}; | args::Options::Required}; | ||||
string_flag auto_lib{cmd, | |||||
"auto-lib", | |||||
"Set the auto-library information for this package", | |||||
{"auto-lib"}}; | |||||
args::Positional<std::string> uri{cmd, | |||||
"<uri>", | |||||
"The URI of the package", | |||||
args::Options::Required}; | |||||
// string_flag auto_lib{cmd, | |||||
// "auto-lib", | |||||
// "Set the auto-library information for this package", | |||||
// {"auto-lib"}}; | |||||
args::ValueFlagList<std::string> deps{cmd, | args::ValueFlagList<std::string> deps{cmd, | ||||
"depends", | "depends", | ||||
"The dependencies of this package", | "The dependencies of this package", | ||||
{"depends", 'd'}}; | {"depends", 'd'}}; | ||||
string_flag git_url{cmd, "git-url", "The Git url for the package", {"git-url"}}; | |||||
string_flag git_ref{cmd, | |||||
"git-ref", | |||||
"The Git ref to from which the source distribution should be created", | |||||
{"git-ref"}}; | |||||
string_flag description{cmd, "description", "A description of the package", {"desc"}}; | string_flag description{cmd, "description", "A description of the package", {"desc"}}; | ||||
int run() { | int run() { | ||||
// deps.push_back({dep_id.name, dep_id.version}); | // deps.push_back({dep_id.name, dep_id.version}); | ||||
} | } | ||||
dds::package_info info{ident, std::move(deps), description.Get(), {}}; | |||||
auto remote = dds::parse_remote_url(uri.Get()); | |||||
if (git_url) { | |||||
if (!git_ref) { | |||||
dds::throw_user_error<dds::errc::git_url_ref_mutual_req>(); | |||||
} | |||||
auto git = dds::git_remote_listing{git_url.Get(), git_ref.Get(), std::nullopt, {}}; | |||||
if (auto_lib) { | |||||
git.auto_lib = lm::split_usage_string(auto_lib.Get()); | |||||
} | |||||
info.remote = std::move(git); | |||||
} else if (git_ref) { | |||||
dds::throw_user_error<dds::errc::git_url_ref_mutual_req>(); | |||||
} | |||||
neo_assertion_breadcrumbs("Running 'catalog add'", | |||||
uri.Get(), | |||||
description.Get(), | |||||
pkg_id.Get()); | |||||
dds::package_info info{ident, std::move(deps), description.Get(), remote}; | |||||
cat_path.open().store(info); | cat_path.open().store(info); | ||||
return 0; | return 0; | ||||
} | } | ||||
} | } | ||||
} | } | ||||
void print_remote_info(const dds::http_remote_listing& http) { | |||||
fmt::print("HTTP/S URL: {}", http.url); | |||||
if (http.auto_lib) { | |||||
fmt::print("Auto-lib: {}/{}", http.auto_lib->name, http.auto_lib->namespace_); | |||||
} | |||||
} | |||||
void print_remote_info(std::monostate) { | void print_remote_info(std::monostate) { | ||||
std::cout << "THIS ENTRY IS MISSING REMOTE INFORMATION!\n"; | std::cout << "THIS ENTRY IS MISSING REMOTE INFORMATION!\n"; | ||||
} | } |
void store_with_remote(neo::sqlite3::statement_cache& stmts, | void store_with_remote(neo::sqlite3::statement_cache& stmts, | ||||
const package_info& pkg, | const package_info& pkg, | ||||
const git_remote_listing& git) { | |||||
auto lm_usage = git.auto_lib.value_or(lm::usage{}); | |||||
const http_remote_listing& http) { | |||||
nsql::exec( // | |||||
stmts(R"( | |||||
INSERT OR REPLACE INTO dds_cat_pkgs ( | |||||
name, | |||||
version, | |||||
remote_url, | |||||
description, | |||||
repo_transform | |||||
) VALUES (?1, ?2, ?3, ?4, ?5) | |||||
)"_sql), | |||||
pkg.ident.name, | |||||
pkg.ident.version.to_string(), | |||||
http.url, | |||||
pkg.description, | |||||
transforms_to_json(http.transforms)); | |||||
} | |||||
void store_with_remote(neo::sqlite3::statement_cache& stmts, | |||||
const package_info& pkg, | |||||
const git_remote_listing& git) { | |||||
std::string url = git.url; | std::string url = git.url; | ||||
if (url.starts_with("https://") || url.starts_with("http://")) { | if (url.starts_with("https://") || url.starts_with("http://")) { | ||||
url = "git+" + url; | url = "git+" + url; |
dds::package_id("foo", semver::version::parse("1.2.3")), | dds::package_id("foo", semver::version::parse("1.2.3")), | ||||
{}, | {}, | ||||
"example", | "example", | ||||
dds::git_remote_listing{"git+http://example.com", "master", std::nullopt, {}}, | |||||
dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "master"}, | |||||
}); | }); | ||||
auto pkgs = db.by_name("foo"); | auto pkgs = db.by_name("foo"); | ||||
dds::package_id("foo", semver::version::parse("1.2.3")), | dds::package_id("foo", semver::version::parse("1.2.3")), | ||||
{}, | {}, | ||||
"example", | "example", | ||||
dds::git_remote_listing{"git+http://example.com", "develop", std::nullopt, {}}, | |||||
dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "develop"}, | |||||
})); | })); | ||||
// The previous pkg_id is still a valid lookup key | // The previous pkg_id is still a valid lookup key | ||||
info = db.get(pkgs[0]); | info = db.get(pkgs[0]); | ||||
{"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}}, | {"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}}, | ||||
}, | }, | ||||
"example", | "example", | ||||
dds::git_remote_listing{"git+http://example.com", "master", std::nullopt, {}}, | |||||
dds::git_remote_listing{std::nullopt, {}, "git+http://example.com", "master"}, | |||||
}); | }); | ||||
auto pkgs = db.by_name("foo"); | auto pkgs = db.by_name("foo"); | ||||
REQUIRE(pkgs.size() == 1); | REQUIRE(pkgs.size() == 1); |
listing.ident.to_string()); | listing.ident.to_string()); | ||||
} | } | ||||
temporary_sdist do_pull_sdist(const package_info& listing, const git_remote_listing& git) { | |||||
template <remote_listing R> | |||||
temporary_sdist do_pull_sdist(const package_info& listing, const R& remote) { | |||||
auto tmpdir = dds::temporary_dir::create(); | auto tmpdir = dds::temporary_dir::create(); | ||||
git.pull_to(listing.ident, tmpdir.path()); | |||||
remote.pull_source(tmpdir.path()); | |||||
remote.apply_transforms(tmpdir.path()); | |||||
remote.generate_auto_lib_files(listing.ident, tmpdir.path()); | |||||
dds_log(info, "Create sdist from clone ..."); | |||||
dds_log(info, "Create sdist ..."); | |||||
sdist_params params; | sdist_params params; | ||||
params.project_dir = tmpdir.path(); | params.project_dir = tmpdir.path(); | ||||
auto sd_tmp_dir = dds::temporary_dir::create(); | auto sd_tmp_dir = dds::temporary_dir::create(); |
try { | try { | ||||
return parse_remote_url(str); | return parse_remote_url(str); | ||||
} catch (const neo::url_validation_error& e) { | } catch (const neo::url_validation_error& e) { | ||||
import_error("{}: Invalid URL: {}", walk.path(), str); | |||||
import_error("{}: Invalid URL [{}]: {}", walk.path(), str, e.what()); | |||||
} catch (const user_error<errc::invalid_remote_url>& e) { | } catch (const user_error<errc::invalid_remote_url>& e) { | ||||
import_error("{}: Invalid URL: {}", walk.path(), e.what()); | import_error("{}: Invalid URL: {}", walk.path(), e.what()); | ||||
} | } |
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <neo/url.hpp> | #include <neo/url.hpp> | ||||
#include <neo/utility.hpp> | |||||
using namespace dds; | using namespace dds; | ||||
dds::remote_listing_var dds::parse_remote_url(std::string_view sv) { | dds::remote_listing_var dds::parse_remote_url(std::string_view sv) { | ||||
neo_assertion_breadcrumbs("Loading package remote from URI string", sv); | |||||
auto url = neo::url::parse(sv); | auto url = neo::url::parse(sv); | ||||
if (url.scheme == "git+https" || url.scheme == "git+http" || url.scheme == "https+git" | |||||
|| url.scheme == "http+git" || url.scheme == "git") { | |||||
if (url.scheme == neo::oper::any_of("git+https", "git+http", "http+git", "https+git", "git")) { | |||||
return git_remote_listing::from_url(sv); | return git_remote_listing::from_url(sv); | ||||
} else if (url.scheme == neo::oper::any_of("http", "https")) { | |||||
return http_remote_listing::from_url(sv); | |||||
} else { | } else { | ||||
throw_user_error< | throw_user_error< | ||||
errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'", | errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'", |
#pragma once | #pragma once | ||||
#include "./remote/git.hpp" | #include "./remote/git.hpp" | ||||
#include "./remote/http.hpp" | |||||
#include <dds/deps.hpp> | #include <dds/deps.hpp> | ||||
#include <dds/package/id.hpp> | #include <dds/package/id.hpp> | ||||
namespace dds { | namespace dds { | ||||
using remote_listing_var = std::variant<std::monostate, git_remote_listing>; | |||||
using remote_listing_var = std::variant<std::monostate, git_remote_listing, http_remote_listing>; | |||||
remote_listing_var parse_remote_url(std::string_view url); | remote_listing_var parse_remote_url(std::string_view url); | ||||
#include "./base.hpp" | |||||
#include <dds/package/id.hpp> | |||||
#include <dds/util/log.hpp> | |||||
#include <nlohmann/json.hpp> | |||||
using namespace dds; | |||||
void remote_listing_base::apply_transforms(path_ref root) const { | |||||
for (const auto& tr : transforms) { | |||||
tr.apply_to(root); | |||||
} | |||||
} | |||||
void remote_listing_base::generate_auto_lib_files(const package_id& pid, path_ref root) const { | |||||
if (auto_lib.has_value()) { | |||||
dds_log(info, "Generating library data automatically"); | |||||
auto pkg_strm = open(root / "package.json5", std::ios::binary | std::ios::out); | |||||
auto man_json = nlohmann::json::object(); | |||||
man_json["name"] = pid.name; | |||||
man_json["version"] = pid.version.to_string(); | |||||
man_json["namespace"] = auto_lib->namespace_; | |||||
pkg_strm << nlohmann::to_string(man_json); | |||||
auto lib_strm = open(root / "library.json5", std::ios::binary | std::ios::out); | |||||
auto lib_json = nlohmann::json::object(); | |||||
lib_json["name"] = auto_lib->name; | |||||
lib_strm << nlohmann::to_string(lib_json); | |||||
} | |||||
} |
#pragma once | |||||
#include <dds/util/fs_transform.hpp> | |||||
#include <libman/package.hpp> | |||||
#include <neo/concepts.hpp> | |||||
#include <optional> | |||||
#include <vector> | |||||
namespace dds { | |||||
struct package_id; | |||||
struct remote_listing_base { | |||||
std::optional<lm::usage> auto_lib{}; | |||||
std::vector<fs_transformation> transforms{}; | |||||
void apply_transforms(path_ref root) const; | |||||
void generate_auto_lib_files(const package_id& pid, path_ref root) const; | |||||
}; | |||||
template <typename T> | |||||
concept remote_listing = neo::derived_from<std::remove_cvref_t<T>, remote_listing_base>; | |||||
} // namespace dds |
#include <neo/url.hpp> | #include <neo/url.hpp> | ||||
#include <neo/url/query.hpp> | #include <neo/url/query.hpp> | ||||
#include <nlohmann/json.hpp> | |||||
using namespace dds; | using namespace dds; | ||||
void git_remote_listing::pull_to(const package_id& pid, path_ref dest) const { | |||||
void git_remote_listing::pull_source(path_ref dest) const { | |||||
fs::remove_all(dest); | fs::remove_all(dest); | ||||
using namespace std::literals; | using namespace std::literals; | ||||
dds_log(info, "Clone Git repository [{}] (at {}) to [{}]", url, ref, dest.string()); | dds_log(info, "Clone Git repository [{}] (at {}) to [{}]", url, ref, dest.string()); | ||||
git_res.retc, | git_res.retc, | ||||
git_res.output); | git_res.output); | ||||
} | } | ||||
for (const auto& tr : transforms) { | |||||
tr.apply_to(dest); | |||||
} | |||||
if (auto_lib.has_value()) { | |||||
dds_log(info, "Generating library data automatically"); | |||||
auto pkg_strm = open(dest / "package.json5", std::ios::binary | std::ios::out); | |||||
auto man_json = nlohmann::json::object(); | |||||
man_json["name"] = pid.name; | |||||
man_json["version"] = pid.version.to_string(); | |||||
man_json["namespace"] = auto_lib->namespace_; | |||||
pkg_strm << nlohmann::to_string(man_json); | |||||
auto lib_strm = open(dest / "library.json5", std::ios::binary | std::ios::out); | |||||
auto lib_json = nlohmann::json::object(); | |||||
lib_json["name"] = auto_lib->name; | |||||
lib_strm << nlohmann::to_string(lib_json); | |||||
} | |||||
} | } | ||||
git_remote_listing git_remote_listing::from_url(std::string_view sv) { | git_remote_listing git_remote_listing::from_url(std::string_view sv) { | ||||
throw_user_error<errc::invalid_remote_url>( | throw_user_error<errc::invalid_remote_url>( | ||||
"Git URL requires a fragment specifying the Git ref to clone"); | "Git URL requires a fragment specifying the Git ref to clone"); | ||||
} | } | ||||
return {.url = url.to_string(), .ref = *ref, .auto_lib = auto_lib, .transforms = {}}; | |||||
return git_remote_listing{ | |||||
{.auto_lib = auto_lib}, | |||||
url.to_string(), | |||||
*ref, | |||||
}; | |||||
} | } |
#pragma once | #pragma once | ||||
#include <dds/catalog/get.hpp> | |||||
#include <dds/util/fs.hpp> | |||||
#include <dds/util/fs_transform.hpp> | |||||
#include "./base.hpp" | |||||
#include <libman/package.hpp> | |||||
#include <optional> | |||||
#include <string> | #include <string> | ||||
#include <string_view> | |||||
namespace dds { | namespace dds { | ||||
struct git_remote_listing { | |||||
std::string url; | |||||
std::string ref; | |||||
std::optional<lm::usage> auto_lib; | |||||
std::vector<fs_transformation> transforms; | |||||
struct git_remote_listing : remote_listing_base { | |||||
std::string url; | |||||
std::string ref; | |||||
void pull_to(const package_id& pid, path_ref path) const; | |||||
void pull_source(path_ref path) const; | |||||
static git_remote_listing from_url(std::string_view sv); | static git_remote_listing from_url(std::string_view sv); | ||||
}; | }; |
res_head.status_message, | res_head.status_message, | ||||
loc->value); | loc->value); | ||||
auto new_url = neo::url::try_parse(loc->value); | auto new_url = neo::url::try_parse(loc->value); | ||||
auto err = std::get_if<neo::url_validation_error>(&new_url); | |||||
auto err = std::get_if<neo::url_parse_error>(&new_url); | |||||
if (err) { | if (err) { | ||||
throw_external_error<errc::http_download_failure>( | throw_external_error<errc::http_download_failure>( | ||||
"Server returned an invalid URL for HTTP redirection [{}]", loc->value); | "Server returned an invalid URL for HTTP redirection [{}]", loc->value); | ||||
} // namespace | } // namespace | ||||
void http_remote_listing::pull_to(path_ref dest) const { | |||||
void http_remote_listing::pull_source(path_ref dest) const { | |||||
neo::url url; | neo::url url; | ||||
try { | try { | ||||
url = neo::url::parse(this->url); | url = neo::url::parse(this->url); | ||||
fs::create_directories(dest); | fs::create_directories(dest); | ||||
dds_log(debug, "Expanding downloaded source distribution into {}", dest.string()); | dds_log(debug, "Expanding downloaded source distribution into {}", dest.string()); | ||||
std::ifstream infile{dl_path, std::ios::binary}; | std::ifstream infile{dl_path, std::ios::binary}; | ||||
neo::expand_directory_targz( | |||||
neo::expand_options{ | |||||
.destination_directory = dest, | |||||
.input_name = dl_path.string(), | |||||
.strip_components = this->strip_components, | |||||
}, | |||||
infile); | |||||
try { | |||||
neo::expand_directory_targz( | |||||
neo::expand_options{ | |||||
.destination_directory = dest, | |||||
.input_name = dl_path.string(), | |||||
.strip_components = this->strip_components, | |||||
}, | |||||
infile); | |||||
} catch (const std::runtime_error& err) { | |||||
throw_external_error<errc::invalid_remote_url>( | |||||
"The file downloaded from [{}] failed to extract (Inner error: {})", | |||||
this->url, | |||||
err.what()); | |||||
} | |||||
} | } | ||||
http_remote_listing http_remote_listing::from_url(std::string_view sv) { | http_remote_listing http_remote_listing::from_url(std::string_view sv) { | ||||
auto url = neo::url::parse(sv); | auto url = neo::url::parse(sv); | ||||
dds_log(trace, "Create HTTP remote listing from URL [{}]", sv); | dds_log(trace, "Create HTTP remote listing from URL [{}]", sv); | ||||
auto q = url.query; | |||||
unsigned strip_components = 0; | |||||
// Because archives most often have one top-level directory, the default strip-components | |||||
// setting is 'one' | |||||
unsigned int strip_components = 1; | |||||
std::optional<lm::usage> auto_lib; | std::optional<lm::usage> auto_lib; | ||||
if (q) { | |||||
neo::basic_query_string_view qsv{*q}; | |||||
if (url.query) { | |||||
neo::basic_query_string_view qsv{*url.query}; | |||||
for (auto qstr : qsv) { | for (auto qstr : qsv) { | ||||
if (qstr.key_raw() == "dds_lm") { | if (qstr.key_raw() == "dds_lm") { | ||||
auto_lib = lm::split_usage_string(qstr.value_decoded()); | auto_lib = lm::split_usage_string(qstr.value_decoded()); | ||||
} | } | ||||
return http_remote_listing{ | return http_remote_listing{ | ||||
.url = url.to_string(), | |||||
.strip_components = strip_components, | |||||
.auto_lib = auto_lib, | |||||
{.auto_lib = auto_lib}, | |||||
url.to_string(), | |||||
strip_components, | |||||
}; | }; | ||||
} | } |
#pragma once | #pragma once | ||||
#include <dds/package/id.hpp> | |||||
#include <dds/util/fs.hpp> | |||||
#include <libman/package.hpp> | |||||
#include "./base.hpp" | |||||
#include <string> | #include <string> | ||||
#include <string_view> | #include <string_view> | ||||
namespace dds { | namespace dds { | ||||
struct http_remote_listing { | |||||
std::string url; | |||||
unsigned strip_components = 0; | |||||
std::optional<lm::usage> auto_lib{}; | |||||
struct http_remote_listing : remote_listing_base { | |||||
std::string url; | |||||
unsigned strip_components = 0; | |||||
void pull_to(path_ref path) const; | |||||
void pull_source(path_ref path) const; | |||||
static http_remote_listing from_url(std::string_view sv); | static http_remote_listing from_url(std::string_view sv); | ||||
}; | }; |
temporary_sdist dds::download_expand_sdist_targz(std::string_view url_str) { | temporary_sdist dds::download_expand_sdist_targz(std::string_view url_str) { | ||||
auto remote = http_remote_listing::from_url(url_str); | auto remote = http_remote_listing::from_url(url_str); | ||||
auto tempdir = temporary_dir::create(); | auto tempdir = temporary_dir::create(); | ||||
remote.pull_to(tempdir.path()); | |||||
remote.pull_source(tempdir.path()); | |||||
return {tempdir, sdist::from_directory(tempdir.path())}; | return {tempdir, sdist::from_directory(tempdir.path())}; | ||||
} | } |
import json | import json | ||||
from contextlib import contextmanager | |||||
from tests import dds, DDS | from tests import dds, DDS | ||||
from tests.fileutil import ensure_dir | from tests.fileutil import ensure_dir | ||||
import pytest | import pytest | ||||
def test_get(dds: DDS): | |||||
def load_catalog(dds: DDS, data): | |||||
dds.scope.enter_context(ensure_dir(dds.build_dir)) | dds.scope.enter_context(ensure_dir(dds.build_dir)) | ||||
dds.catalog_create() | dds.catalog_create() | ||||
json_path = dds.build_dir / 'catalog.json' | json_path = dds.build_dir / 'catalog.json' | ||||
import_data = { | |||||
'version': 2, | |||||
'packages': { | |||||
'neo-sqlite3': { | |||||
'0.3.0': { | |||||
'url': | |||||
'git+https://github.com/vector-of-bool/neo-sqlite3.git#0.3.0', | |||||
}, | |||||
}, | |||||
}, | |||||
} | |||||
dds.scope.enter_context( | dds.scope.enter_context( | ||||
dds.set_contents(json_path, | dds.set_contents(json_path, | ||||
json.dumps(import_data).encode())) | |||||
json.dumps(data).encode())) | |||||
dds.catalog_import(json_path) | dds.catalog_import(json_path) | ||||
def test_get(dds: DDS): | |||||
load_catalog( | |||||
dds, { | |||||
'version': 2, | |||||
'packages': { | |||||
'neo-sqlite3': { | |||||
'0.3.0': { | |||||
'url': | |||||
'git+https://github.com/vector-of-bool/neo-sqlite3.git#0.3.0', | |||||
}, | |||||
}, | |||||
}, | |||||
}) | |||||
dds.catalog_get('neo-sqlite3@0.3.0') | dds.catalog_get('neo-sqlite3@0.3.0') | ||||
assert (dds.source_root / 'neo-sqlite3@0.3.0').is_dir() | |||||
assert (dds.source_root / 'neo-sqlite3@0.3.0/package.jsonc').is_file() | |||||
assert (dds.scratch_dir / 'neo-sqlite3@0.3.0').is_dir() | |||||
assert (dds.scratch_dir / 'neo-sqlite3@0.3.0/package.jsonc').is_file() | |||||
def test_get_http(dds: DDS): | |||||
load_catalog( | |||||
dds, { | |||||
'version': 2, | |||||
'packages': { | |||||
'cmcstl2': { | |||||
'2020.2.24': { | |||||
'url': | |||||
'https://github.com/CaseyCarter/cmcstl2/archive/684a96d527e4dc733897255c0177b784dc280980.tar.gz?dds_lm=cmc/stl2;', | |||||
}, | |||||
}, | |||||
}, | |||||
}) | |||||
dds.catalog_get('cmcstl2@2020.2.24') | |||||
assert dds.scratch_dir.joinpath('cmcstl2@2020.2.24/include').is_dir() |
'url': 'git+http://example.com#master', | 'url': 'git+http://example.com#master', | ||||
}, | }, | ||||
}, | }, | ||||
'bar': { | |||||
'1.5.1': { | |||||
'url': 'http://example.com/bar-1.5.2.tgz' | |||||
}, | |||||
} | |||||
}, | }, | ||||
} | } | ||||
dds.scope.enter_context( | dds.scope.enter_context( |
'catalog', | 'catalog', | ||||
'get', | 'get', | ||||
f'--catalog={self.catalog_path}', | f'--catalog={self.catalog_path}', | ||||
f'--out-dir={self.scratch_dir}', | |||||
req, | req, | ||||
]) | ]) | ||||
], | ], | ||||
"link_flags": [ | "link_flags": [ | ||||
"-fuse-ld=lld", | "-fuse-ld=lld", | ||||
"-fsanitize=address,undefined" | |||||
"-fsanitize=address,undefined", | |||||
"-l:libssl.a", | |||||
"-l:libcrypto.a", | |||||
"-ldl", | |||||
], | ], | ||||
"debug": true, | "debug": true, | ||||
"compiler_launcher": "ccache" | "compiler_launcher": "ccache" |
{ | |||||
"$schema": "../res/toolchain-schema.json", | |||||
"compiler_id": "gnu", | |||||
"c_compiler": "gcc-9", | |||||
"cxx_compiler": "g++-9", | |||||
"warning_flags": [ | |||||
"-Werror", | |||||
], | |||||
"flags": [ | |||||
"-I/usr/local/opt/openssl@1.1/include", | |||||
], | |||||
"cxx_flags": [ | |||||
"-fconcepts", | |||||
"-std=c++2a", | |||||
], | |||||
"link_flags": [ | |||||
"-static-libgcc", | |||||
"-static-libstdc++", | |||||
"/usr/local/opt/openssl@1.1/lib/libssl.a", | |||||
"/usr/local/opt/openssl@1.1/lib/libcrypto.a", | |||||
], | |||||
"optimize": true | |||||
} |
], | ], | ||||
"link_flags": [ | "link_flags": [ | ||||
"-static-libgcc", | "-static-libgcc", | ||||
"-static-libstdc++" | |||||
"-static-libstdc++", | |||||
"-l:libssl.a", | |||||
"-l:libcrypto.a", | |||||
"-ldl", | |||||
], | ], | ||||
"optimize": true | "optimize": true | ||||
} | } |
"-std=c++2a", | "-std=c++2a", | ||||
], | ], | ||||
"link_flags": [ | "link_flags": [ | ||||
"-static" | |||||
"-static", | |||||
"-l:libssl.a", | |||||
"-l:libcrypto.a", | |||||
"-ldl", | |||||
// WORKAROUND: https://sourceware.org/legacy-ml/glibc-bugs/2018-09/msg00009.html | |||||
"-Wl,-u,pthread_mutex_lock,-u,pthread_mutex_unlock,-u,pthread_self", | |||||
], | ], | ||||
"optimize": true | "optimize": true | ||||
} | } |
github_package('neo-buffer', 'vector-of-bool/neo-buffer', | github_package('neo-buffer', 'vector-of-bool/neo-buffer', | ||||
['0.2.1', '0.3.0', '0.4.0', '0.4.1', '0.4.2']), | ['0.2.1', '0.3.0', '0.4.0', '0.4.1', '0.4.2']), | ||||
github_package('neo-compress', 'vector-of-bool/neo-compress', ['0.1.0', '0.1.1']), | github_package('neo-compress', 'vector-of-bool/neo-compress', ['0.1.0', '0.1.1']), | ||||
github_package('neo-url', 'vector-of-bool/neo-url', ['0.1.0', '0.1.1', '0.1.2']), | |||||
github_package('neo-url', 'vector-of-bool/neo-url', ['0.1.0', '0.1.1', '0.1.2', '0.2.0', '0.2.1']), | |||||
github_package('neo-sqlite3', 'vector-of-bool/neo-sqlite3', | github_package('neo-sqlite3', 'vector-of-bool/neo-sqlite3', | ||||
['0.2.3', '0.3.0', '0.4.0', '0.4.1']), | ['0.2.3', '0.3.0', '0.4.0', '0.4.1']), | ||||
github_package('neo-fun', 'vector-of-bool/neo-fun', [ | github_package('neo-fun', 'vector-of-bool/neo-fun', [ | ||||
'0.1.1', '0.2.0', '0.2.1', '0.3.0', '0.3.1', '0.3.2', '0.4.0', '0.4.1', | '0.1.1', '0.2.0', '0.2.1', '0.3.0', '0.3.1', '0.3.2', '0.4.0', '0.4.1', | ||||
'0.4.2', '0.5.0', '0.5.1', '0.5.2', '0.5.3', '0.5.4', | |||||
'0.4.2', '0.5.0', '0.5.1', '0.5.2', '0.5.3', '0.5.4', '0.5.5', | |||||
]), | ]), | ||||
github_package('neo-io', 'vector-of-bool/neo-io', ['0.1.0']), | github_package('neo-io', 'vector-of-bool/neo-io', ['0.1.0']), | ||||
github_package('neo-http', 'vector-of-bool/neo-http', ['0.1.0']), | github_package('neo-http', 'vector-of-bool/neo-http', ['0.1.0']), |
[CmdletBinding()] | |||||
param () | |||||
$ErrorActionPreference = "Stop" | |||||
$this_script = $MyInvocation.MyCommand.Definition | |||||
$tools_dir = Split-Path -Parent $this_script | |||||
$root_dir = Split-Path -Parent $tools_dir | |||||
$build_dir = Join-Path $root_dir "_build" | |||||
New-Item -ItemType Container $build_dir -ErrorAction Ignore | |||||
$local_tgz = Join-Path $build_dir "openssl.tgz" | |||||
# This is the path to the release static vs2019 x64 build of OpenSSL in bintray | |||||
$conan_ssl_path = "_/openssl/1.1.1h/_/7098aea4e4f2247cc9b5dcaaa1ebddbe/package/a79a557254fabcb77849dd623fed97c9c5ab7651/141ef2c6711a254707ba1f7f4fd07ad4" | |||||
$openssl_url = "https://dl.bintray.com/conan/conan-center/$conan_ssl_path/conan_package.tgz" | |||||
Write-Host "Downloading OpenSSL for Windows" | |||||
Invoke-WebRequest ` | |||||
-Uri $openssl_url ` | |||||
-OutFile $local_tgz | |||||
$openssl_tree = Join-Path $root_dir "external/OpenSSL" | |||||
Write-Host "Expanding OpenSSL archive..." | |||||
Remove-Item $openssl_tree -Recurse -Force -ErrorAction Ignore | |||||
New-Item $openssl_tree -ItemType Container | Out-Null | |||||
& cmake -E chdir $openssl_tree cmake -E tar xf $local_tgz | |||||
if ($LASTEXITCODE) { | |||||
throw "Archive expansion failed" | |||||
} |
"/Zc:preprocessor", | "/Zc:preprocessor", | ||||
"/std:c++latest", | "/std:c++latest", | ||||
"/DNOMINMAX", | "/DNOMINMAX", | ||||
// OpenSSL headers: | |||||
"/Iexternal/OpenSSL/include", | |||||
], | ], | ||||
"link_flags": [ | "link_flags": [ | ||||
"rpcrt4.lib", | "rpcrt4.lib", | ||||
// Networking: | |||||
"Ws2_32.lib", | |||||
// Deps for OpenSSL: | |||||
"AdvApi32.lib", | |||||
"Crypt32.lib", | |||||
"User32.lib", | |||||
// Link in our external OpenSSL: | |||||
"/link", | |||||
"/LibPath:external/OpenSSL/lib", | |||||
"libssl.lib", | |||||
"libcrypto.lib", | |||||
], | ], | ||||
// "debug": true, | // "debug": true, | ||||
"optimize": true | "optimize": true |