visus
/
dds
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

Dependency downloading using `dds deps get`

default_compile_flags
vector-of-bool 5 years ago
parent
675f8d365a
commit
69c7fd2112
10 changed files with 390 additions and 109 deletions
Split View Show Diff Stats
  1. +4
    -4
      src/dds/build.cpp
  2. +55
    -4
      src/dds/dds.main.cpp
  3. +4
    -32
      src/dds/deps.cpp
  4. +4
    -7
      src/dds/deps.hpp
  5. +141
    -0
      src/dds/repo/remote.cpp
  6. +73
    -0
      src/dds/repo/remote.hpp
  7. +51
    -44
      src/dds/repo/repo.cpp
  8. +36
    -14
      src/dds/repo/repo.hpp
  9. +18
    -0
      src/dds/sdist.hpp
  10. +4
    -4
      src/libman/parse.cpp

+ 4
- 4
src/dds/build.cpp View File

@@ -112,10 +112,10 @@ load_usage_requirements(path_ref project_root, path_ref build_root, path_ref use
return usage_requirement_map::from_lm_index(idx);
}

void prepare_catch2_driver(library_build_params& lib_params,
test_lib test_driver,
const build_params& params,
const package_manifest& man) {
void prepare_catch2_driver(library_build_params& lib_params,
test_lib test_driver,
const build_params& params,
const package_manifest&) {
fs::path test_include_root = params.out_root / "_test_inc";
lib_params.test_include_dirs.emplace_back(test_include_root);


+ 55
- 4
src/dds/dds.main.cpp View File

@@ -1,5 +1,6 @@
#include <dds/build.hpp>
#include <dds/logging.hpp>
#include <dds/repo/remote.hpp>
#include <dds/repo/repo.hpp>
#include <dds/sdist.hpp>
#include <dds/toolchain/from_dds.hpp>
@@ -120,8 +121,8 @@ struct cli_repo {
auto same_name
= [](auto&& a, auto&& b) { return a.manifest.name == b.manifest.name; };

auto all_sdists = repo.load_sdists();
auto grp_by_name = all_sdists //
auto all = repo.iter_sdists();
auto grp_by_name = all //
| ranges::views::group_by(same_name) //
| ranges::views::transform(ranges::to_vector) //
| ranges::views::transform([](auto&& grp) {
@@ -132,8 +133,7 @@ struct cli_repo {
for (const auto& [name, grp] : grp_by_name) {
spdlog::info("{}:", name);
for (const dds::sdist& sd : grp) {
spdlog::info(" - {}",
sd.manifest.version.to_string());
spdlog::info(" - {}", sd.manifest.version.to_string());
}
}

@@ -381,6 +381,55 @@ struct cli_deps {
}
} ls{*this};

struct {
cli_deps& parent;
args::Command cmd{parent.deps_group,
"get",
"Ensure we have local copies of the project dependencies"};
common_flags _common{cmd};

repo_where_flag repo_where{cmd};
path_flag remote_listing_file{
cmd,
"remote-listing",
"Path to a file containing listing of remote sdists and how to obtain them",
{'R', "remote-list"},
"remote.dds"};

int run() {
auto man = parent.load_package_manifest();
auto rd = dds::remote_directory::load_from_file(remote_listing_file.Get());
bool failed = false;
dds::repository::with_repository( //
repo_where.Get(),
dds::repo_flags::write_lock | dds::repo_flags::create_if_absent,
[&](dds::repository repo) {
for (auto& dep : man.dependencies) {
auto exists = !!repo.find(dep.name, dep.version);
if (!exists) {
spdlog::info("Pull remote: {} {}", dep.name, dep.version.to_string());
auto opt_remote = rd.find(dep.name, dep.version);
if (opt_remote) {
auto tsd = opt_remote->pull_sdist();
repo.add_sdist(tsd.sdist, dds::if_exists::ignore);
} else {
spdlog::error("No remote listing for {} {}",
dep.name,
dep.version.to_string());
failed = true;
}
} else {
spdlog::info("Okay: {} {}", dep.name, dep.version.to_string());
}
}
});
if (failed) {
return 1;
}
return 0;
}
} get{*this};

struct {
cli_deps& parent;
args::Command cmd{parent.deps_group, "build", "Build project dependencies"};
@@ -435,6 +484,8 @@ struct cli_deps {
return ls.run();
} else if (build.cmd) {
return build.run();
} else if (get.cmd) {
return get.run();
}
std::terminate();
}

+ 4
- 32
src/dds/deps.cpp View File

@@ -43,41 +43,13 @@ dependency dependency::parse_depends_string(std::string_view str) {
}

std::vector<sdist> dds::find_dependencies(const repository& repo, const dependency& dep) {
auto all_dists = repo.load_sdists();
detail::sort_sdists(all_dists);
std::vector<sdist> acc;
detail::do_find_deps(all_dists, dep, acc);
detail::do_find_deps(repo, dep, acc);
return acc;
}

auto tie_sdist(const sdist& sd) {
return std::tuple(sd.manifest.name, sd.manifest.version.to_string());
}

auto sdist_compare
= [](const sdist& lhs, const sdist& rhs) { return tie_sdist(lhs) < tie_sdist(rhs); };

void detail::sort_sdists(std::vector<sdist>& sd) { std::sort(sd.begin(), sd.end(), sdist_compare); }

namespace {

const sdist*
get_sdist(const std::vector<sdist>& sorted_sds, std::string_view name, std::string_view version) {
auto found
= std::partition_point(sorted_sds.begin(), sorted_sds.end(), [&](const auto& candidate) {
return tie_sdist(candidate) < std::tie(name, version);
});
if (found->manifest.name == name && found->manifest.version.to_string() == version) {
return &*found;
}
return nullptr;
}
} // namespace

void detail::do_find_deps(const std::vector<sdist>& sdists,
const dependency& dep,
std::vector<sdist>& sd) {
auto sdist_opt = get_sdist(sdists, dep.name, dep.version.to_string());
void detail::do_find_deps(const repository& repo, const dependency& dep, std::vector<sdist>& sd) {
auto sdist_opt = repo.find(dep.name, dep.version);
if (!sdist_opt) {
throw std::runtime_error(
fmt::format("Unable to find dependency to satisfy requirement: {} {}",
@@ -86,7 +58,7 @@ void detail::do_find_deps(const std::vector<sdist>& sdists,
}
const sdist& new_sd = *sdist_opt;
for (const auto& inner_dep : new_sd.manifest.dependencies) {
do_find_deps(sdists, inner_dep, sd);
do_find_deps(repo, inner_dep, sd);
}
auto insert_point = std::partition_point(sd.begin(), sd.end(), [&](const sdist& cand) {
return cand.path < new_sd.path;

+ 4
- 7
src/dds/deps.hpp View File

@@ -1,7 +1,6 @@
#pragma once

#include <dds/build/plan/full.hpp>
#include <dds/repo/repo.hpp>

#include <semver/version.hpp>

@@ -10,6 +9,7 @@
namespace dds {

struct sdist;
class repository;

enum class version_strength {
exact,
@@ -27,8 +27,7 @@ struct dependency {

namespace detail {

void do_find_deps(const std::vector<sdist>&, const dependency& dep, std::vector<sdist>& acc);
void sort_sdists(std::vector<sdist>& sds);
void do_find_deps(const repository&, const dependency& dep, std::vector<sdist>& acc);

} // namespace detail

@@ -37,10 +36,8 @@ std::vector<sdist> find_dependencies(const repository& repo, const dependency& d
template <typename Iter, typename Snt>
inline std::vector<sdist> find_dependencies(const repository& repo, Iter it, Snt stop) {
std::vector<sdist> acc;
auto all_sds = repo.load_sdists();
detail::sort_sdists(all_sds);
while (it != stop) {
detail::do_find_deps(all_sds, *it++, acc);
detail::do_find_deps(repo, *it++, acc);
}
return acc;
}
@@ -49,4 +46,4 @@ build_plan create_deps_build_plan(const std::vector<sdist>& deps, build_env_ref

void write_libman_index(path_ref where, const build_plan& plan, const build_env& env);

} // namespace dds
} // namespace dds

+ 141
- 0
src/dds/repo/remote.cpp View File

@@ -0,0 +1,141 @@
#include "./remote.hpp"

#include <dds/deps.hpp>
#include <dds/proc.hpp>
#include <dds/repo/repo.hpp>
#include <dds/sdist.hpp>
#include <dds/temp.hpp>
#include <dds/toolchain/toolchain.hpp>

#include <spdlog/spdlog.h>

#include <libman/parse.hpp>

#include <algorithm>

using namespace dds;

namespace {

struct read_listing_item {
std::string_view _key;
std::set<remote_listing, remote_listing_compare_t>& out;

bool operator()(std::string_view context, std::string_view key, std::string_view value) {
if (key != _key) {
return false;
}

auto nested = lm::nested_kvlist::parse(value);
auto name_ver_pair = split_shell_string(nested.primary);
if (name_ver_pair.size() != 2) {
throw std::runtime_error(
fmt::format("{}: Invalid Remote-Package identity: '{}'", context, nested.primary));
}
auto name = name_ver_pair[0];
auto version = semver::version::parse(name_ver_pair[1]);
put_listing(context, name, version, nested.pairs);
return true;
}

void put_listing(std::string_view context,
std::string name,
semver::version version,
const lm::pair_list& pairs) {
if (pairs.find("git")) {
std::string url;
std::string ref;
lm::read(fmt::format("{}: Parsing Git remote listing", context),
pairs,
lm::read_required("url", url),
lm::read_required("ref", ref),
lm::read_check_eq("git", ""),
lm::reject_unknown());
auto did_insert
= out.emplace(
remote_listing{std::move(name), version, git_remote_listing{url, ref}})
.second;
if (!did_insert) {
spdlog::warn("Duplicate remote package defintion for {} {}",
name,
version.to_string());
}
} else {
throw std::runtime_error(fmt::format("Unable to determine remote type of package {} {}",
name,
version.to_string()));
}
}
};

temporary_sdist do_pull_sdist(const git_remote_listing& git) {
auto tmpdir = dds::temporary_dir::create();
using namespace std::literals;
spdlog::info("Cloning repository: {} [{}] ...", git.url, git.ref);
auto command = {"git"s,
"clone"s,
"--depth=1"s,
"--branch"s,
git.ref,
git.url,
tmpdir.path().generic_string()};
auto git_res = run_proc(command);
if (!git_res.okay()) {
throw std::runtime_error(
fmt::format("Git clone operation failed [Git command: {}] [Exitted {}]:\n{}",
quote_command(command),
git_res.retc,
git_res.output));
}
spdlog::info("Create sdist from clone ...");
sdist_params params;
params.project_dir = tmpdir.path();
auto sd_tmp_dir = dds::temporary_dir::create();
params.dest_path = sd_tmp_dir.path();
auto sd = create_sdist(params);
return {sd_tmp_dir, sd};
}

} // namespace

temporary_sdist remote_listing::pull_sdist() const {
auto tsd = visit([](auto&& actual) { return do_pull_sdist(actual); });
if (tsd.sdist.manifest.name != name) {
throw std::runtime_error(
fmt::format("The name in the generated sdist ('{}') does not match the name listed in "
"the remote listing file (expected '{}')",
tsd.sdist.manifest.name,
name));
}
if (tsd.sdist.manifest.version != version) {
throw std::runtime_error(
fmt::format("The version of the generated sdist is '{}', which does not match the "
"expected version '{}'",
tsd.sdist.manifest.version.to_string(),
version.to_string()));
}
return tsd;
}

remote_directory remote_directory::load_from_file(path_ref filepath) {
auto kvs = lm::parse_file(filepath);
listing_set listings;
lm::read(fmt::format("Loading remote package listing from {}", filepath.string()),
kvs,
read_listing_item{"Remote-Package", listings},
lm::reject_unknown());
return {std::move(listings)};
}

const remote_listing* remote_directory::find(std::string_view name, semver::version ver) const
noexcept {
auto found = _remotes.find(std::tie(name, ver));
if (found == _remotes.end()) {
return nullptr;
}
return &*found;
}

void remote_directory::ensure_all_local(const repository&) const {
spdlog::critical("Dependency download is not fully implemented!");
}

+ 73
- 0
src/dds/repo/remote.hpp View File

@@ -0,0 +1,73 @@
#pragma once

#include <dds/util/fs.hpp>

#include <dds/sdist.hpp>
#include <dds/temp.hpp>
#include <semver/version.hpp>

#include <set>
#include <string>
#include <tuple>
#include <utility>
#include <variant>

namespace dds {

struct temporary_sdist {
temporary_dir tmpdir;
dds::sdist sdist;
};

struct git_remote_listing {
std::string url;
std::string ref;

void clone(path_ref path) const;
};

struct remote_listing {
std::string name;
semver::version version;
std::variant<git_remote_listing> remote;

template <typename Func>
decltype(auto) visit(Func&& fn) const {
return std::visit(std::forward<Func>(fn), remote);
}

temporary_sdist pull_sdist() const;
};

inline constexpr struct remote_listing_compare_t {
using is_transparent = int;
auto tie(const remote_listing& rl) const { return std::tie(rl.name, rl.version); }
bool operator()(const remote_listing& lhs, const remote_listing& rhs) const {
return tie(lhs) < tie(rhs);
}
template <typename Name, typename Version>
bool operator()(const remote_listing& lhs, const std::tuple<Name, Version>& rhs) const {
return tie(lhs) < rhs;
}
template <typename Name, typename Version>
bool operator()(const std::tuple<Name, Version>& lhs, const remote_listing& rhs) const {
return lhs < tie(rhs);
}
} remote_listing_compare;

class remote_directory {
using listing_set = std::set<remote_listing, remote_listing_compare_t>;
listing_set _remotes;

remote_directory(listing_set s)
: _remotes(std::move(s)) {}

public:
static remote_directory load_from_file(path_ref);

void ensure_all_local(const class repository& repo) const;

const remote_listing* find(std::string_view name, semver::version ver) const noexcept;
};

} // namespace dds

+ 51
- 44
src/dds/repo/repo.cpp View File

@@ -6,6 +6,7 @@

#include <spdlog/spdlog.h>

#include <range/v3/range/conversion.hpp>
#include <range/v3/view/filter.hpp>
#include <range/v3/view/transform.hpp>

@@ -13,26 +14,63 @@ using namespace dds;

using namespace ranges;

namespace {

auto load_sdists(path_ref root) {
using namespace ranges;
using namespace ranges::views;

auto try_read_sdist = [](path_ref p) -> std::optional<sdist> {
if (starts_with(p.filename().string(), ".")) {
return std::nullopt;
}
try {
return sdist::from_directory(p);
} catch (const std::runtime_error& e) {
spdlog::error("Failed to load source distribution from directory '{}': {}",
p.string(),
e.what());
return std::nullopt;
}
};

return
// Get the top-level `name-version` dirs
fs::directory_iterator(root) //
// // Convert each dir into an `sdist` object
| transform(try_read_sdist) //
// // Drop items that failed to load
| filter([](auto&& opt) { return opt.has_value(); }) //
| transform([](auto&& opt) { return *opt; }) //
;
}

} // namespace

void repository::_log_blocking(path_ref dirpath) noexcept {
spdlog::warn("Another process has the repository directory locked [{}]", dirpath.string());
spdlog::warn("Waiting for repository to be released...");
}

void repository::_init_repo_dir(path_ref dirpath) noexcept {
fs::create_directories(dirpath);
}
void repository::_init_repo_dir(path_ref dirpath) noexcept { fs::create_directories(dirpath); }

fs::path repository::default_local_path() noexcept { return dds_data_dir() / "repo"; }

repository repository::open_for_directory(path_ref dirpath) {
repository repository::_open_for_directory(bool writeable, path_ref dirpath) {
auto dist_dir = dirpath;
auto entries = fs::directory_iterator(dist_dir) | to_vector;
return {dirpath};
auto entries = load_sdists(dirpath) | to<sdist_set>;
return {writeable, dirpath, std::move(entries)};
}

void repository::add_sdist(const sdist& sd, if_exists ife_action) {
auto sd_dest
= _root / fmt::format("{}_{}", sd.manifest.name, sd.manifest.version.to_string());
if (!_write_enabled) {
spdlog::critical(
"DDS attempted to write into a repository that wasn't opened with a write-lock. This "
"is a hard bug and should be reported. For the safety and integrity of the local "
"repository, we'll hard-exit immediately.");
std::terminate();
}
auto sd_dest = _root / fmt::format("{}_{}", sd.manifest.name, sd.manifest.version.to_string());
if (fs::exists(sd_dest)) {
auto msg = fmt::format("Source distribution '{}' is already available in the local repo",
sd.path.string());
@@ -59,41 +97,10 @@ void repository::add_sdist(const sdist& sd, if_exists ife_action) {
spdlog::info("Source distribution '{}' successfully exported", sd.ident());
}

std::vector<sdist> repository::load_sdists() const {
using namespace ranges;
using namespace ranges::views;

auto try_read_sdist = [](path_ref p) -> std::optional<sdist> {
if (starts_with(p.filename().string(), ".")) {
return std::nullopt;
}
try {
return sdist::from_directory(p);
} catch (const std::runtime_error& e) {
spdlog::error("Failed to load source distribution from directory '{}': {}",
p.string(),
e.what());
return std::nullopt;
}
};

return
// Get the top-level `name-version` dirs
fs::directory_iterator(_root) //
// // Convert each dir into an `sdist` object
| transform(try_read_sdist) //
// // Drop items that failed to load
| filter([](auto&& opt) { return opt.has_value(); }) //
| transform([](auto&& opt) { return *opt; }) //
| to_vector //
;
}

std::optional<sdist> repository::get_sdist(std::string_view name, std::string_view version) const {
auto expect_path = _root / fmt::format("{}_{}", name, version);
if (!fs::is_directory(expect_path)) {
return std::nullopt;
const sdist* repository::find(std::string_view name, semver::version ver) const noexcept {
auto found = _sdists.find(std::tie(name, ver));
if (found == _sdists.end()) {
return nullptr;
}

return sdist::from_directory(expect_path);
return &*found;
}

+ 36
- 14
src/dds/repo/repo.hpp View File

@@ -1,17 +1,17 @@
#pragma once

#include <dds/sdist.hpp>
#include <dds/util/flock.hpp>
#include <dds/util/fs.hpp>

#include <functional>
#include <optional>
#include <set>
#include <shared_mutex>
#include <vector>

namespace dds {

struct sdist;

enum repo_flags {
none = 0b00,
read = none,
@@ -30,13 +30,20 @@ inline repo_flags operator|(repo_flags a, repo_flags b) {
}

class repository {
fs::path _root;
using sdist_set = std::set<sdist, sdist_compare_t>;

bool _write_enabled = false;
fs::path _root;
sdist_set _sdists;

repository(path_ref p)
: _root(p) {}
repository(bool writeable, path_ref p, sdist_set sds)
: _write_enabled(writeable)
, _root(p)
, _sdists(std::move(sds)) {}

static void _log_blocking(path_ref dir) noexcept;
static void _init_repo_dir(path_ref dir) noexcept;
static void _log_blocking(path_ref dir) noexcept;
static void _init_repo_dir(path_ref dir) noexcept;
static repository _open_for_directory(bool writeable, path_ref);

public:
template <typename Func>
@@ -51,7 +58,9 @@ public:
std::shared_lock shared_lk{mut, std::defer_lock};
std::unique_lock excl_lk{mut, std::defer_lock};

if (flags & repo_flags::write_lock) {
bool writeable = (flags & repo_flags::write_lock) != repo_flags::none;

if (writeable) {
if (!excl_lk.try_lock()) {
_log_blocking(dirpath);
excl_lk.lock();
@@ -63,16 +72,29 @@ public:
}
}

return std::invoke((Func &&) fn, open_for_directory(dirpath));
auto repo = _open_for_directory(writeable, dirpath);
return std::invoke((Func &&) fn, std::move(repo));
}

static repository open_for_directory(path_ref);

static fs::path default_local_path() noexcept;

void add_sdist(const sdist&, if_exists = if_exists::throw_exc);
std::optional<sdist> get_sdist(std::string_view name, std::string_view version) const;
std::vector<sdist> load_sdists() const;
void add_sdist(const sdist&, if_exists = if_exists::throw_exc);

const sdist* find(std::string_view name, semver::version ver) const noexcept;

auto iter_sdists() const noexcept {
class ret {
const sdist_set& s;

public:
ret(const sdist_set& s)
: s(s) {}

auto begin() const { return s.cbegin(); }
auto end() const { return s.cend(); }
} r{_sdists};
return r;
}
};

} // namespace dds

+ 18
- 0
src/dds/sdist.hpp View File

@@ -1,5 +1,7 @@
#pragma once

#include <tuple>

#include <dds/package_manifest.hpp>
#include <dds/util/fs.hpp>

@@ -28,6 +30,22 @@ struct sdist {
}
};

inline constexpr struct sdist_compare_t {
bool operator()(const sdist& lhs, const sdist& rhs) const {
return std::tie(lhs.manifest.name, lhs.manifest.version)
< std::tie(rhs.manifest.name, rhs.manifest.version);
}
template <typename Name, typename Version>
bool operator()(const sdist& lhs, const std::tuple<Name, Version>& rhs) const {
return std::tie(lhs.manifest.name, lhs.manifest.version) < rhs;
}
template <typename Name, typename Version>
bool operator()(const std::tuple<Name, Version>& lhs, const sdist& rhs) const {
return lhs < std::tie(rhs.manifest.name, rhs.manifest.version);
}
using is_transparent = int;
} sdist_compare;

sdist create_sdist(const sdist_params&);
sdist create_sdist_in_dir(path_ref, const sdist_params&);


+ 4
- 4
src/libman/parse.cpp View File

@@ -84,10 +84,10 @@ void lm::write_pairs(fs::path fpath, const std::vector<pair>& pairs) {
}

nested_kvlist nested_kvlist::parse(const std::string_view line_) {
const auto line = trim(line_);
const auto line = trim_view(line_);
const auto semi_pos = line.find(';');
const auto primary = trim(line.substr(0, semi_pos));
auto tail = semi_pos == line.npos ? ""sv : trim(line.substr(semi_pos + 1));
const auto primary = trim_view(line.substr(0, semi_pos));
auto tail = semi_pos == line.npos ? ""sv : trim_view(line.substr(semi_pos + 1));

std::vector<pair> pairs;
while (!tail.empty()) {
@@ -106,7 +106,7 @@ nested_kvlist nested_kvlist::parse(const std::string_view line_) {
if (space_pos == tail.npos) {
break;
}
tail = trim(tail.substr(space_pos + 1));
tail = trim_view(tail.substr(space_pos + 1));
}

return nested_kvlist{std::string(primary), pair_list{std::move(pairs)}};

Write Preview
Loading…
Cancel
Save