Procházet zdrojové kódy

Merge branch 'feature/version-ranges' into develop

default_compile_flags
vector-of-bool před 5 roky
rodič
revize
5bace8b3c3
16 změnil soubory, kde provedl 265 přidání a 119 odebrání
  1. +2
    -1
      library.dds
  2. +3
    -2
      package.dds
  3. +3
    -1
      remote.dds
  4. +23
    -20
      src/dds/catalog/catalog.cpp
  5. +4
    -2
      src/dds/catalog/catalog.hpp
  6. +42
    -4
      src/dds/catalog/catalog.test.cpp
  7. +14
    -17
      src/dds/dds.main.cpp
  8. +9
    -41
      src/dds/deps.cpp
  9. +6
    -19
      src/dds/deps.hpp
  10. +31
    -2
      src/dds/repo/repo.cpp
  11. +3
    -1
      src/dds/repo/repo.hpp
  12. +4
    -8
      src/dds/sdist.hpp
  13. +102
    -0
      src/dds/solve/solve.cpp
  14. +16
    -0
      src/dds/solve/solve.hpp
  15. +1
    -0
      src/libman/parse.hpp
  16. +2
    -1
      tools/gcc-9.dds

+ 2
- 1
library.dds Zobrazit soubor

@@ -6,4 +6,5 @@ Uses: Niebler/range-v3
Uses: nlohmann/json
Uses: neo/buffer
Uses: neo/sqlite3
Uses: semver/semver
Uses: semver/semver
Uses: pubgrub/pubgrub

+ 3
- 2
package.dds Zobrazit soubor

@@ -4,9 +4,10 @@ Version: 0.1.0
Depends: neo-buffer 0.1.0
Depends: spdlog 1.4.2
Depends: ms-wil 2019.11.10
Depends: range-v3 0.9.1
Depends: range-v3 0.10.0
Depends: nlohmann-json 3.7.1
Depends: neo-sqlite3 0.2.2
Depends: semver 0.2.0
Depends: semver 0.2.1
Depends: pubgrub 0.1.2

Test-Driver: Catch-Main

+ 3
- 1
remote.dds Zobrazit soubor

@@ -1,4 +1,5 @@
Remote-Package: range-v3 0.9.1; git url=https://github.com/ericniebler/range-v3.git ref=0.9.1 auto=Niebler/range-v3
Remote-Package: range-v3 0.10.0; git url=https://github.com/ericniebler/range-v3.git ref=0.10.0 auto=Niebler/range-v3
Remote-Package: spdlog 1.4.2; git url=https://github.com/gabime/spdlog.git ref=v1.4.2 auto=spdlog/spdlog

# Even a shallow clone of nlohmann-json is HUGE. This fork has only the minimal
@@ -11,4 +12,5 @@ Remote-Package: ms-wil 2019.11.10; git url=https://github.com/vector-of-bool/wil
Remote-Package: neo-buffer 0.1.0; git url=https://github.com/vector-of-bool/neo-buffer.git ref=develop

Remote-Package: neo-sqlite3 0.2.2; git url=https://github.com/vector-of-bool/neo-sqlite3.git ref=0.2.2
Remote-Package: semver 0.2.0; git url=https://github.com/vector-of-bool/semver.git ref=0.2.0
Remote-Package: semver 0.2.1; git url=https://github.com/vector-of-bool/semver.git ref=0.2.1
Remote-Package: pubgrub 0.1.2; git url=https://github.com/vector-of-bool/pubgrub.git ref=0.1.2

+ 23
- 20
src/dds/catalog/catalog.cpp Zobrazit soubor

@@ -1,5 +1,7 @@
#include "./catalog.hpp"

#include <dds/solve/solve.hpp>

#include <neo/sqlite3/exec.hpp>
#include <neo/sqlite3/iter_tuples.hpp>
#include <neo/sqlite3/single.hpp>
@@ -157,11 +159,16 @@ void catalog::store(const package_info& pkg) {
)"_sql);
for (const auto& dep : pkg.deps) {
new_dep_st.reset();
if (dep.versions.num_intervals() != 1) {
throw std::runtime_error(
"Package dependency may only contain a single version interval");
}
auto iv_1 = *dep.versions.iter_intervals().begin();
sqlite3::exec(new_dep_st,
std::forward_as_tuple(db_pkg_id,
dep.name,
dep.version.to_string(),
"[placeholder]"));
iv_1.low.to_string(),
iv_1.high.to_string()));
}
}

@@ -196,23 +203,11 @@ std::optional<package_info> catalog::get(const package_id& pk_id) const noexcept
assert(git_url);
assert(git_ref);

auto deps = sqlite3::exec_iter<std::string, std::string>( //
_stmt_cache,
R"(
SELECT dep_name, low
FROM dds_cat_pkg_deps
WHERE pkg_id = ?
)"_sql,
std::tie(pkg_id))
| ranges::views::transform([](auto&& pair) {
const auto& [name, ver] = pair;
return dependency{name, semver::version::parse(ver)};
}) //
| ranges::to_vector;
auto deps = dependencies_of(pk_id);

return package_info{
pk_id,
deps,
std::move(deps),
git_remote_listing{
*git_url,
*git_ref,
@@ -239,6 +234,7 @@ std::vector<package_id> catalog::by_name(std::string_view sv) const noexcept {

std::vector<dependency> catalog::dependencies_of(const package_id& pkg) const noexcept {
return sqlite3::exec_iter<std::string,
std::string,
std::string>( //
_stmt_cache,
R"(
@@ -247,15 +243,15 @@ std::vector<dependency> catalog::dependencies_of(const package_id& pkg) const no
FROM dds_cat_pkgs
WHERE name = ? AND version = ?
)
SELECT dep_name, low
SELECT dep_name, low, high
FROM dds_cat_pkg_deps
WHERE pkg_id IN this_pkg_id
ORDER BY dep_name
)"_sql,
std::forward_as_tuple(pkg.name, pkg.version.to_string())) //
| ranges::views::transform([](auto&& pair) {
auto& [name, ver] = pair;
return dependency{name, semver::version::parse(ver)};
auto& [name, low, high] = pair;
return dependency{name, {semver::version::parse(low), semver::version::parse(high)}};
}) //
| ranges::to_vector;
}
@@ -308,9 +304,10 @@ void catalog::import_json_str(std::string_view content) {
pkg_name,
version_,
dep_name));
auto range = semver::range::parse(std::string(dep_version));
info.deps.push_back({
std::string(dep_name),
semver::version::parse(std::string(dep_version)),
{range.low(), range.high()},
});
}

@@ -334,3 +331,9 @@ void catalog::import_json_str(std::string_view content) {
}
}
}

std::vector<package_id> catalog::solve_requirements(const std::vector<dependency>& deps) const {
return dds::solve(deps,
[&](std::string_view pkg_name) { return this->by_name(pkg_name); },
[&](const package_id& pkg) { return this->dependencies_of(pkg); });
}

+ 4
- 2
src/dds/catalog/catalog.hpp Zobrazit soubor

@@ -1,9 +1,9 @@
#pragma once

#include <dds/catalog/git.hpp>
#include <dds/deps.hpp>
#include <dds/package_id.hpp>
#include <dds/util/fs.hpp>
#include <dds/catalog/git.hpp>

#include <neo/sqlite3/database.hpp>
#include <neo/sqlite3/statement.hpp>
@@ -38,7 +38,7 @@ public:
static catalog open(const std::string& db_path);
static catalog open(path_ref db_path) { return open(db_path.string()); }

void store(const package_info& info);
void store(const package_info& info);
std::optional<package_info> get(const package_id& id) const noexcept;

std::vector<package_id> by_name(std::string_view sv) const noexcept;
@@ -49,6 +49,8 @@ public:
auto content = dds::slurp_file(json_path);
import_json_str(content);
}

std::vector<package_id> solve_requirements(const std::vector<dependency>& deps) const;
};

} // namespace dds

+ 42
- 4
src/dds/catalog/catalog.test.cpp Zobrazit soubor

@@ -34,8 +34,8 @@ TEST_CASE_METHOD(catalog_test_case, "Package requirements") {
db.store(dds::package_info{
dds::package_id{"foo", semver::version::parse("1.2.3")},
{
{"bar", semver::version::parse("1.2.5")},
{"baz", semver::version::parse("5.3.2")},
{"bar", {semver::version::parse("1.2.3"), semver::version::parse("1.4.0")}},
{"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}},
},
dds::git_remote_listing{"http://example.com", "master", std::nullopt},
});
@@ -55,7 +55,7 @@ TEST_CASE_METHOD(catalog_test_case, "Parse JSON repo") {
"foo": {
"1.2.3": {
"depends": {
"bar": "4.2.1"
"bar": "~4.2.1"
},
"git": {
"url": "http://example.com",
@@ -72,5 +72,43 @@ TEST_CASE_METHOD(catalog_test_case, "Parse JSON repo") {
auto deps = db.dependencies_of(pkgs[0]);
REQUIRE(deps.size() == 1);
CHECK(deps[0].name == "bar");
CHECK(deps[0].version == semver::version::parse("4.2.1"));
CHECK(deps[0].versions
== dds::version_range_set{semver::version::parse("4.2.1"),
semver::version::parse("4.3.0")});
}

TEST_CASE_METHOD(catalog_test_case, "Simple solve") {
db.import_json_str(R"({
"version": 1,
"packages": {
"foo": {
"1.2.3": {
"depends": {
"bar": "~4.2.1"
},
"git": {
"url": "http://example.com",
"ref": "master"
}
}
},
"bar": {
"4.2.3": {
"depends": {},
"git": {
"url": "http://example.com",
"ref": "master"
}
}
}
}
})");
auto sln = db.solve_requirements({{"foo",
dds::version_range_set{semver::version::parse("1.0.0"),
semver::version::parse("2.0.0")}}});
REQUIRE(sln.size() == 2);
CHECK(sln[0].name == "foo");
CHECK(sln[0].version == semver::version::parse("1.2.3"));
CHECK(sln[1].name == "bar");
CHECK(sln[1].version == semver::version::parse("4.2.3"));
}

+ 14
- 17
src/dds/dds.main.cpp Zobrazit soubor

@@ -227,7 +227,8 @@ struct cli_catalog {
std::vector<dds::dependency> deps;
for (const auto& dep : this->deps.Get()) {
auto dep_id = dds::package_id::parse(dep);
deps.push_back({dep_id.name, dep_id.version});
assert(false && "TODO");
// deps.push_back({dep_id.name, dep_id.version});
}

dds::package_info info{ident, std::move(deps), {}};
@@ -540,7 +541,7 @@ struct cli_deps {
int run() {
const auto man = parent.load_package_manifest();
for (const auto& dep : man.dependencies) {
std::cout << dep.name << " " << dep.version.to_string() << '\n';
std::cout << dep.name << " " << dep.versions << '\n';
}
return 0;
}
@@ -556,31 +557,29 @@ struct cli_deps {
repo_path_flag repo_where{cmd};
catalog_path_flag catalog_path{cmd};


int run() {
auto man = parent.load_package_manifest();
auto catalog = catalog_path.open();
bool failed = false;
auto man = parent.load_package_manifest();
auto catalog = catalog_path.open();
bool failed = false;
auto solved_deps = catalog.solve_requirements(man.dependencies);
dds::repository::with_repository( //
repo_where.Get(),
dds::repo_flags::write_lock | dds::repo_flags::create_if_absent,
[&](dds::repository repo) {
for (auto& dep : man.dependencies) {
auto exists = !!repo.find(dep.name, dep.version);
for (const dds::package_id& pk : solved_deps) {
auto exists = !!repo.find(pk);
if (!exists) {
spdlog::info("Pull remote: {}@{}", dep.name, dep.version.to_string());
auto opt_pkg = catalog.get(dds::package_id{dep.name, dep.version});
spdlog::info("Pull remote: {}", pk.to_string());
auto opt_pkg = catalog.get(pk);
if (opt_pkg) {
auto tsd = dds::get_package_sdist(*opt_pkg);
repo.add_sdist(tsd.sdist, dds::if_exists::ignore);
} else {
spdlog::error("No remote listing for {}@{}",
dep.name,
dep.version.to_string());
spdlog::error("No remote listing for {}", pk.to_string());
failed = true;
}
} else {
spdlog::info("Okay: {} {}", dep.name, dep.version.to_string());
spdlog::info("Okay: {}", pk.to_string());
}
}
});
@@ -621,9 +620,7 @@ struct cli_deps {
repo_where.Get(),
dds::repo_flags::read,
[&](dds::repository repo) {
return find_dependencies(repo,
man.dependencies.begin(),
man.dependencies.end());
return repo.solve(man.dependencies);
});

auto tc = tc_filepath.get_toolchain();

+ 9
- 41
src/dds/deps.cpp Zobrazit soubor

@@ -10,6 +10,7 @@
#include <range/v3/range/conversion.hpp>
#include <range/v3/view/transform.hpp>
#include <spdlog/spdlog.h>
#include <spdlog/fmt/ostr.h>

#include <cctype>
#include <map>
@@ -29,49 +30,16 @@ dependency dependency::parse_depends_string(std::string_view str) {
auto name = trim_view(std::string_view(str_begin, str_iter - str_begin));
auto version_str = trim_view(std::string_view(str_iter, str_end - str_iter));

semver::version version;
try {
version = semver::version::parse(version_str);
} catch (const semver::invalid_version&) {
throw std::runtime_error(
fmt::format("Invalid version string '{}' in dependency declaration '{}' (Should be a "
"semver string. See https://semver.org/ for info)",
version_str,
str));
auto rng = semver::range::parse_restricted(version_str);
return dependency{std::string(name), {rng.low(), rng.high()}};
} catch (const semver::invalid_range&) {
throw std::runtime_error(fmt::format(
"Invalid version range string '{}' in dependency declaration '{}' (Should be a "
"semver range string. See https://semver.org/ for info)",
version_str,
str));
}
return dependency{std::string(name), version};
}

std::vector<sdist> dds::find_dependencies(const repository& repo, const dependency& dep) {
std::vector<sdist> acc;
detail::do_find_deps(repo, dep, acc);
return acc;
}

void detail::do_find_deps(const repository& repo, const dependency& dep, std::vector<sdist>& sd) {
auto sdist_opt = repo.find(dep.name, dep.version);
if (!sdist_opt) {
throw std::runtime_error(
fmt::format("Unable to find dependency to satisfy requirement: {} {}",
dep.name,
dep.version.to_string()));
}
const sdist& new_sd = *sdist_opt;
for (const auto& inner_dep : new_sd.manifest.dependencies) {
do_find_deps(repo, inner_dep, sd);
}
auto insert_point = std::partition_point(sd.begin(), sd.end(), [&](const sdist& cand) {
return cand.path < new_sd.path;
});
if (insert_point != sd.end()
&& insert_point->manifest.pk_id.name == new_sd.manifest.pk_id.name) {
if (insert_point->manifest.pk_id.version != new_sd.manifest.pk_id.version) {
assert(false && "Version conflict resolution not implemented yet");
std::terminate();
}
return;
}
sd.insert(insert_point, std::move(new_sd));
}

using sdist_index_type = std::map<std::string, std::reference_wrapper<const sdist>>;

+ 6
- 19
src/dds/deps.hpp Zobrazit soubor

@@ -2,6 +2,8 @@

#include <dds/build/plan/full.hpp>

#include <pubgrub/interval.hpp>
#include <semver/range.hpp>
#include <semver/version.hpp>

#include <string_view>
@@ -18,30 +20,15 @@ enum class version_strength {
major,
};

using version_range_set = pubgrub::interval_set<semver::version>;

struct dependency {
std::string name;
semver::version version;
std::string name;
version_range_set versions;

static dependency parse_depends_string(std::string_view str);
};

namespace detail {

void do_find_deps(const repository&, const dependency& dep, std::vector<sdist>& acc);

} // namespace detail

std::vector<sdist> find_dependencies(const repository& repo, const dependency& dep);

template <typename Iter, typename Snt>
inline std::vector<sdist> find_dependencies(const repository& repo, Iter it, Snt stop) {
std::vector<sdist> acc;
while (it != stop) {
detail::do_find_deps(repo, *it++, acc);
}
return acc;
}

build_plan create_deps_build_plan(const std::vector<sdist>& deps, build_env_ref env);

void write_libman_index(path_ref where, const build_plan& plan, const build_env& env);

+ 31
- 2
src/dds/repo/repo.cpp Zobrazit soubor

@@ -1,11 +1,13 @@
#include "./repo.hpp"

#include <dds/sdist.hpp>
#include <dds/solve/solve.hpp>
#include <dds/util/paths.hpp>
#include <dds/util/string.hpp>

#include <spdlog/spdlog.h>

#include <range/v3/action/sort.hpp>
#include <range/v3/range/conversion.hpp>
#include <range/v3/view/filter.hpp>
#include <range/v3/view/transform.hpp>
@@ -96,10 +98,37 @@ void repository::add_sdist(const sdist& sd, if_exists ife_action) {
spdlog::info("Source distribution '{}' successfully exported", sd.manifest.pk_id.to_string());
}

const sdist* repository::find(std::string_view name, semver::version ver) const noexcept {
auto found = _sdists.find(std::tie(name, ver));
const sdist* repository::find(const package_id& pkg) const noexcept {
auto found = _sdists.find(pkg);
if (found == _sdists.end()) {
return nullptr;
}
return &*found;
}

std::vector<sdist> repository::solve(const std::vector<dependency>& deps) const {
auto ids = dds::solve(deps,
[&](std::string_view name) -> std::vector<package_id> {
auto items = ranges::views::all(_sdists) //
| ranges::views::filter([&](const sdist& sd) {
return sd.manifest.pk_id.name == name;
})
| ranges::views::transform(
[](const sdist& sd) { return sd.manifest.pk_id; })
| ranges::to_vector;
ranges::sort(items, std::less<>{});
return items;
},
[&](const package_id& pkg_id) {
auto found = find(pkg_id);
assert(found);
return found->manifest.dependencies;
});
return ids //
| ranges::views::transform([&](const package_id& pk_id) {
auto found = find(pk_id);
assert(found);
return *found;
}) //
| ranges::to_vector;
}

+ 3
- 1
src/dds/repo/repo.hpp Zobrazit soubor

@@ -80,7 +80,7 @@ public:

void add_sdist(const sdist&, if_exists = if_exists::throw_exc);

const sdist* find(std::string_view name, semver::version ver) const noexcept;
const sdist* find(const package_id& pk) const noexcept;

auto iter_sdists() const noexcept {
class ret {
@@ -95,6 +95,8 @@ public:
} r{_sdists};
return r;
}

std::vector<sdist> solve(const std::vector<dependency>& deps) const;
};

} // namespace dds

+ 4
- 8
src/dds/sdist.hpp Zobrazit soubor

@@ -30,15 +30,11 @@ inline constexpr struct sdist_compare_t {
bool operator()(const sdist& lhs, const sdist& rhs) const {
return lhs.manifest.pk_id < rhs.manifest.pk_id;
}
template <typename Name, typename Version>
bool operator()(const sdist& lhs, const std::tuple<Name, Version>& rhs) const {
auto&& [name, ver] = rhs;
return lhs.manifest.pk_id < package_id{name, ver};
bool operator()(const sdist& lhs, const package_id& rhs) const {
return lhs.manifest.pk_id < rhs;
}
template <typename Name, typename Version>
bool operator()(const std::tuple<Name, Version>& lhs, const sdist& rhs) const {
auto&& [name, ver] = lhs;
return package_id{name, ver} < rhs.manifest.pk_id;
bool operator()(const package_id& lhs, const sdist& rhs) const {
return lhs < rhs.manifest.pk_id;
}
using is_transparent = int;
} sdist_compare;

+ 102
- 0
src/dds/solve/solve.cpp Zobrazit soubor

@@ -0,0 +1,102 @@
#include "./solve.hpp"

#include <pubgrub/solve.hpp>

#include <range/v3/range/conversion.hpp>
#include <range/v3/view/transform.hpp>

using namespace dds;

namespace {

struct req_type {
dependency dep;

using req_ref = const req_type&;

bool implied_by(req_ref other) const noexcept {
return dep.versions.contains(other.dep.versions);
}

bool excludes(req_ref other) const noexcept {
return dep.versions.disjoint(other.dep.versions);
}

std::optional<req_type> intersection(req_ref other) const noexcept {
auto range = dep.versions.intersection(other.dep.versions);
if (range.empty()) {
return std::nullopt;
}
return req_type{dependency{dep.name, std::move(range)}};
}

std::optional<req_type> union_(req_ref other) const noexcept {
auto range = dep.versions.union_(other.dep.versions);
if (range.empty()) {
return std::nullopt;
}
return req_type{dependency{dep.name, std::move(range)}};
}

std::optional<req_type> difference(req_ref other) const noexcept {
auto range = dep.versions.difference(other.dep.versions);
if (range.empty()) {
return std::nullopt;
}
return req_type{dependency{dep.name, std::move(range)}};
}

auto key() const noexcept { return dep.name; }

friend bool operator==(req_ref lhs, req_ref rhs) noexcept {
return lhs.dep.name == rhs.dep.name && lhs.dep.versions == rhs.dep.versions;
}
};

auto as_pkg_id(const req_type& req) {
const version_range_set& versions = req.dep.versions;
assert(versions.num_intervals() == 1);
return package_id{req.dep.name, (*versions.iter_intervals().begin()).low};
}

struct solver_provider {
pkg_id_provider_fn& pkgs_for_name;
deps_provider_fn& deps_for_pkg;

mutable std::map<std::string, std::vector<package_id>> pkgs_by_name = {};

std::optional<req_type> best_candidate(const req_type& req) const {
auto found = pkgs_by_name.find(req.dep.name);
if (found == pkgs_by_name.end()) {
found = pkgs_by_name.emplace(req.dep.name, pkgs_for_name(req.dep.name)).first;
}
auto& vec = found->second;
auto cand = std::find_if(vec.cbegin(), vec.cend(), [&](const package_id& pk) {
return req.dep.versions.contains(pk.version);
});
if (cand == vec.cend()) {
return std::nullopt;
}
return req_type{dependency{cand->name, {cand->version, cand->version.next_after()}}};
}

std::vector<req_type> requirements_of(const req_type& req) const {
auto pk_id = as_pkg_id(req);
auto deps = deps_for_pkg(pk_id);
return deps //
| ranges::views::transform([](const dependency& dep) { return req_type{dep}; }) //
| ranges::to_vector;
}
};

} // namespace

std::vector<package_id> dds::solve(const std::vector<dependency>& deps,
pkg_id_provider_fn pkgs_prov,
deps_provider_fn deps_prov) {
auto wrap_req
= deps | ranges::v3::views::transform([](const dependency& dep) { return req_type{dep}; });

auto solution = pubgrub::solve(wrap_req, solver_provider{pkgs_prov, deps_prov});
return solution | ranges::views::transform(as_pkg_id) | ranges::to_vector;
}

+ 16
- 0
src/dds/solve/solve.hpp Zobrazit soubor

@@ -0,0 +1,16 @@
#pragma once

#include <dds/deps.hpp>
#include <dds/package_id.hpp>

#include <functional>

namespace dds {

using pkg_id_provider_fn = std::function<std::vector<package_id>(std::string_view)>;
using deps_provider_fn = std::function<std::vector<dependency>(const package_id& pk)>;

std::vector<package_id>
solve(const std::vector<dependency>& deps, pkg_id_provider_fn, deps_provider_fn);

} // namespace dds

+ 1
- 0
src/libman/parse.hpp Zobrazit soubor

@@ -58,6 +58,7 @@ public:
}

inline bool operator!=(const pair_iterator& o) const noexcept { return _iter != o._iter; }
inline bool operator==(const pair_iterator& o) const noexcept { return _iter == o._iter; }

auto begin() const noexcept { return *this; }
auto end() const noexcept { return pair_iterator(_end, _end, _key); }

+ 2
- 1
tools/gcc-9.dds Zobrazit soubor

@@ -2,7 +2,8 @@ Compiler-ID: GNU
C++-Version: C++17
C-Compiler: gcc-9
C++-Compiler: g++-9
Flags: -D SPDLOG_COMPILED_LIB -fconcepts -Werror=return-type
# Range-v3 0.10.0 contains an accidental conversion warning
Flags: -D SPDLOG_COMPILED_LIB -fconcepts -Werror=return-type -Wno-conversion
Link-Flags: -static-libgcc -static-libstdc++
Debug: True
Compiler-Launcher: ccache

Načítá se…
Zrušit
Uložit