Merge branch 'feature/version-ranges' into develop

library.dds

 Uses: nlohmann/json
 Uses: neo/buffer
 Uses: neo/sqlite3
-Uses: semver/semver
+Uses: semver/semver
+Uses: pubgrub/pubgrub

package.dds

 Depends: neo-buffer 0.1.0
 Depends: spdlog 1.4.2
 Depends: ms-wil 2019.11.10
-Depends: range-v3 0.9.1
+Depends: range-v3 0.10.0
 Depends: nlohmann-json 3.7.1
 Depends: neo-sqlite3 0.2.2
-Depends: semver 0.2.0
+Depends: semver 0.2.1
+Depends: pubgrub 0.1.2
 
 Test-Driver: Catch-Main

remote.dds

 Remote-Package: range-v3 0.9.1; git url=https://github.com/ericniebler/range-v3.git ref=0.9.1 auto=Niebler/range-v3
+Remote-Package: range-v3 0.10.0; git url=https://github.com/ericniebler/range-v3.git ref=0.10.0 auto=Niebler/range-v3
 Remote-Package: spdlog 1.4.2; git url=https://github.com/gabime/spdlog.git ref=v1.4.2 auto=spdlog/spdlog
 
 # Even a shallow clone of nlohmann-json is HUGE. This fork has only the minimal
 Remote-Package: neo-buffer 0.1.0; git url=https://github.com/vector-of-bool/neo-buffer.git ref=develop
 
 Remote-Package: neo-sqlite3 0.2.2; git url=https://github.com/vector-of-bool/neo-sqlite3.git ref=0.2.2
-Remote-Package: semver 0.2.0; git url=https://github.com/vector-of-bool/semver.git ref=0.2.0
+Remote-Package: semver 0.2.1; git url=https://github.com/vector-of-bool/semver.git ref=0.2.1
+Remote-Package: pubgrub 0.1.2; git url=https://github.com/vector-of-bool/pubgrub.git ref=0.1.2

src/dds/catalog/catalog.cpp

 #include "./catalog.hpp"
 
+#include <dds/solve/solve.hpp>
+
 #include <neo/sqlite3/exec.hpp>
 #include <neo/sqlite3/iter_tuples.hpp>
 #include <neo/sqlite3/single.hpp>
     )"_sql);
     for (const auto& dep : pkg.deps) {
         new_dep_st.reset();
+        if (dep.versions.num_intervals() != 1) {
+            throw std::runtime_error(
+                "Package dependency may only contain a single version interval");
+        }
+        auto iv_1 = *dep.versions.iter_intervals().begin();
         sqlite3::exec(new_dep_st,
                       std::forward_as_tuple(db_pkg_id,
                                             dep.name,
-                                            dep.version.to_string(),
-                                            "[placeholder]"));
+                                            iv_1.low.to_string(),
+                                            iv_1.high.to_string()));
     }
 }
 
     assert(git_url);
     assert(git_ref);
 
-    auto deps = sqlite3::exec_iter<std::string, std::string>(  //
-                    _stmt_cache,
-                    R"(
-                        SELECT dep_name, low
-                          FROM dds_cat_pkg_deps
-                         WHERE pkg_id = ?
-                    )"_sql,
-                    std::tie(pkg_id))
-        | ranges::views::transform([](auto&& pair) {
-                    const auto& [name, ver] = pair;
-                    return dependency{name, semver::version::parse(ver)};
-                })  //
-        | ranges::to_vector;
+    auto deps = dependencies_of(pk_id);
 
     return package_info{
         pk_id,
-        deps,
+        std::move(deps),
         git_remote_listing{
             *git_url,
             *git_ref,
 
 std::vector<dependency> catalog::dependencies_of(const package_id& pkg) const noexcept {
     return sqlite3::exec_iter<std::string,
+                              std::string,
                               std::string>(  //
                _stmt_cache,
                R"(
                       FROM dds_cat_pkgs
                      WHERE name = ? AND version = ?
                 )
-                SELECT dep_name, low
+                SELECT dep_name, low, high
                   FROM dds_cat_pkg_deps
                  WHERE pkg_id IN this_pkg_id
               ORDER BY dep_name
                 )"_sql,
                std::forward_as_tuple(pkg.name, pkg.version.to_string()))  //
         | ranges::views::transform([](auto&& pair) {
-               auto& [name, ver] = pair;
-               return dependency{name, semver::version::parse(ver)};
+               auto& [name, low, high] = pair;
+               return dependency{name, {semver::version::parse(low), semver::version::parse(high)}};
            })  //
         | ranges::to_vector;
 }
                                        pkg_name,
                                        version_,
                                        dep_name));
+                auto range = semver::range::parse(std::string(dep_version));
                 info.deps.push_back({
                     std::string(dep_name),
-                    semver::version::parse(std::string(dep_version)),
+                    {range.low(), range.high()},
                 });
             }
 
         }
     }
 }
+
+std::vector<package_id> catalog::solve_requirements(const std::vector<dependency>& deps) const {
+    return dds::solve(deps,
+                      [&](std::string_view pkg_name) { return this->by_name(pkg_name); },
+                      [&](const package_id& pkg) { return this->dependencies_of(pkg); });
+}

src/dds/catalog/catalog.hpp

 #pragma once
 
+#include <dds/catalog/git.hpp>
 #include <dds/deps.hpp>
 #include <dds/package_id.hpp>
 #include <dds/util/fs.hpp>
-#include <dds/catalog/git.hpp>
 
 #include <neo/sqlite3/database.hpp>
 #include <neo/sqlite3/statement.hpp>
     static catalog open(const std::string& db_path);
     static catalog open(path_ref db_path) { return open(db_path.string()); }
 
-    void store(const package_info& info);
+    void                        store(const package_info& info);
     std::optional<package_info> get(const package_id& id) const noexcept;
 
     std::vector<package_id> by_name(std::string_view sv) const noexcept;
         auto content = dds::slurp_file(json_path);
         import_json_str(content);
     }
+
+    std::vector<package_id> solve_requirements(const std::vector<dependency>& deps) const;
 };
 
 }  // namespace dds

src/dds/catalog/catalog.test.cpp

     db.store(dds::package_info{
         dds::package_id{"foo", semver::version::parse("1.2.3")},
         {
-            {"bar", semver::version::parse("1.2.5")},
-            {"baz", semver::version::parse("5.3.2")},
+            {"bar", {semver::version::parse("1.2.3"), semver::version::parse("1.4.0")}},
+            {"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}},
         },
         dds::git_remote_listing{"http://example.com", "master", std::nullopt},
     });
             "foo": {
                 "1.2.3": {
                     "depends": {
-                        "bar": "4.2.1"
+                        "bar": "~4.2.1"
                     },
                     "git": {
                         "url": "http://example.com",
     auto deps = db.dependencies_of(pkgs[0]);
     REQUIRE(deps.size() == 1);
     CHECK(deps[0].name == "bar");
-    CHECK(deps[0].version == semver::version::parse("4.2.1"));
+    CHECK(deps[0].versions
+          == dds::version_range_set{semver::version::parse("4.2.1"),
+                                    semver::version::parse("4.3.0")});
+}
+
+TEST_CASE_METHOD(catalog_test_case, "Simple solve") {
+    db.import_json_str(R"({
+        "version": 1,
+        "packages": {
+            "foo": {
+                "1.2.3": {
+                    "depends": {
+                        "bar": "~4.2.1"
+                    },
+                    "git": {
+                        "url": "http://example.com",
+                        "ref": "master"
+                    }
+                }
+            },
+            "bar": {
+                "4.2.3": {
+                    "depends": {},
+                    "git": {
+                        "url": "http://example.com",
+                        "ref": "master"
+                    }
+                }
+            }
+        }
+    })");
+    auto sln = db.solve_requirements({{"foo",
+                                       dds::version_range_set{semver::version::parse("1.0.0"),
+                                                              semver::version::parse("2.0.0")}}});
+    REQUIRE(sln.size() == 2);
+    CHECK(sln[0].name == "foo");
+    CHECK(sln[0].version == semver::version::parse("1.2.3"));
+    CHECK(sln[1].name == "bar");
+    CHECK(sln[1].version == semver::version::parse("4.2.3"));
 }

src/dds/dds.main.cpp

             std::vector<dds::dependency> deps;
             for (const auto& dep : this->deps.Get()) {
                 auto dep_id = dds::package_id::parse(dep);
-                deps.push_back({dep_id.name, dep_id.version});
+                assert(false && "TODO");
+                // deps.push_back({dep_id.name, dep_id.version});
             }
 
             dds::package_info info{ident, std::move(deps), {}};
         int run() {
             const auto man = parent.load_package_manifest();
             for (const auto& dep : man.dependencies) {
-                std::cout << dep.name << " " << dep.version.to_string() << '\n';
+                std::cout << dep.name << " " << dep.versions << '\n';
             }
             return 0;
         }
         repo_path_flag    repo_where{cmd};
         catalog_path_flag catalog_path{cmd};
 
-
         int run() {
-            auto man     = parent.load_package_manifest();
-            auto catalog = catalog_path.open();
-            bool failed  = false;
+            auto man         = parent.load_package_manifest();
+            auto catalog     = catalog_path.open();
+            bool failed      = false;
+            auto solved_deps = catalog.solve_requirements(man.dependencies);
             dds::repository::with_repository(  //
                 repo_where.Get(),
                 dds::repo_flags::write_lock | dds::repo_flags::create_if_absent,
                 [&](dds::repository repo) {
-                    for (auto& dep : man.dependencies) {
-                        auto exists = !!repo.find(dep.name, dep.version);
+                    for (const dds::package_id& pk : solved_deps) {
+                        auto exists = !!repo.find(pk);
                         if (!exists) {
-                            spdlog::info("Pull remote: {}@{}", dep.name, dep.version.to_string());
-                            auto opt_pkg = catalog.get(dds::package_id{dep.name, dep.version});
+                            spdlog::info("Pull remote: {}", pk.to_string());
+                            auto opt_pkg = catalog.get(pk);
                             if (opt_pkg) {
                                 auto tsd = dds::get_package_sdist(*opt_pkg);
                                 repo.add_sdist(tsd.sdist, dds::if_exists::ignore);
                             } else {
-                                spdlog::error("No remote listing for {}@{}",
-                                              dep.name,
-                                              dep.version.to_string());
+                                spdlog::error("No remote listing for {}", pk.to_string());
                                 failed = true;
                             }
                         } else {
-                            spdlog::info("Okay: {} {}", dep.name, dep.version.to_string());
+                            spdlog::info("Okay: {}", pk.to_string());
                         }
                     }
                 });
                 repo_where.Get(),
                 dds::repo_flags::read,
                 [&](dds::repository repo) {
-                    return find_dependencies(repo,
-                                             man.dependencies.begin(),
-                                             man.dependencies.end());
+                    return repo.solve(man.dependencies);
                 });
 
             auto tc   = tc_filepath.get_toolchain();

src/dds/deps.cpp

 #include <range/v3/range/conversion.hpp>
 #include <range/v3/view/transform.hpp>
 #include <spdlog/spdlog.h>
+#include <spdlog/fmt/ostr.h>
 
 #include <cctype>
 #include <map>
     auto name        = trim_view(std::string_view(str_begin, str_iter - str_begin));
     auto version_str = trim_view(std::string_view(str_iter, str_end - str_iter));
 
-    semver::version version;
     try {
-        version = semver::version::parse(version_str);
-    } catch (const semver::invalid_version&) {
-        throw std::runtime_error(
-            fmt::format("Invalid version string '{}' in dependency declaration '{}' (Should be a "
-                        "semver string. See https://semver.org/ for info)",
-                        version_str,
-                        str));
+        auto rng = semver::range::parse_restricted(version_str);
+        return dependency{std::string(name), {rng.low(), rng.high()}};
+    } catch (const semver::invalid_range&) {
+        throw std::runtime_error(fmt::format(
+            "Invalid version range string '{}' in dependency declaration '{}' (Should be a "
+            "semver range string. See https://semver.org/ for info)",
+            version_str,
+            str));
     }
-    return dependency{std::string(name), version};
-}
-
-std::vector<sdist> dds::find_dependencies(const repository& repo, const dependency& dep) {
-    std::vector<sdist> acc;
-    detail::do_find_deps(repo, dep, acc);
-    return acc;
-}
-
-void detail::do_find_deps(const repository& repo, const dependency& dep, std::vector<sdist>& sd) {
-    auto sdist_opt = repo.find(dep.name, dep.version);
-    if (!sdist_opt) {
-        throw std::runtime_error(
-            fmt::format("Unable to find dependency to satisfy requirement: {} {}",
-                        dep.name,
-                        dep.version.to_string()));
-    }
-    const sdist& new_sd = *sdist_opt;
-    for (const auto& inner_dep : new_sd.manifest.dependencies) {
-        do_find_deps(repo, inner_dep, sd);
-    }
-    auto insert_point = std::partition_point(sd.begin(), sd.end(), [&](const sdist& cand) {
-        return cand.path < new_sd.path;
-    });
-    if (insert_point != sd.end()
-        && insert_point->manifest.pk_id.name == new_sd.manifest.pk_id.name) {
-        if (insert_point->manifest.pk_id.version != new_sd.manifest.pk_id.version) {
-            assert(false && "Version conflict resolution not implemented yet");
-            std::terminate();
-        }
-        return;
-    }
-    sd.insert(insert_point, std::move(new_sd));
 }
 
 using sdist_index_type = std::map<std::string, std::reference_wrapper<const sdist>>;

src/dds/deps.hpp

 
 #include <dds/build/plan/full.hpp>
 
+#include <pubgrub/interval.hpp>
+#include <semver/range.hpp>
 #include <semver/version.hpp>
 
 #include <string_view>
     major,
 };
 
+using version_range_set = pubgrub::interval_set<semver::version>;
+
 struct dependency {
-    std::string     name;
-    semver::version version;
+    std::string       name;
+    version_range_set versions;
 
     static dependency parse_depends_string(std::string_view str);
 };
 
-namespace detail {
-
-void do_find_deps(const repository&, const dependency& dep, std::vector<sdist>& acc);
-
-}  // namespace detail
-
-std::vector<sdist> find_dependencies(const repository& repo, const dependency& dep);
-
-template <typename Iter, typename Snt>
-inline std::vector<sdist> find_dependencies(const repository& repo, Iter it, Snt stop) {
-    std::vector<sdist> acc;
-    while (it != stop) {
-        detail::do_find_deps(repo, *it++, acc);
-    }
-    return acc;
-}
-
 build_plan create_deps_build_plan(const std::vector<sdist>& deps, build_env_ref env);
 
 void write_libman_index(path_ref where, const build_plan& plan, const build_env& env);

src/dds/repo/repo.cpp

 #include "./repo.hpp"
 
 #include <dds/sdist.hpp>
+#include <dds/solve/solve.hpp>
 #include <dds/util/paths.hpp>
 #include <dds/util/string.hpp>
 
 #include <spdlog/spdlog.h>
 
+#include <range/v3/action/sort.hpp>
 #include <range/v3/range/conversion.hpp>
 #include <range/v3/view/filter.hpp>
 #include <range/v3/view/transform.hpp>
     spdlog::info("Source distribution '{}' successfully exported", sd.manifest.pk_id.to_string());
 }
 
-const sdist* repository::find(std::string_view name, semver::version ver) const noexcept {
-    auto found = _sdists.find(std::tie(name, ver));
+const sdist* repository::find(const package_id& pkg) const noexcept {
+    auto found = _sdists.find(pkg);
     if (found == _sdists.end()) {
         return nullptr;
     }
     return &*found;
 }
+
+std::vector<sdist> repository::solve(const std::vector<dependency>& deps) const {
+    auto ids = dds::solve(deps,
+                          [&](std::string_view name) -> std::vector<package_id> {
+                              auto items = ranges::views::all(_sdists)  //
+                                  | ranges::views::filter([&](const sdist& sd) {
+                                               return sd.manifest.pk_id.name == name;
+                                           })
+                                  | ranges::views::transform(
+                                               [](const sdist& sd) { return sd.manifest.pk_id; })
+                                  | ranges::to_vector;
+                              ranges::sort(items, std::less<>{});
+                              return items;
+                          },
+                          [&](const package_id& pkg_id) {
+                              auto found = find(pkg_id);
+                              assert(found);
+                              return found->manifest.dependencies;
+                          });
+    return ids  //
+        | ranges::views::transform([&](const package_id& pk_id) {
+               auto found = find(pk_id);
+               assert(found);
+               return *found;
+           })  //
+        | ranges::to_vector;
+}

src/dds/repo/repo.hpp

 
     void add_sdist(const sdist&, if_exists = if_exists::throw_exc);
 
-    const sdist* find(std::string_view name, semver::version ver) const noexcept;
+    const sdist* find(const package_id& pk) const noexcept;
 
     auto iter_sdists() const noexcept {
         class ret {
         } r{_sdists};
         return r;
     }
+
+    std::vector<sdist> solve(const std::vector<dependency>& deps) const;
 };
 
 }  // namespace dds

src/dds/sdist.hpp

     bool operator()(const sdist& lhs, const sdist& rhs) const {
         return lhs.manifest.pk_id < rhs.manifest.pk_id;
     }
-    template <typename Name, typename Version>
-    bool operator()(const sdist& lhs, const std::tuple<Name, Version>& rhs) const {
-        auto&& [name, ver] = rhs;
-        return lhs.manifest.pk_id < package_id{name, ver};
+    bool operator()(const sdist& lhs, const package_id& rhs) const {
+        return lhs.manifest.pk_id < rhs;
     }
-    template <typename Name, typename Version>
-    bool operator()(const std::tuple<Name, Version>& lhs, const sdist& rhs) const {
-        auto&& [name, ver] = lhs;
-        return package_id{name, ver} < rhs.manifest.pk_id;
+    bool operator()(const package_id& lhs, const sdist& rhs) const {
+        return lhs < rhs.manifest.pk_id;
     }
     using is_transparent = int;
 } sdist_compare;

src/dds/solve/solve.cpp

+#include "./solve.hpp"
+
+#include <pubgrub/solve.hpp>
+
+#include <range/v3/range/conversion.hpp>
+#include <range/v3/view/transform.hpp>
+
+using namespace dds;
+
+namespace {
+
+struct req_type {
+    dependency dep;
+
+    using req_ref = const req_type&;
+
+    bool implied_by(req_ref other) const noexcept {
+        return dep.versions.contains(other.dep.versions);
+    }
+
+    bool excludes(req_ref other) const noexcept {
+        return dep.versions.disjoint(other.dep.versions);
+    }
+
+    std::optional<req_type> intersection(req_ref other) const noexcept {
+        auto range = dep.versions.intersection(other.dep.versions);
+        if (range.empty()) {
+            return std::nullopt;
+        }
+        return req_type{dependency{dep.name, std::move(range)}};
+    }
+
+    std::optional<req_type> union_(req_ref other) const noexcept {
+        auto range = dep.versions.union_(other.dep.versions);
+        if (range.empty()) {
+            return std::nullopt;
+        }
+        return req_type{dependency{dep.name, std::move(range)}};
+    }
+
+    std::optional<req_type> difference(req_ref other) const noexcept {
+        auto range = dep.versions.difference(other.dep.versions);
+        if (range.empty()) {
+            return std::nullopt;
+        }
+        return req_type{dependency{dep.name, std::move(range)}};
+    }
+
+    auto key() const noexcept { return dep.name; }
+
+    friend bool operator==(req_ref lhs, req_ref rhs) noexcept {
+        return lhs.dep.name == rhs.dep.name && lhs.dep.versions == rhs.dep.versions;
+    }
+};
+
+auto as_pkg_id(const req_type& req) {
+    const version_range_set& versions = req.dep.versions;
+    assert(versions.num_intervals() == 1);
+    return package_id{req.dep.name, (*versions.iter_intervals().begin()).low};
+}
+
+struct solver_provider {
+    pkg_id_provider_fn& pkgs_for_name;
+    deps_provider_fn&   deps_for_pkg;
+
+    mutable std::map<std::string, std::vector<package_id>> pkgs_by_name = {};
+
+    std::optional<req_type> best_candidate(const req_type& req) const {
+        auto found = pkgs_by_name.find(req.dep.name);
+        if (found == pkgs_by_name.end()) {
+            found = pkgs_by_name.emplace(req.dep.name, pkgs_for_name(req.dep.name)).first;
+        }
+        auto& vec  = found->second;
+        auto  cand = std::find_if(vec.cbegin(), vec.cend(), [&](const package_id& pk) {
+            return req.dep.versions.contains(pk.version);
+        });
+        if (cand == vec.cend()) {
+            return std::nullopt;
+        }
+        return req_type{dependency{cand->name, {cand->version, cand->version.next_after()}}};
+    }
+
+    std::vector<req_type> requirements_of(const req_type& req) const {
+        auto pk_id = as_pkg_id(req);
+        auto deps  = deps_for_pkg(pk_id);
+        return deps                                                                          //
+            | ranges::views::transform([](const dependency& dep) { return req_type{dep}; })  //
+            | ranges::to_vector;
+    }
+};
+
+}  // namespace
+
+std::vector<package_id> dds::solve(const std::vector<dependency>& deps,
+                                   pkg_id_provider_fn             pkgs_prov,
+                                   deps_provider_fn               deps_prov) {
+    auto wrap_req
+        = deps | ranges::v3::views::transform([](const dependency& dep) { return req_type{dep}; });
+
+    auto solution = pubgrub::solve(wrap_req, solver_provider{pkgs_prov, deps_prov});
+    return solution | ranges::views::transform(as_pkg_id) | ranges::to_vector;
+}

src/dds/solve/solve.hpp

+#pragma once
+
+#include <dds/deps.hpp>
+#include <dds/package_id.hpp>
+
+#include <functional>
+
+namespace dds {
+
+using pkg_id_provider_fn = std::function<std::vector<package_id>(std::string_view)>;
+using deps_provider_fn   = std::function<std::vector<dependency>(const package_id& pk)>;
+
+std::vector<package_id>
+solve(const std::vector<dependency>& deps, pkg_id_provider_fn, deps_provider_fn);
+
+}  // namespace dds

src/libman/parse.hpp

     }
 
     inline bool operator!=(const pair_iterator& o) const noexcept { return _iter != o._iter; }
+    inline bool operator==(const pair_iterator& o) const noexcept { return _iter == o._iter; }
 
     auto begin() const noexcept { return *this; }
     auto end() const noexcept { return pair_iterator(_end, _end, _key); }

tools/gcc-9.dds

 C++-Version: C++17
 C-Compiler: gcc-9
 C++-Compiler: g++-9
-Flags: -D SPDLOG_COMPILED_LIB -fconcepts -Werror=return-type
+# Range-v3 0.10.0 contains an accidental conversion warning
+Flags: -D SPDLOG_COMPILED_LIB -fconcepts -Werror=return-type -Wno-conversion
 Link-Flags: -static-libgcc -static-libstdc++
 Debug:  True
 Compiler-Launcher: ccache