Browse Source

src/catalog/ is gone. package_id->pkg_id

default_compile_flags
vector-of-bool 3 years ago
parent
commit
e60ba72b17
38 changed files with 147 additions and 155 deletions
  1. +1
    -1
      src/dds/build/builder.cpp
  2. +0
    -16
      src/dds/catalog/get.hpp
  3. +3
    -3
      src/dds/cli/cmd/build_common.cpp
  4. +4
    -4
      src/dds/cli/cmd/build_deps.cpp
  5. +2
    -2
      src/dds/cli/cmd/pkg_get.cpp
  6. +6
    -7
      src/dds/cli/cmd/pkg_ls.cpp
  7. +2
    -2
      src/dds/cli/cmd/repoman_import.cpp
  8. +2
    -2
      src/dds/cli/cmd/repoman_ls.cpp
  9. +6
    -6
      src/dds/cli/cmd/repoman_remove.cpp
  10. +1
    -1
      src/dds/cli/cmd/sdist_create.cpp
  11. +2
    -2
      src/dds/package/manifest.cpp
  12. +2
    -2
      src/dds/package/manifest.hpp
  13. +11
    -11
      src/dds/pkg/cache.cpp
  14. +2
    -2
      src/dds/pkg/cache.hpp
  15. +11
    -13
      src/dds/pkg/db.cpp
  16. +8
    -8
      src/dds/pkg/db.hpp
  17. +6
    -6
      src/dds/pkg/db.test.cpp
  18. +2
    -2
      src/dds/pkg/get/base.cpp
  19. +2
    -2
      src/dds/pkg/get/base.hpp
  20. +7
    -7
      src/dds/pkg/get/get.cpp
  21. +16
    -0
      src/dds/pkg/get/get.hpp
  22. +0
    -0
      src/dds/pkg/get/git.cpp
  23. +0
    -0
      src/dds/pkg/get/git.hpp
  24. +0
    -0
      src/dds/pkg/get/http.cpp
  25. +0
    -0
      src/dds/pkg/get/http.hpp
  26. +1
    -1
      src/dds/pkg/get/http.test.cpp
  27. +4
    -4
      src/dds/pkg/id.cpp
  28. +10
    -10
      src/dds/pkg/id.hpp
  29. +4
    -4
      src/dds/pkg/id.test.cpp
  30. +1
    -1
      src/dds/pkg/info.cpp
  31. +5
    -5
      src/dds/pkg/info.hpp
  32. +6
    -6
      src/dds/repoman/repoman.cpp
  33. +3
    -3
      src/dds/repoman/repoman.hpp
  34. +2
    -3
      src/dds/repoman/repoman.test.cpp
  35. +6
    -6
      src/dds/solve/solve.cpp
  36. +4
    -4
      src/dds/solve/solve.hpp
  37. +2
    -2
      src/dds/source/dist.cpp
  38. +3
    -7
      src/dds/source/dist.hpp

+ 1
- 1
src/dds/build/builder.cpp View File

} }


package_plan prepare_one(state& st, const sdist_target& sd) { package_plan prepare_one(state& st, const sdist_target& sd) {
package_plan pkg{sd.sd.manifest.pkg_id.name, sd.sd.manifest.namespace_};
package_plan pkg{sd.sd.manifest.id.name, sd.sd.manifest.namespace_};
auto libs = collect_libraries(sd.sd.path); auto libs = collect_libraries(sd.sd.path);
for (const auto& lib : libs) { for (const auto& lib : libs) {
pkg.add_library(prepare_library(st, sd, lib, sd.sd.manifest)); pkg.add_library(prepare_library(st, sd, lib, sd.sd.manifest));

+ 0
- 16
src/dds/catalog/get.hpp View File

#pragma once

#include <dds/source/dist.hpp>
#include <dds/temp.hpp>

namespace dds {

class pkg_cache;
class pkg_db;
struct package_info;

temporary_sdist get_package_sdist(const package_info&);

void get_all(const std::vector<package_id>& pkgs, dds::pkg_cache& repo, const pkg_db& cat);

} // namespace dds

+ 3
- 3
src/dds/cli/cmd/build_common.cpp View File

#include "./build_common.hpp" #include "./build_common.hpp"


#include <dds/catalog/get.hpp>
#include <dds/pkg/cache.hpp> #include <dds/pkg/cache.hpp>
#include <dds/pkg/db.hpp> #include <dds/pkg/db.hpp>
#include <dds/pkg/get/get.hpp>


using namespace dds; using namespace dds;


// Download dependencies // Download dependencies
auto deps = repo.solve(man.dependencies, cat); auto deps = repo.solve(man.dependencies, cat);
get_all(deps, repo, cat); get_all(deps, repo, cat);
for (const package_id& pk : deps) {
for (const pkg_id& pk : deps) {
auto sdist_ptr = repo.find(pk); auto sdist_ptr = repo.find(pk);
assert(sdist_ptr); assert(sdist_ptr);
sdist_build_params deps_params; sdist_build_params deps_params;
deps_params.subdir = fs::path("_deps") / sdist_ptr->manifest.pkg_id.to_string();
deps_params.subdir = fs::path("_deps") / sdist_ptr->manifest.id.to_string();
builder.add(*sdist_ptr, deps_params); builder.add(*sdist_ptr, deps_params);
} }
}); });

+ 4
- 4
src/dds/cli/cmd/build_deps.cpp View File



#include <dds/build/builder.hpp> #include <dds/build/builder.hpp>
#include <dds/build/params.hpp> #include <dds/build/params.hpp>
#include <dds/catalog/get.hpp>
#include <dds/pkg/cache.hpp> #include <dds/pkg/cache.hpp>
#include <dds/pkg/get/get.hpp>


#include <range/v3/action/join.hpp> #include <range/v3/action/join.hpp>
#include <range/v3/range/conversion.hpp> #include <range/v3/range/conversion.hpp>
dds_log(info, "Loading {} dependencies", all_deps.size()); dds_log(info, "Loading {} dependencies", all_deps.size());
auto deps = repo.solve(all_deps, cat); auto deps = repo.solve(all_deps, cat);
dds::get_all(deps, repo, cat); dds::get_all(deps, repo, cat);
for (const dds::package_id& pk : deps) {
for (const dds::pkg_id& pk : deps) {
auto sdist_ptr = repo.find(pk); auto sdist_ptr = repo.find(pk);
assert(sdist_ptr); assert(sdist_ptr);
dds::sdist_build_params deps_params; dds::sdist_build_params deps_params;
deps_params.subdir = sdist_ptr->manifest.pkg_id.to_string();
dds_log(info, "Dependency: {}", sdist_ptr->manifest.pkg_id.to_string());
deps_params.subdir = sdist_ptr->manifest.id.to_string();
dds_log(info, "Dependency: {}", sdist_ptr->manifest.id.to_string());
bd.add(*sdist_ptr, deps_params); bd.add(*sdist_ptr, deps_params);
} }
}); });

+ 2
- 2
src/dds/cli/cmd/pkg_get.cpp View File

#include "../options.hpp" #include "../options.hpp"


#include <dds/catalog/get.hpp>
#include <dds/dym.hpp> #include <dds/dym.hpp>
#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>
#include <dds/http/session.hpp> #include <dds/http/session.hpp>
#include <dds/pkg/db.hpp> #include <dds/pkg/db.hpp>
#include <dds/pkg/get/get.hpp>
#include <dds/util/result.hpp> #include <dds/util/result.hpp>


#include <boost/leaf/handle_exception.hpp> #include <boost/leaf/handle_exception.hpp>
static int _pkg_get(const options& opts) { static int _pkg_get(const options& opts) {
auto cat = opts.open_catalog(); auto cat = opts.open_catalog();
for (const auto& item : opts.pkg.get.pkgs) { for (const auto& item : opts.pkg.get.pkgs) {
auto id = package_id::parse(item);
auto id = pkg_id::parse(item);
dds::dym_target dym; dds::dym_target dym;
auto info = cat.get(id); auto info = cat.get(id);
if (!info) { if (!info) {

+ 6
- 7
src/dds/cli/cmd/pkg_ls.cpp View File

static int _pkg_ls(const options& opts) { static int _pkg_ls(const options& opts) {
auto list_contents = [&](pkg_cache repo) { auto list_contents = [&](pkg_cache repo) {
auto same_name auto same_name
= [](auto&& a, auto&& b) { return a.manifest.pkg_id.name == b.manifest.pkg_id.name; };
= [](auto&& a, auto&& b) { return a.manifest.id.name == b.manifest.id.name; };


auto all = repo.iter_sdists(); auto all = repo.iter_sdists();
auto grp_by_name = all // auto grp_by_name = all //
| ranges::views::transform(ranges::to_vector) // | ranges::views::transform(ranges::to_vector) //
| ranges::views::transform([](auto&& grp) { | ranges::views::transform([](auto&& grp) {
assert(grp.size() > 0); assert(grp.size() > 0);
return std::pair(grp[0].manifest.pkg_id.name, grp);
return std::pair(grp[0].manifest.id.name, grp);
}); });


for (const auto& [name, grp] : grp_by_name) { for (const auto& [name, grp] : grp_by_name) {
dds_log(info, "{}:", name); dds_log(info, "{}:", name);
for (const dds::sdist& sd : grp) { for (const dds::sdist& sd : grp) {
dds_log(info, " - {}", sd.manifest.pkg_id.version.to_string());
dds_log(info, " - {}", sd.manifest.id.version.to_string());
} }
} }


return 0; return 0;
}; };


return dds::pkg_cache::with_cache(opts.pkg_cache_dir.value_or(
pkg_cache::default_local_path()),
dds::pkg_cache_flags::read,
list_contents);
return dds::pkg_cache::with_cache(opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()),
dds::pkg_cache_flags::read,
list_contents);
} }


int pkg_ls(const options& opts) { int pkg_ls(const options& opts) {

+ 2
- 2
src/dds/cli/cmd/repoman_import.cpp View File

[](dds::e_sqlite3_error_exc, [](dds::e_sqlite3_error_exc,
boost::leaf::match<neo::sqlite3::errc, neo::sqlite3::errc::constraint_unique>, boost::leaf::match<neo::sqlite3::errc, neo::sqlite3::errc::constraint_unique>,
dds::e_repo_import_targz tgz, dds::e_repo_import_targz tgz,
dds::package_id pkg_id) {
dds::pkg_id pkid) {
dds_log(error, dds_log(error,
"Package {} (from {}) is already present in the repository", "Package {} (from {}) is already present in the repository",
pkg_id.to_string(),
pkid.to_string(),
tgz.path); tgz.path);
return 1; return 1;
}, },

+ 2
- 2
src/dds/cli/cmd/repoman_ls.cpp View File



static int _repoman_ls(const options& opts) { static int _repoman_ls(const options& opts) {
auto repo = repo_manager::open(opts.repoman.repo_dir); auto repo = repo_manager::open(opts.repoman.repo_dir);
for (auto pkg_id : repo.all_packages()) {
std::cout << pkg_id.to_string() << '\n';
for (auto id : repo.all_packages()) {
std::cout << id.to_string() << '\n';
} }
return 0; return 0;
} }

+ 6
- 6
src/dds/cli/cmd/repoman_remove.cpp View File

static int _repoman_remove(const options& opts) { static int _repoman_remove(const options& opts) {
auto repo = repo_manager::open(opts.repoman.repo_dir); auto repo = repo_manager::open(opts.repoman.repo_dir);
for (auto& str : opts.repoman.remove.pkgs) { for (auto& str : opts.repoman.remove.pkgs) {
auto pkg_id = dds::package_id::parse(str);
repo.delete_package(pkg_id);
auto id = dds::pkg_id::parse(str);
repo.delete_package(id);
} }
return 0; return 0;
} }
[](dds::e_sqlite3_error_exc, [](dds::e_sqlite3_error_exc,
boost::leaf::match<neo::sqlite3::errc, neo::sqlite3::errc::constraint_unique>, boost::leaf::match<neo::sqlite3::errc, neo::sqlite3::errc::constraint_unique>,
dds::e_repo_import_targz tgz, dds::e_repo_import_targz tgz,
dds::package_id pkg_id) {
dds::pkg_id pkid) {
dds_log(error, dds_log(error,
"Package {} (from {}) is already present in the repository", "Package {} (from {}) is already present in the repository",
pkg_id.to_string(),
pkid.to_string(),
tgz.path); tgz.path);
return 1; return 1;
}, },
[](dds::e_system_error_exc e, dds::e_repo_delete_path tgz, dds::package_id pkg_id) {
[](dds::e_system_error_exc e, dds::e_repo_delete_path tgz, dds::pkg_id pkid) {
dds_log(error, dds_log(error,
"Cannot delete requested package '{}' from repository (Path {}): {}", "Cannot delete requested package '{}' from repository (Path {}): {}",
pkg_id.to_string(),
pkid.to_string(),
tgz.path, tgz.path,
e.message); e.message);
return 1; return 1;

+ 1
- 1
src/dds/cli/cmd/sdist_create.cpp View File

.include_tests = true, .include_tests = true,
}; };
auto pkg_man = package_manifest::load_from_directory(params.project_dir); auto pkg_man = package_manifest::load_from_directory(params.project_dir);
auto default_filename = fmt::format("{}.tar.gz", pkg_man->pkg_id.to_string());
auto default_filename = fmt::format("{}.tar.gz", pkg_man->id.to_string());
auto filepath = opts.out_path.value_or(fs::current_path() / default_filename); auto filepath = opts.out_path.value_or(fs::current_path() / default_filename);
create_sdist_targz(filepath, params); create_sdist_targz(filepath, params);
return 0; return 0;

+ 2
- 2
src/dds/package/manifest.cpp View File

required_key{"name", required_key{"name",
"A string 'name' is required", "A string 'name' is required",
require_str{"'name' must be a string"}, require_str{"'name' must be a string"},
put_into{ret.pkg_id.name}},
put_into{ret.id.name}},
required_key{"namespace", required_key{"namespace",
"A string 'namespace' is a required ", "A string 'namespace' is a required ",
require_str{"'namespace' must be a string"}, require_str{"'namespace' must be a string"},
required_key{"version", required_key{"version",
"A 'version' string is requried", "A 'version' string is requried",
require_str{"'version' must be a string"}, require_str{"'version' must be a string"},
put_into{ret.pkg_id.version,
put_into{ret.id.version,
[](std::string s) { return semver::version::parse(s); }}}, [](std::string s) { return semver::version::parse(s); }}},
if_key{"depends", if_key{"depends",
[&](auto&& dat) { [&](auto&& dat) {

+ 2
- 2
src/dds/package/manifest.hpp View File

#pragma once #pragma once


#include <dds/deps.hpp> #include <dds/deps.hpp>
#include <dds/package/id.hpp>
#include <dds/pkg/id.hpp>
#include <dds/util/fs.hpp> #include <dds/util/fs.hpp>


#include <optional> #include <optional>
*/ */
struct package_manifest { struct package_manifest {
/// The package ID, as determined by `Name` and `Version` together /// The package ID, as determined by `Name` and `Version` together
package_id pkg_id;
dds::pkg_id id;
/// The declared `Namespace` of the package. This directly corresponds with the libman Namespace /// The declared `Namespace` of the package. This directly corresponds with the libman Namespace
std::string namespace_; std::string namespace_;
/// The `test_driver` that this package declares, or `nullopt` if absent. /// The `test_driver` that this package declares, or `nullopt` if absent.

+ 11
- 11
src/dds/pkg/cache.cpp View File

} }


void pkg_cache::add_sdist(const sdist& sd, if_exists ife_action) { void pkg_cache::add_sdist(const sdist& sd, if_exists ife_action) {
neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.pkg_id.to_string());
neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.id.to_string());
if (!_write_enabled) { if (!_write_enabled) {
dds_log(critical, dds_log(critical,
"DDS attempted to write into a cache that wasn't opened with a write-lock. This " "DDS attempted to write into a cache that wasn't opened with a write-lock. This "
"cache, we'll hard-exit immediately."); "cache, we'll hard-exit immediately.");
std::terminate(); std::terminate();
} }
auto sd_dest = _root / sd.manifest.pkg_id.to_string();
auto sd_dest = _root / sd.manifest.id.to_string();
if (fs::exists(sd_dest)) { if (fs::exists(sd_dest)) {
auto msg = fmt:: auto msg = fmt::
format("Package '{}' (Importing from [{}]) is already available in the local cache", format("Package '{}' (Importing from [{}]) is already available in the local cache",
sd.manifest.pkg_id.to_string(),
sd.manifest.id.to_string(),
sd.path.string()); sd.path.string());
if (ife_action == if_exists::throw_exc) { if (ife_action == if_exists::throw_exc) {
throw_user_error<errc::sdist_exists>(msg); throw_user_error<errc::sdist_exists>(msg);
} }
fs::rename(tmp_copy, sd_dest); fs::rename(tmp_copy, sd_dest);
_sdists.insert(sdist::from_directory(sd_dest)); _sdists.insert(sdist::from_directory(sd_dest));
dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string());
dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.id.to_string());
} }


const sdist* pkg_cache::find(const package_id& pkg) const noexcept {
const sdist* pkg_cache::find(const pkg_id& pkg) const noexcept {
auto found = _sdists.find(pkg); auto found = _sdists.find(pkg);
if (found == _sdists.end()) { if (found == _sdists.end()) {
return nullptr; return nullptr;
return &*found; return &*found;
} }


std::vector<package_id> pkg_cache::solve(const std::vector<dependency>& deps,
const pkg_db& ctlg) const {
std::vector<pkg_id> pkg_cache::solve(const std::vector<dependency>& deps,
const pkg_db& ctlg) const {
return dds::solve( return dds::solve(
deps, deps,
[&](std::string_view name) -> std::vector<package_id> {
[&](std::string_view name) -> std::vector<pkg_id> {
auto mine = ranges::views::all(_sdists) // auto mine = ranges::views::all(_sdists) //
| ranges::views::filter( | ranges::views::filter(
[&](const sdist& sd) { return sd.manifest.pkg_id.name == name; })
| ranges::views::transform([](const sdist& sd) { return sd.manifest.pkg_id; });
[&](const sdist& sd) { return sd.manifest.id.name == name; })
| ranges::views::transform([](const sdist& sd) { return sd.manifest.id; });
auto avail = ctlg.by_name(name); auto avail = ctlg.by_name(name);
auto all = ranges::views::concat(mine, avail) | ranges::to_vector; auto all = ranges::views::concat(mine, avail) | ranges::to_vector;
ranges::sort(all, std::less{}); ranges::sort(all, std::less{});
ranges::unique(all, std::less{}); ranges::unique(all, std::less{});
return all; return all;
}, },
[&](const package_id& pkg_id) {
[&](const pkg_id& pkg_id) {
auto found = find(pkg_id); auto found = find(pkg_id);
if (found) { if (found) {
return found->manifest.dependencies; return found->manifest.dependencies;

+ 2
- 2
src/dds/pkg/cache.hpp View File



void add_sdist(const sdist&, if_exists = if_exists::throw_exc); void add_sdist(const sdist&, if_exists = if_exists::throw_exc);


const sdist* find(const package_id& pk) const noexcept;
const sdist* find(const pkg_id& pk) const noexcept;


auto iter_sdists() const noexcept { auto iter_sdists() const noexcept {
class ret { class ret {
return r; return r;
} }


std::vector<package_id> solve(const std::vector<dependency>& deps, const pkg_db&) const;
std::vector<pkg_id> solve(const std::vector<dependency>& deps, const pkg_db&) const;
}; };


} // namespace dds } // namespace dds

+ 11
- 13
src/dds/pkg/db.cpp View File

)"); )");
} }


void store_with_remote(const neo::sqlite3::statement_cache&,
const package_info& pkg,
std::monostate) {
void store_with_remote(const neo::sqlite3::statement_cache&, const pkg_info& pkg, std::monostate) {
neo_assert_always( neo_assert_always(
invariant, invariant,
false, false,
} }


void store_with_remote(neo::sqlite3::statement_cache& stmts, void store_with_remote(neo::sqlite3::statement_cache& stmts,
const package_info& pkg,
const pkg_info& pkg,
const http_remote_listing& http) { const http_remote_listing& http) {
nsql::exec( // nsql::exec( //
stmts(R"( stmts(R"(
} }


void store_with_remote(neo::sqlite3::statement_cache& stmts, void store_with_remote(neo::sqlite3::statement_cache& stmts,
const package_info& pkg,
const pkg_info& pkg,
const git_remote_listing& git) { const git_remote_listing& git) {
std::string url = git.url; std::string url = git.url;
if (url.starts_with("https://") || url.starts_with("http://")) { if (url.starts_with("https://") || url.starts_with("http://")) {


void do_store_pkg(neo::sqlite3::database& db, void do_store_pkg(neo::sqlite3::database& db,
neo::sqlite3::statement_cache& st_cache, neo::sqlite3::statement_cache& st_cache,
const package_info& pkg) {
const pkg_info& pkg) {
dds_log(debug, "Recording package {}@{}", pkg.ident.name, pkg.ident.version.to_string()); dds_log(debug, "Recording package {}@{}", pkg.ident.name, pkg.ident.version.to_string());
std::visit([&](auto&& remote) { store_with_remote(st_cache, pkg, remote); }, pkg.remote); std::visit([&](auto&& remote) { store_with_remote(st_cache, pkg, remote); }, pkg.remote);
auto db_pkg_id = db.last_insert_rowid(); auto db_pkg_id = db.last_insert_rowid();
pkg_db::pkg_db(nsql::database db) pkg_db::pkg_db(nsql::database db)
: _db(std::move(db)) {} : _db(std::move(db)) {}


void pkg_db::store(const package_info& pkg) {
void pkg_db::store(const pkg_info& pkg) {
nsql::transaction_guard tr{_db}; nsql::transaction_guard tr{_db};
do_store_pkg(_db, _stmt_cache, pkg); do_store_pkg(_db, _stmt_cache, pkg);
} }


std::optional<package_info> pkg_db::get(const package_id& pk_id) const noexcept {
std::optional<pkg_info> pkg_db::get(const pkg_id& pk_id) const noexcept {
auto ver_str = pk_id.version.to_string(); auto ver_str = pk_id.version.to_string();
dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str); dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str);
auto& st = _stmt_cache(R"( auto& st = _stmt_cache(R"(


auto deps = dependencies_of(pk_id); auto deps = dependencies_of(pk_id);


auto info = package_info{
auto info = pkg_info{
pk_id, pk_id,
std::move(deps), std::move(deps),
std::move(description), std::move(description),


auto pair_to_pkg_id = [](auto&& pair) { auto pair_to_pkg_id = [](auto&& pair) {
const auto& [name, ver] = pair; const auto& [name, ver] = pair;
return package_id{name, semver::version::parse(ver)};
return pkg_id{name, semver::version::parse(ver)};
}; };


std::vector<package_id> pkg_db::all() const noexcept {
std::vector<pkg_id> pkg_db::all() const noexcept {
return nsql::exec_tuples<std::string, std::string>( return nsql::exec_tuples<std::string, std::string>(
_stmt_cache("SELECT name, version FROM dds_cat_pkgs"_sql)) _stmt_cache("SELECT name, version FROM dds_cat_pkgs"_sql))
| neo::lref // | neo::lref //
| ranges::to_vector; | ranges::to_vector;
} }


std::vector<package_id> pkg_db::by_name(std::string_view sv) const noexcept {
std::vector<pkg_id> pkg_db::by_name(std::string_view sv) const noexcept {
return nsql::exec_tuples<std::string, std::string>( // return nsql::exec_tuples<std::string, std::string>( //
_stmt_cache( _stmt_cache(
R"( R"(
| ranges::to_vector; | ranges::to_vector;
} }


std::vector<dependency> pkg_db::dependencies_of(const package_id& pkg) const noexcept {
std::vector<dependency> pkg_db::dependencies_of(const pkg_id& pkg) const noexcept {
dds_log(trace, "Lookup dependencies of {}@{}", pkg.name, pkg.version.to_string()); dds_log(trace, "Lookup dependencies of {}@{}", pkg.name, pkg.version.to_string());
return nsql::exec_tuples<std::string, return nsql::exec_tuples<std::string,
std::string, std::string,

+ 8
- 8
src/dds/pkg/db.hpp View File

#pragma once #pragma once


#include "./info.hpp"

#include <dds/deps.hpp> #include <dds/deps.hpp>
#include <dds/package/id.hpp>
#include <dds/pkg/id.hpp>
#include <dds/util/fs.hpp> #include <dds/util/fs.hpp>
#include <dds/util/glob.hpp> #include <dds/util/glob.hpp>


#include <dds/catalog/package_info.hpp>

#include <neo/sqlite3/database.hpp> #include <neo/sqlite3/database.hpp>
#include <neo/sqlite3/statement.hpp> #include <neo/sqlite3/statement.hpp>
#include <neo/sqlite3/statement_cache.hpp> #include <neo/sqlite3/statement_cache.hpp>


static fs::path default_path() noexcept; static fs::path default_path() noexcept;


void store(const package_info& info);
std::optional<package_info> get(const package_id& id) const noexcept;
void store(const pkg_info& info);
std::optional<pkg_info> get(const pkg_id& id) const noexcept;


std::vector<package_id> all() const noexcept;
std::vector<package_id> by_name(std::string_view sv) const noexcept;
std::vector<dependency> dependencies_of(const package_id& pkg) const noexcept;
std::vector<pkg_id> all() const noexcept;
std::vector<pkg_id> by_name(std::string_view sv) const noexcept;
std::vector<dependency> dependencies_of(const pkg_id& pkg) const noexcept;


auto& database() noexcept { return _db; } auto& database() noexcept { return _db; }
auto& database() const noexcept { return _db; } auto& database() const noexcept { return _db; }

+ 6
- 6
src/dds/pkg/db.test.cpp View File

}; };


TEST_CASE_METHOD(catalog_test_case, "Store a simple package") { TEST_CASE_METHOD(catalog_test_case, "Store a simple package") {
db.store(dds::package_info{
dds::package_id("foo", semver::version::parse("1.2.3")),
db.store(dds::pkg_info{
dds::pkg_id("foo", semver::version::parse("1.2.3")),
{}, {},
"example", "example",
dds::git_remote_listing{std::nullopt, "git+http://example.com", "master"}, dds::git_remote_listing{std::nullopt, "git+http://example.com", "master"},
CHECK(std::get<dds::git_remote_listing>(info->remote).ref == "master"); CHECK(std::get<dds::git_remote_listing>(info->remote).ref == "master");


// Update the entry with a new git remote ref // Update the entry with a new git remote ref
CHECK_NOTHROW(db.store(dds::package_info{
dds::package_id("foo", semver::version::parse("1.2.3")),
CHECK_NOTHROW(db.store(dds::pkg_info{
dds::pkg_id("foo", semver::version::parse("1.2.3")),
{}, {},
"example", "example",
dds::git_remote_listing{std::nullopt, "git+http://example.com", "develop"}, dds::git_remote_listing{std::nullopt, "git+http://example.com", "develop"},
} }


TEST_CASE_METHOD(catalog_test_case, "Package requirements") { TEST_CASE_METHOD(catalog_test_case, "Package requirements") {
db.store(dds::package_info{
dds::package_id{"foo", semver::version::parse("1.2.3")},
db.store(dds::pkg_info{
dds::pkg_id{"foo", semver::version::parse("1.2.3")},
{ {
{"bar", {semver::version::parse("1.2.3"), semver::version::parse("1.4.0")}}, {"bar", {semver::version::parse("1.2.3"), semver::version::parse("1.4.0")}},
{"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}}, {"baz", {semver::version::parse("5.3.0"), semver::version::parse("6.0.0")}},

src/dds/catalog/remote/base.cpp → src/dds/pkg/get/base.cpp View File

#include "./base.hpp" #include "./base.hpp"


#include <dds/package/id.hpp>
#include <dds/pkg/id.hpp>
#include <dds/util/log.hpp> #include <dds/util/log.hpp>


#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>


using namespace dds; using namespace dds;


void remote_listing_base::generate_auto_lib_files(const package_id& pid, path_ref root) const {
void remote_listing_base::generate_auto_lib_files(const pkg_id& pid, path_ref root) const {
if (auto_lib.has_value()) { if (auto_lib.has_value()) {
dds_log(info, "Generating library data automatically"); dds_log(info, "Generating library data automatically");



src/dds/catalog/remote/base.hpp → src/dds/pkg/get/base.hpp View File



namespace dds { namespace dds {


struct package_id;
struct pkg_id;


struct remote_listing_base { struct remote_listing_base {
std::optional<lm::usage> auto_lib{}; std::optional<lm::usage> auto_lib{};


void generate_auto_lib_files(const package_id& pid, path_ref root) const;
void generate_auto_lib_files(const pkg_id& pid, path_ref root) const;
}; };


template <typename T> template <typename T>

src/dds/catalog/get.cpp → src/dds/pkg/get/get.cpp View File



namespace { namespace {


temporary_sdist do_pull_sdist(const package_info& listing, std::monostate) {
temporary_sdist do_pull_sdist(const pkg_info& listing, std::monostate) {
neo_assert_always( neo_assert_always(
invariant, invariant,
false, false,
} }


template <remote_listing R> template <remote_listing R>
temporary_sdist do_pull_sdist(const package_info& listing, const R& remote) {
temporary_sdist do_pull_sdist(const pkg_info& listing, const R& remote) {
auto tmpdir = dds::temporary_dir::create(); auto tmpdir = dds::temporary_dir::create();


remote.pull_source(tmpdir.path()); remote.pull_source(tmpdir.path());


} // namespace } // namespace


temporary_sdist dds::get_package_sdist(const package_info& pkg) {
temporary_sdist dds::get_package_sdist(const pkg_info& pkg) {
auto tsd = std::visit([&](auto&& remote) { return do_pull_sdist(pkg, remote); }, pkg.remote); auto tsd = std::visit([&](auto&& remote) { return do_pull_sdist(pkg, remote); }, pkg.remote);
if (!(tsd.sdist.manifest.pkg_id == pkg.ident)) {
if (!(tsd.sdist.manifest.id == pkg.ident)) {
throw_external_error<errc::sdist_ident_mismatch>( throw_external_error<errc::sdist_ident_mismatch>(
"The package name@version in the generated source distribution does not match the name " "The package name@version in the generated source distribution does not match the name "
"listed in the remote listing file (expected '{}', but got '{}')", "listed in the remote listing file (expected '{}', but got '{}')",
pkg.ident.to_string(), pkg.ident.to_string(),
tsd.sdist.manifest.pkg_id.to_string());
tsd.sdist.manifest.id.to_string());
} }
return tsd; return tsd;
} }


void dds::get_all(const std::vector<package_id>& pkgs, pkg_cache& repo, const pkg_db& cat) {
void dds::get_all(const std::vector<pkg_id>& pkgs, pkg_cache& repo, const pkg_db& cat) {
std::mutex repo_mut; std::mutex repo_mut;


auto absent_pkg_infos = pkgs // auto absent_pkg_infos = pkgs //
return *info; return *info;
}); });


auto okay = parallel_run(absent_pkg_infos, 8, [&](package_info inf) {
auto okay = parallel_run(absent_pkg_infos, 8, [&](pkg_info inf) {
dds_log(info, "Download package: {}", inf.ident.to_string()); dds_log(info, "Download package: {}", inf.ident.to_string());
auto tsd = get_package_sdist(inf); auto tsd = get_package_sdist(inf);
std::scoped_lock lk{repo_mut}; std::scoped_lock lk{repo_mut};

+ 16
- 0
src/dds/pkg/get/get.hpp View File

#pragma once

#include <dds/source/dist.hpp>
#include <dds/temp.hpp>

namespace dds {

class pkg_cache;
class pkg_db;
struct pkg_info;

temporary_sdist get_package_sdist(const pkg_info&);

void get_all(const std::vector<pkg_id>& pkgs, dds::pkg_cache& repo, const pkg_db& cat);

} // namespace dds

src/dds/catalog/remote/git.cpp → src/dds/pkg/get/git.cpp View File


src/dds/catalog/remote/git.hpp → src/dds/pkg/get/git.hpp View File


src/dds/catalog/remote/http.cpp → src/dds/pkg/get/http.cpp View File


src/dds/catalog/remote/http.hpp → src/dds/pkg/get/http.hpp View File


src/dds/catalog/remote/http.test.cpp → src/dds/pkg/get/http.test.cpp View File

#include <dds/catalog/remote/http.hpp>
#include "./http.hpp"


#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>
#include <dds/source/dist.hpp> #include <dds/source/dist.hpp>

src/dds/package/id.cpp → src/dds/pkg/id.cpp View File

#include <dds/package/id.hpp>
#include <dds/pkg/id.hpp>


#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>




using namespace dds; using namespace dds;


package_id package_id::parse(std::string_view s) {
pkg_id pkg_id::parse(std::string_view s) {
auto at_pos = s.find('@'); auto at_pos = s.find('@');
if (at_pos == s.npos) { if (at_pos == s.npos) {
throw_user_error<errc::invalid_pkg_id>("Invalid package ID '{}'", s); throw_user_error<errc::invalid_pkg_id>("Invalid package ID '{}'", s);
return {std::string(name), semver::version::parse(ver_str)}; return {std::string(name), semver::version::parse(ver_str)};
} }


package_id::package_id(std::string_view n, semver::version v)
pkg_id::pkg_id(std::string_view n, semver::version v)
: name(n) : name(n)
, version(std::move(v)) { , version(std::move(v)) {
if (name.find('@') != name.npos) { if (name.find('@') != name.npos) {
} }
} }


std::string package_id::to_string() const noexcept { return name + "@" + version.to_string(); }
std::string pkg_id::to_string() const noexcept { return name + "@" + version.to_string(); }

src/dds/package/id.hpp → src/dds/pkg/id.hpp View File

* `to_string` method convert between this textual representation, and supports * `to_string` method convert between this textual representation, and supports
* full round-trips. * full round-trips.
*/ */
struct package_id {
struct pkg_id {
/// The name of the package /// The name of the package
std::string name; std::string name;
/// The version of the package /// The version of the package
semver::version version; semver::version version;


/// Default-initialize a package_id with a blank name and a default version
package_id() = default;
/// Default-initialize a pkg_id with a blank name and a default version
pkg_id() = default;
/// Construct a package ID from a name-version pair /// Construct a package ID from a name-version pair
package_id(std::string_view s, semver::version v);
pkg_id(std::string_view s, semver::version v);


/** /**
* Parse the given string into a package_id object.
* Parse the given string into a pkg_id object.
*/ */
static package_id parse(std::string_view);
static pkg_id parse(std::string_view);


/**
* Convert this package_id into its corresponding textual representation.
/**d
* Convert this pkg_id into its corresponding textual representation.
* The returned string can be passed back to `parse()` for a round-trip * The returned string can be passed back to `parse()` for a round-trip
*/ */
std::string to_string() const noexcept; std::string to_string() const noexcept;


friend bool operator<(const package_id& lhs, const package_id& rhs) noexcept {
friend bool operator<(const pkg_id& lhs, const pkg_id& rhs) noexcept {
return std::tie(lhs.name, lhs.version) < std::tie(rhs.name, rhs.version); return std::tie(lhs.name, lhs.version) < std::tie(rhs.name, rhs.version);
} }
friend bool operator==(const package_id& lhs, const package_id& rhs) noexcept {
friend bool operator==(const pkg_id& lhs, const pkg_id& rhs) noexcept {
return std::tie(lhs.name, lhs.version) == std::tie(rhs.name, rhs.version); return std::tie(lhs.name, lhs.version) == std::tie(rhs.name, rhs.version);
} }
}; };

src/dds/package/id.test.cpp → src/dds/pkg/id.test.cpp View File

#include <dds/package/id.hpp>
#include <dds/pkg/id.hpp>


#include <catch2/catch.hpp> #include <catch2/catch.hpp>


{"foo@1.2.3-alpha", "foo", "1.2.3-alpha"}, {"foo@1.2.3-alpha", "foo", "1.2.3-alpha"},
})); }));


auto pk_id = dds::package_id::parse(id_str);
auto pk_id = dds::pkg_id::parse(id_str);
CHECK(pk_id.to_string() == id_str); CHECK(pk_id.to_string() == id_str);
CHECK(pk_id.name == exp_name); CHECK(pk_id.name == exp_name);
CHECK(pk_id.version.to_string() == exp_ver); CHECK(pk_id.version.to_string() == exp_ver);
{"foo@0.1.2-alpha", less_than, "foo@1.0.0"}, {"foo@0.1.2-alpha", less_than, "foo@1.0.0"},
})); }));


auto lhs = dds::package_id::parse(lhs_str);
auto rhs = dds::package_id::parse(rhs_str);
auto lhs = dds::pkg_id::parse(lhs_str);
auto rhs = dds::pkg_id::parse(rhs_str);


if (ord == less_than) { if (ord == less_than) {
CHECK(lhs < rhs); CHECK(lhs < rhs);

src/dds/catalog/package_info.cpp → src/dds/pkg/info.cpp View File

#include "./package_info.hpp"
#include "./info.hpp"


#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>
#include <dds/util/string.hpp> #include <dds/util/string.hpp>

src/dds/catalog/package_info.hpp → src/dds/pkg/info.hpp View File

#pragma once #pragma once


#include "./remote/git.hpp"
#include "./remote/http.hpp"
#include "./get/git.hpp"
#include "./get/http.hpp"


#include <dds/deps.hpp> #include <dds/deps.hpp>
#include <dds/package/id.hpp>
#include <dds/pkg/id.hpp>
#include <dds/util/glob.hpp> #include <dds/util/glob.hpp>


#include <optional> #include <optional>


remote_listing_var parse_remote_url(std::string_view url); remote_listing_var parse_remote_url(std::string_view url);


struct package_info {
package_id ident;
struct pkg_info {
pkg_id ident;
std::vector<dependency> deps; std::vector<dependency> deps;
std::string description; std::string description;



+ 6
- 6
src/dds/repoman/repoman.cpp View File

throw std::runtime_error("Invalid package archive"); throw std::runtime_error("Invalid package archive");
} }


DDS_E_SCOPE(man->pkg_id);
DDS_E_SCOPE(man->id);


neo::sqlite3::transaction_guard tr{_db}; neo::sqlite3::transaction_guard tr{_db};


dds_log(debug, "Recording package {}@{}", man->pkg_id.name, man->pkg_id.version.to_string());
dds_log(debug, "Recording package {}@{}", man->id.name, man->id.version.to_string());
nsql::exec( // nsql::exec( //
_stmts(R"( _stmts(R"(
INSERT INTO dds_repo_packages (name, version, description, url) INSERT INTO dds_repo_packages (name, version, description, url)
printf('dds:%s@%s', ?1, ?2) printf('dds:%s@%s', ?1, ?2)
) )
)"_sql), )"_sql),
man->pkg_id.name,
man->pkg_id.version.to_string());
man->id.name,
man->id.version.to_string());


auto package_id = _db.last_insert_rowid(); auto package_id = _db.last_insert_rowid();


} }


auto dest_path auto dest_path
= pkg_dir() / man->pkg_id.name / man->pkg_id.version.to_string() / "sdist.tar.gz";
= pkg_dir() / man->id.name / man->id.version.to_string() / "sdist.tar.gz";
fs::create_directories(dest_path.parent_path()); fs::create_directories(dest_path.parent_path());
fs::copy(tgz_file, dest_path); fs::copy(tgz_file, dest_path);


tr.commit(); tr.commit();
} }


void repo_manager::delete_package(package_id pkg_id) {
void repo_manager::delete_package(pkg_id pkg_id) {
neo::sqlite3::transaction_guard tr{_db}; neo::sqlite3::transaction_guard tr{_db};


DDS_E_SCOPE(pkg_id); DDS_E_SCOPE(pkg_id);

+ 3
- 3
src/dds/repoman/repoman.hpp View File

#pragma once #pragma once


#include <dds/package/id.hpp>
#include <dds/pkg/id.hpp>
#include <dds/util/fs.hpp> #include <dds/util/fs.hpp>


#include <neo/sqlite3/database.hpp> #include <neo/sqlite3/database.hpp>
std::string name() const noexcept; std::string name() const noexcept;


void import_targz(path_ref tgz_path); void import_targz(path_ref tgz_path);
void delete_package(package_id id);
void delete_package(pkg_id id);


auto all_packages() const noexcept { auto all_packages() const noexcept {
using namespace neo::sqlite3::literals; using namespace neo::sqlite3::literals;
auto tups = neo::sqlite3::iter_tuples<std::string, std::string>(st); auto tups = neo::sqlite3::iter_tuples<std::string, std::string>(st);
return tups | ranges::views::transform([](auto&& pair) { return tups | ranges::views::transform([](auto&& pair) {
auto [name, version] = pair; auto [name, version] = pair;
return package_id{name, semver::version::parse(version)};
return pkg_id{name, semver::version::parse(version)};
}); });
} }
}; };

+ 2
- 3
src/dds/repoman/repoman.test.cpp View File

CHECK(dds::fs::is_directory(repo.pkg_dir() / "neo-url/")); CHECK(dds::fs::is_directory(repo.pkg_dir() / "neo-url/"));
CHECK(dds::fs::is_regular_file(repo.pkg_dir() / "neo-url/0.2.1/sdist.tar.gz")); CHECK(dds::fs::is_regular_file(repo.pkg_dir() / "neo-url/0.2.1/sdist.tar.gz"));
CHECK_THROWS_AS(repo.import_targz(neo_url_tgz), neo::sqlite3::constraint_unique_error); CHECK_THROWS_AS(repo.import_targz(neo_url_tgz), neo::sqlite3::constraint_unique_error);
repo.delete_package(dds::package_id::parse("neo-url@0.2.1"));
repo.delete_package(dds::pkg_id::parse("neo-url@0.2.1"));
CHECK_FALSE(dds::fs::is_regular_file(repo.pkg_dir() / "neo-url/0.2.1/sdist.tar.gz")); CHECK_FALSE(dds::fs::is_regular_file(repo.pkg_dir() / "neo-url/0.2.1/sdist.tar.gz"));
CHECK_FALSE(dds::fs::is_directory(repo.pkg_dir() / "neo-url")); CHECK_FALSE(dds::fs::is_directory(repo.pkg_dir() / "neo-url"));
CHECK_THROWS_AS(repo.delete_package(dds::package_id::parse("neo-url@0.2.1")),
std::system_error);
CHECK_THROWS_AS(repo.delete_package(dds::pkg_id::parse("neo-url@0.2.1")), std::system_error);
CHECK_NOTHROW(repo.import_targz(neo_url_tgz)); CHECK_NOTHROW(repo.import_targz(neo_url_tgz));
} }

+ 6
- 6
src/dds/solve/solve.cpp View File

auto as_pkg_id(const req_type& req) { auto as_pkg_id(const req_type& req) {
const version_range_set& versions = req.dep.versions; const version_range_set& versions = req.dep.versions;
assert(versions.num_intervals() == 1); assert(versions.num_intervals() == 1);
return package_id{req.dep.name, (*versions.iter_intervals().begin()).low};
return pkg_id{req.dep.name, (*versions.iter_intervals().begin()).low};
} }


struct solver_provider { struct solver_provider {
pkg_id_provider_fn& pkgs_for_name; pkg_id_provider_fn& pkgs_for_name;
deps_provider_fn& deps_for_pkg; deps_provider_fn& deps_for_pkg;


mutable std::map<std::string, std::vector<package_id>> pkgs_by_name = {};
mutable std::map<std::string, std::vector<pkg_id>> pkgs_by_name = {};


std::optional<req_type> best_candidate(const req_type& req) const { std::optional<req_type> best_candidate(const req_type& req) const {
dds_log(debug, "Find best candidate of {}", req.dep.to_string()); dds_log(debug, "Find best candidate of {}", req.dep.to_string());
} }
// Find the first package with the version contained by the ranges in the requirement // Find the first package with the version contained by the ranges in the requirement
auto& for_name = found->second; auto& for_name = found->second;
auto cand = std::find_if(for_name.cbegin(), for_name.cend(), [&](const package_id& pk) {
auto cand = std::find_if(for_name.cbegin(), for_name.cend(), [&](const pkg_id& pk) {
return req.dep.versions.contains(pk.version); return req.dep.versions.contains(pk.version);
}); });
if (cand == for_name.cend()) { if (cand == for_name.cend()) {


} // namespace } // namespace


std::vector<package_id> dds::solve(const std::vector<dependency>& deps,
pkg_id_provider_fn pkgs_prov,
deps_provider_fn deps_prov) {
std::vector<pkg_id> dds::solve(const std::vector<dependency>& deps,
pkg_id_provider_fn pkgs_prov,
deps_provider_fn deps_prov) {
auto wrap_req auto wrap_req
= deps | ranges::views::transform([](const dependency& dep) { return req_type{dep}; }); = deps | ranges::views::transform([](const dependency& dep) { return req_type{dep}; });



+ 4
- 4
src/dds/solve/solve.hpp View File

#pragma once #pragma once


#include <dds/deps.hpp> #include <dds/deps.hpp>
#include <dds/package/id.hpp>
#include <dds/pkg/id.hpp>


#include <functional> #include <functional>


namespace dds { namespace dds {


using pkg_id_provider_fn = std::function<std::vector<package_id>(std::string_view)>;
using deps_provider_fn = std::function<std::vector<dependency>(const package_id& pk)>;
using pkg_id_provider_fn = std::function<std::vector<pkg_id>(std::string_view)>;
using deps_provider_fn = std::function<std::vector<dependency>(const pkg_id& pk)>;


std::vector<package_id>
std::vector<pkg_id>
solve(const std::vector<dependency>& deps, pkg_id_provider_fn, deps_provider_fn); solve(const std::vector<dependency>& deps, pkg_id_provider_fn, deps_provider_fn);


} // namespace dds } // namespace dds

+ 2
- 2
src/dds/source/dist.cpp View File

#include "./dist.hpp" #include "./dist.hpp"


#include <dds/catalog/remote/http.hpp>
#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>
#include <dds/library/root.hpp> #include <dds/library/root.hpp>
#include <dds/pkg/get/http.hpp>
#include <dds/temp.hpp> #include <dds/temp.hpp>
#include <dds/util/fs.hpp> #include <dds/util/fs.hpp>
#include <dds/util/log.hpp> #include <dds/util/log.hpp>


auto pkg_man = package_manifest::load_from_file(*man_path); auto pkg_man = package_manifest::load_from_file(*man_path);
sdist_export_file(out, params.project_dir, *man_path); sdist_export_file(out, params.project_dir, *man_path);
dds_log(info, "Generated export as {}", pkg_man.pkg_id.to_string());
dds_log(info, "Generated export as {}", pkg_man.id.to_string());
return sdist::from_directory(out); return sdist::from_directory(out);
} }



+ 3
- 7
src/dds/source/dist.hpp View File



inline constexpr struct sdist_compare_t { inline constexpr struct sdist_compare_t {
bool operator()(const sdist& lhs, const sdist& rhs) const { bool operator()(const sdist& lhs, const sdist& rhs) const {
return lhs.manifest.pkg_id < rhs.manifest.pkg_id;
}
bool operator()(const sdist& lhs, const package_id& rhs) const {
return lhs.manifest.pkg_id < rhs;
}
bool operator()(const package_id& lhs, const sdist& rhs) const {
return lhs < rhs.manifest.pkg_id;
return lhs.manifest.id < rhs.manifest.id;
} }
bool operator()(const sdist& lhs, const pkg_id& rhs) const { return lhs.manifest.id < rhs; }
bool operator()(const pkg_id& lhs, const sdist& rhs) const { return lhs < rhs.manifest.id; }
using is_transparent = int; using is_transparent = int;
} sdist_compare; } sdist_compare;



Loading…
Cancel
Save