Browse Source

The "repository" is now the package cache

default_compile_flags
vector-of-bool 3 years ago
parent
commit
a0e2e6ed91
9 changed files with 56 additions and 57 deletions
  1. +1
    -1
      src/dds/catalog/catalog.cpp
  2. +2
    -2
      src/dds/catalog/get.cpp
  3. +2
    -2
      src/dds/catalog/get.hpp
  4. +5
    -5
      src/dds/cli/cmd/build_common.cpp
  5. +5
    -5
      src/dds/cli/cmd/build_deps.cpp
  6. +4
    -4
      src/dds/cli/cmd/pkg_import.cpp
  7. +6
    -6
      src/dds/cli/cmd/pkg_ls.cpp
  8. +16
    -17
      src/dds/pkg/cache.cpp
  9. +15
    -15
      src/dds/pkg/cache.hpp

+ 1
- 1
src/dds/catalog/catalog.cpp View File

ensure_migrated(db); ensure_migrated(db);
} catch (const nsql::sqlite3_error& e) { } catch (const nsql::sqlite3_error& e) {
dds_log(critical, dds_log(critical,
"Failed to load the repository database. It appears to be invalid/corrupted. The "
"Failed to load the package database. It appears to be invalid/corrupted. The "
"exception message is: {}", "exception message is: {}",
e.what()); e.what());
throw_external_error<errc::corrupted_catalog_db>(); throw_external_error<errc::corrupted_catalog_db>();

+ 2
- 2
src/dds/catalog/get.cpp View File



#include <dds/catalog/catalog.hpp> #include <dds/catalog/catalog.hpp>
#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>
#include <dds/repo/repo.hpp>
#include <dds/pkg/cache.hpp>
#include <dds/util/log.hpp> #include <dds/util/log.hpp>
#include <dds/util/parallel.hpp> #include <dds/util/parallel.hpp>


return tsd; return tsd;
} }


void dds::get_all(const std::vector<package_id>& pkgs, repository& repo, const catalog& cat) {
void dds::get_all(const std::vector<package_id>& pkgs, pkg_cache& repo, const catalog& cat) {
std::mutex repo_mut; std::mutex repo_mut;


auto absent_pkg_infos = pkgs // auto absent_pkg_infos = pkgs //

+ 2
- 2
src/dds/catalog/get.hpp View File



namespace dds { namespace dds {


class repository;
class pkg_cache;
class catalog; class catalog;
struct package_info; struct package_info;


temporary_sdist get_package_sdist(const package_info&); temporary_sdist get_package_sdist(const package_info&);


void get_all(const std::vector<package_id>& pkgs, dds::repository& repo, const catalog& cat);
void get_all(const std::vector<package_id>& pkgs, dds::pkg_cache& repo, const catalog& cat);


} // namespace dds } // namespace dds

+ 5
- 5
src/dds/cli/cmd/build_common.cpp View File



#include <dds/catalog/catalog.hpp> #include <dds/catalog/catalog.hpp>
#include <dds/catalog/get.hpp> #include <dds/catalog/get.hpp>
#include <dds/repo/repo.hpp>
#include <dds/pkg/cache.hpp>


using namespace dds; using namespace dds;




auto man = package_manifest::load_from_directory(opts.project_dir).value_or(package_manifest{}); auto man = package_manifest::load_from_directory(opts.project_dir).value_or(package_manifest{});
auto cat_path = opts.pkg_db_dir.value_or(catalog::default_path()); auto cat_path = opts.pkg_db_dir.value_or(catalog::default_path());
auto repo_path = opts.pkg_cache_dir.value_or(repository::default_local_path());
auto repo_path = opts.pkg_cache_dir.value_or(pkg_cache::default_local_path());


builder builder; builder builder;
if (!opts.build.lm_index.has_value()) { if (!opts.build.lm_index.has_value()) {
auto cat = catalog::open(cat_path); auto cat = catalog::open(cat_path);
// Build the dependencies // Build the dependencies
repository::with_repository( //
pkg_cache::with_cache( //
repo_path, repo_path,
repo_flags::write_lock | repo_flags::create_if_absent,
[&](repository repo) {
pkg_cache_flags::write_lock | pkg_cache_flags::create_if_absent,
[&](pkg_cache repo) {
// Download dependencies // Download dependencies
auto deps = repo.solve(man.dependencies, cat); auto deps = repo.solve(man.dependencies, cat);
get_all(deps, repo, cat); get_all(deps, repo, cat);

+ 5
- 5
src/dds/cli/cmd/build_deps.cpp View File

#include <dds/build/builder.hpp> #include <dds/build/builder.hpp>
#include <dds/build/params.hpp> #include <dds/build/params.hpp>
#include <dds/catalog/get.hpp> #include <dds/catalog/get.hpp>
#include <dds/repo/repo.hpp>
#include <dds/pkg/cache.hpp>


#include <range/v3/action/join.hpp> #include <range/v3/action/join.hpp>
#include <range/v3/range/conversion.hpp> #include <range/v3/range/conversion.hpp>
auto all_deps = ranges::views::concat(all_file_deps, cmd_deps) | ranges::to_vector; auto all_deps = ranges::views::concat(all_file_deps, cmd_deps) | ranges::to_vector;


auto cat = opts.open_catalog(); auto cat = opts.open_catalog();
dds::repository::with_repository( //
opts.pkg_cache_dir.value_or(repository::default_local_path()),
dds::repo_flags::write_lock | dds::repo_flags::create_if_absent,
[&](dds::repository repo) {
dds::pkg_cache::with_cache( //
opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()),
dds::pkg_cache_flags::write_lock | dds::pkg_cache_flags::create_if_absent,
[&](dds::pkg_cache repo) {
// Download dependencies // Download dependencies
dds_log(info, "Loading {} dependencies", all_deps.size()); dds_log(info, "Loading {} dependencies", all_deps.size());
auto deps = repo.solve(all_deps, cat); auto deps = repo.solve(all_deps, cat);

+ 4
- 4
src/dds/cli/cmd/pkg_import.cpp View File

#include "../options.hpp" #include "../options.hpp"


#include <dds/http/session.hpp> #include <dds/http/session.hpp>
#include <dds/repo/repo.hpp>
#include <dds/pkg/cache.hpp>
#include <dds/source/dist.hpp> #include <dds/source/dist.hpp>
#include <dds/util/result.hpp> #include <dds/util/result.hpp>




namespace dds::cli::cmd { namespace dds::cli::cmd {
static int _pkg_import(const options& opts) { static int _pkg_import(const options& opts) {
return repository::with_repository( //
opts.pkg_cache_dir.value_or(repository::default_local_path()),
repo_flags::write_lock | repo_flags::create_if_absent,
return pkg_cache::with_cache( //
opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()),
pkg_cache_flags::write_lock | pkg_cache_flags::create_if_absent,
[&](auto repo) { [&](auto repo) {
for (std::string_view tgz_where : opts.pkg.import.items) { for (std::string_view tgz_where : opts.pkg.import.items) {
neo_assertion_breadcrumbs("Importing sdist", tgz_where); neo_assertion_breadcrumbs("Importing sdist", tgz_where);

+ 6
- 6
src/dds/cli/cmd/pkg_ls.cpp View File

#include "../options.hpp" #include "../options.hpp"


#include <dds/repo/repo.hpp>
#include <dds/pkg/cache.hpp>
#include <dds/source/dist.hpp> #include <dds/source/dist.hpp>
#include <dds/util/result.hpp> #include <dds/util/result.hpp>




namespace dds::cli::cmd { namespace dds::cli::cmd {
static int _pkg_ls(const options& opts) { static int _pkg_ls(const options& opts) {
auto list_contents = [&](repository repo) {
auto list_contents = [&](pkg_cache repo) {
auto same_name auto same_name
= [](auto&& a, auto&& b) { return a.manifest.pkg_id.name == b.manifest.pkg_id.name; }; = [](auto&& a, auto&& b) { return a.manifest.pkg_id.name == b.manifest.pkg_id.name; };


return 0; return 0;
}; };


return dds::repository::with_repository(opts.pkg_cache_dir.value_or(
repository::default_local_path()),
dds::repo_flags::read,
list_contents);
return dds::pkg_cache::with_cache(opts.pkg_cache_dir.value_or(
pkg_cache::default_local_path()),
dds::pkg_cache_flags::read,
list_contents);
} }


int pkg_ls(const options& opts) { int pkg_ls(const options& opts) {

src/dds/repo/repo.cpp → src/dds/pkg/cache.cpp View File

#include "./repo.hpp"
#include "./cache.hpp"


#include <dds/catalog/catalog.hpp> #include <dds/catalog/catalog.hpp>
#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>


using namespace ranges; using namespace ranges;


void repository::_log_blocking(path_ref dirpath) noexcept {
dds_log(warn, "Another process has the repository directory locked [{}]", dirpath.string());
dds_log(warn, "Waiting for repository to be released...");
void pkg_cache::_log_blocking(path_ref dirpath) noexcept {
dds_log(warn, "Another process has the package cache directory locked [{}]", dirpath.string());
dds_log(warn, "Waiting for cache to be released...");
} }


void repository::_init_repo_dir(path_ref dirpath) noexcept { fs::create_directories(dirpath); }
void pkg_cache::_init_cache_dir(path_ref dirpath) noexcept { fs::create_directories(dirpath); }


fs::path repository::default_local_path() noexcept { return dds_data_dir() / "repo"; }
fs::path pkg_cache::default_local_path() noexcept { return dds_data_dir() / "pkg"; }


repository repository::_open_for_directory(bool writeable, path_ref dirpath) {
pkg_cache pkg_cache::_open_for_directory(bool writeable, path_ref dirpath) {
auto try_read_sdist = [](path_ref p) -> std::optional<sdist> { auto try_read_sdist = [](path_ref p) -> std::optional<sdist> {
if (starts_with(p.filename().string(), ".")) { if (starts_with(p.filename().string(), ".")) {
return std::nullopt; return std::nullopt;
return {writeable, dirpath, std::move(entries)}; return {writeable, dirpath, std::move(entries)};
} }


void repository::add_sdist(const sdist& sd, if_exists ife_action) {
void pkg_cache::add_sdist(const sdist& sd, if_exists ife_action) {
neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.pkg_id.to_string()); neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.pkg_id.to_string());
if (!_write_enabled) { if (!_write_enabled) {
dds_log(
critical,
"DDS attempted to write into a repository that wasn't opened with a write-lock. This "
"is a hard bug and should be reported. For the safety and integrity of the local "
"repository, we'll hard-exit immediately.");
dds_log(critical,
"DDS attempted to write into a cache that wasn't opened with a write-lock. This "
"is a hard bug and should be reported. For the safety and integrity of the local "
"cache, we'll hard-exit immediately.");
std::terminate(); std::terminate();
} }
auto sd_dest = _root / sd.manifest.pkg_id.to_string(); auto sd_dest = _root / sd.manifest.pkg_id.to_string();
if (fs::exists(sd_dest)) { if (fs::exists(sd_dest)) {
auto msg = fmt:: auto msg = fmt::
format("Package '{}' (Importing from [{}]) is already available in the local repo",
format("Package '{}' (Importing from [{}]) is already available in the local cache",
sd.manifest.pkg_id.to_string(), sd.manifest.pkg_id.to_string(),
sd.path.string()); sd.path.string());
if (ife_action == if_exists::throw_exc) { if (ife_action == if_exists::throw_exc) {
dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string()); dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string());
} }


const sdist* repository::find(const package_id& pkg) const noexcept {
const sdist* pkg_cache::find(const package_id& pkg) const noexcept {
auto found = _sdists.find(pkg); auto found = _sdists.find(pkg);
if (found == _sdists.end()) { if (found == _sdists.end()) {
return nullptr; return nullptr;
return &*found; return &*found;
} }


std::vector<package_id> repository::solve(const std::vector<dependency>& deps,
const catalog& ctlg) const {
std::vector<package_id> pkg_cache::solve(const std::vector<dependency>& deps,
const catalog& ctlg) const {
return dds::solve( return dds::solve(
deps, deps,
[&](std::string_view name) -> std::vector<package_id> { [&](std::string_view name) -> std::vector<package_id> {

src/dds/repo/repo.hpp → src/dds/pkg/cache.hpp View File



namespace dds { namespace dds {


enum repo_flags {
enum pkg_cache_flags {
none = 0b00, none = 0b00,
read = none, read = none,
create_if_absent = 0b01, create_if_absent = 0b01,
ignore, ignore,
}; };


inline repo_flags operator|(repo_flags a, repo_flags b) {
return static_cast<repo_flags>(int(a) | int(b));
inline pkg_cache_flags operator|(pkg_cache_flags a, pkg_cache_flags b) {
return static_cast<pkg_cache_flags>(int(a) | int(b));
} }


class repository {
class pkg_cache {
using sdist_set = std::set<sdist, sdist_compare_t>; using sdist_set = std::set<sdist, sdist_compare_t>;


bool _write_enabled = false; bool _write_enabled = false;
fs::path _root; fs::path _root;
sdist_set _sdists; sdist_set _sdists;


repository(bool writeable, path_ref p, sdist_set sds)
pkg_cache(bool writeable, path_ref p, sdist_set sds)
: _write_enabled(writeable) : _write_enabled(writeable)
, _root(p) , _root(p)
, _sdists(std::move(sds)) {} , _sdists(std::move(sds)) {}


static void _log_blocking(path_ref dir) noexcept;
static void _init_repo_dir(path_ref dir) noexcept;
static repository _open_for_directory(bool writeable, path_ref);
static void _log_blocking(path_ref dir) noexcept;
static void _init_cache_dir(path_ref dir) noexcept;
static pkg_cache _open_for_directory(bool writeable, path_ref);


public: public:
template <typename Func> template <typename Func>
static decltype(auto) with_repository(path_ref dirpath, repo_flags flags, Func&& fn) {
static decltype(auto) with_cache(path_ref dirpath, pkg_cache_flags flags, Func&& fn) {
if (!fs::exists(dirpath)) { if (!fs::exists(dirpath)) {
if (flags & repo_flags::create_if_absent) {
_init_repo_dir(dirpath);
if (flags & pkg_cache_flags::create_if_absent) {
_init_cache_dir(dirpath);
} }
} }


shared_file_mutex mut{dirpath / ".dds-repo-lock"};
shared_file_mutex mut{dirpath / ".dds-cache-lock"};
std::shared_lock shared_lk{mut, std::defer_lock}; std::shared_lock shared_lk{mut, std::defer_lock};
std::unique_lock excl_lk{mut, std::defer_lock}; std::unique_lock excl_lk{mut, std::defer_lock};


bool writeable = (flags & repo_flags::write_lock) != repo_flags::none;
bool writeable = (flags & pkg_cache_flags::write_lock) != pkg_cache_flags::none;


if (writeable) { if (writeable) {
if (!excl_lk.try_lock()) { if (!excl_lk.try_lock()) {
} }
} }


auto repo = _open_for_directory(writeable, dirpath);
return std::invoke(NEO_FWD(fn), std::move(repo));
auto cache = _open_for_directory(writeable, dirpath);
return std::invoke(NEO_FWD(fn), std::move(cache));
} }


static fs::path default_local_path() noexcept; static fs::path default_local_path() noexcept;

Loading…
Cancel
Save