ensure_migrated(db); | ensure_migrated(db); | ||||
} catch (const nsql::sqlite3_error& e) { | } catch (const nsql::sqlite3_error& e) { | ||||
dds_log(critical, | dds_log(critical, | ||||
"Failed to load the repository database. It appears to be invalid/corrupted. The " | |||||
"Failed to load the package database. It appears to be invalid/corrupted. The " | |||||
"exception message is: {}", | "exception message is: {}", | ||||
e.what()); | e.what()); | ||||
throw_external_error<errc::corrupted_catalog_db>(); | throw_external_error<errc::corrupted_catalog_db>(); |
#include <dds/catalog/catalog.hpp> | #include <dds/catalog/catalog.hpp> | ||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/repo/repo.hpp> | |||||
#include <dds/pkg/cache.hpp> | |||||
#include <dds/util/log.hpp> | #include <dds/util/log.hpp> | ||||
#include <dds/util/parallel.hpp> | #include <dds/util/parallel.hpp> | ||||
return tsd; | return tsd; | ||||
} | } | ||||
void dds::get_all(const std::vector<package_id>& pkgs, repository& repo, const catalog& cat) { | |||||
void dds::get_all(const std::vector<package_id>& pkgs, pkg_cache& repo, const catalog& cat) { | |||||
std::mutex repo_mut; | std::mutex repo_mut; | ||||
auto absent_pkg_infos = pkgs // | auto absent_pkg_infos = pkgs // |
namespace dds { | namespace dds { | ||||
class repository; | |||||
class pkg_cache; | |||||
class catalog; | class catalog; | ||||
struct package_info; | struct package_info; | ||||
temporary_sdist get_package_sdist(const package_info&); | temporary_sdist get_package_sdist(const package_info&); | ||||
void get_all(const std::vector<package_id>& pkgs, dds::repository& repo, const catalog& cat); | |||||
void get_all(const std::vector<package_id>& pkgs, dds::pkg_cache& repo, const catalog& cat); | |||||
} // namespace dds | } // namespace dds |
#include <dds/catalog/catalog.hpp> | #include <dds/catalog/catalog.hpp> | ||||
#include <dds/catalog/get.hpp> | #include <dds/catalog/get.hpp> | ||||
#include <dds/repo/repo.hpp> | |||||
#include <dds/pkg/cache.hpp> | |||||
using namespace dds; | using namespace dds; | ||||
auto man = package_manifest::load_from_directory(opts.project_dir).value_or(package_manifest{}); | auto man = package_manifest::load_from_directory(opts.project_dir).value_or(package_manifest{}); | ||||
auto cat_path = opts.pkg_db_dir.value_or(catalog::default_path()); | auto cat_path = opts.pkg_db_dir.value_or(catalog::default_path()); | ||||
auto repo_path = opts.pkg_cache_dir.value_or(repository::default_local_path()); | |||||
auto repo_path = opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()); | |||||
builder builder; | builder builder; | ||||
if (!opts.build.lm_index.has_value()) { | if (!opts.build.lm_index.has_value()) { | ||||
auto cat = catalog::open(cat_path); | auto cat = catalog::open(cat_path); | ||||
// Build the dependencies | // Build the dependencies | ||||
repository::with_repository( // | |||||
pkg_cache::with_cache( // | |||||
repo_path, | repo_path, | ||||
repo_flags::write_lock | repo_flags::create_if_absent, | |||||
[&](repository repo) { | |||||
pkg_cache_flags::write_lock | pkg_cache_flags::create_if_absent, | |||||
[&](pkg_cache repo) { | |||||
// Download dependencies | // Download dependencies | ||||
auto deps = repo.solve(man.dependencies, cat); | auto deps = repo.solve(man.dependencies, cat); | ||||
get_all(deps, repo, cat); | get_all(deps, repo, cat); |
#include <dds/build/builder.hpp> | #include <dds/build/builder.hpp> | ||||
#include <dds/build/params.hpp> | #include <dds/build/params.hpp> | ||||
#include <dds/catalog/get.hpp> | #include <dds/catalog/get.hpp> | ||||
#include <dds/repo/repo.hpp> | |||||
#include <dds/pkg/cache.hpp> | |||||
#include <range/v3/action/join.hpp> | #include <range/v3/action/join.hpp> | ||||
#include <range/v3/range/conversion.hpp> | #include <range/v3/range/conversion.hpp> | ||||
auto all_deps = ranges::views::concat(all_file_deps, cmd_deps) | ranges::to_vector; | auto all_deps = ranges::views::concat(all_file_deps, cmd_deps) | ranges::to_vector; | ||||
auto cat = opts.open_catalog(); | auto cat = opts.open_catalog(); | ||||
dds::repository::with_repository( // | |||||
opts.pkg_cache_dir.value_or(repository::default_local_path()), | |||||
dds::repo_flags::write_lock | dds::repo_flags::create_if_absent, | |||||
[&](dds::repository repo) { | |||||
dds::pkg_cache::with_cache( // | |||||
opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()), | |||||
dds::pkg_cache_flags::write_lock | dds::pkg_cache_flags::create_if_absent, | |||||
[&](dds::pkg_cache repo) { | |||||
// Download dependencies | // Download dependencies | ||||
dds_log(info, "Loading {} dependencies", all_deps.size()); | dds_log(info, "Loading {} dependencies", all_deps.size()); | ||||
auto deps = repo.solve(all_deps, cat); | auto deps = repo.solve(all_deps, cat); |
#include "../options.hpp" | #include "../options.hpp" | ||||
#include <dds/http/session.hpp> | #include <dds/http/session.hpp> | ||||
#include <dds/repo/repo.hpp> | |||||
#include <dds/pkg/cache.hpp> | |||||
#include <dds/source/dist.hpp> | #include <dds/source/dist.hpp> | ||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
namespace dds::cli::cmd { | namespace dds::cli::cmd { | ||||
static int _pkg_import(const options& opts) { | static int _pkg_import(const options& opts) { | ||||
return repository::with_repository( // | |||||
opts.pkg_cache_dir.value_or(repository::default_local_path()), | |||||
repo_flags::write_lock | repo_flags::create_if_absent, | |||||
return pkg_cache::with_cache( // | |||||
opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()), | |||||
pkg_cache_flags::write_lock | pkg_cache_flags::create_if_absent, | |||||
[&](auto repo) { | [&](auto repo) { | ||||
for (std::string_view tgz_where : opts.pkg.import.items) { | for (std::string_view tgz_where : opts.pkg.import.items) { | ||||
neo_assertion_breadcrumbs("Importing sdist", tgz_where); | neo_assertion_breadcrumbs("Importing sdist", tgz_where); |
#include "../options.hpp" | #include "../options.hpp" | ||||
#include <dds/repo/repo.hpp> | |||||
#include <dds/pkg/cache.hpp> | |||||
#include <dds/source/dist.hpp> | #include <dds/source/dist.hpp> | ||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
namespace dds::cli::cmd { | namespace dds::cli::cmd { | ||||
static int _pkg_ls(const options& opts) { | static int _pkg_ls(const options& opts) { | ||||
auto list_contents = [&](repository repo) { | |||||
auto list_contents = [&](pkg_cache repo) { | |||||
auto same_name | auto same_name | ||||
= [](auto&& a, auto&& b) { return a.manifest.pkg_id.name == b.manifest.pkg_id.name; }; | = [](auto&& a, auto&& b) { return a.manifest.pkg_id.name == b.manifest.pkg_id.name; }; | ||||
return 0; | return 0; | ||||
}; | }; | ||||
return dds::repository::with_repository(opts.pkg_cache_dir.value_or( | |||||
repository::default_local_path()), | |||||
dds::repo_flags::read, | |||||
list_contents); | |||||
return dds::pkg_cache::with_cache(opts.pkg_cache_dir.value_or( | |||||
pkg_cache::default_local_path()), | |||||
dds::pkg_cache_flags::read, | |||||
list_contents); | |||||
} | } | ||||
int pkg_ls(const options& opts) { | int pkg_ls(const options& opts) { |
#include "./repo.hpp" | |||||
#include "./cache.hpp" | |||||
#include <dds/catalog/catalog.hpp> | #include <dds/catalog/catalog.hpp> | ||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
using namespace ranges; | using namespace ranges; | ||||
void repository::_log_blocking(path_ref dirpath) noexcept { | |||||
dds_log(warn, "Another process has the repository directory locked [{}]", dirpath.string()); | |||||
dds_log(warn, "Waiting for repository to be released..."); | |||||
void pkg_cache::_log_blocking(path_ref dirpath) noexcept { | |||||
dds_log(warn, "Another process has the package cache directory locked [{}]", dirpath.string()); | |||||
dds_log(warn, "Waiting for cache to be released..."); | |||||
} | } | ||||
void repository::_init_repo_dir(path_ref dirpath) noexcept { fs::create_directories(dirpath); } | |||||
void pkg_cache::_init_cache_dir(path_ref dirpath) noexcept { fs::create_directories(dirpath); } | |||||
fs::path repository::default_local_path() noexcept { return dds_data_dir() / "repo"; } | |||||
fs::path pkg_cache::default_local_path() noexcept { return dds_data_dir() / "pkg"; } | |||||
repository repository::_open_for_directory(bool writeable, path_ref dirpath) { | |||||
pkg_cache pkg_cache::_open_for_directory(bool writeable, path_ref dirpath) { | |||||
auto try_read_sdist = [](path_ref p) -> std::optional<sdist> { | auto try_read_sdist = [](path_ref p) -> std::optional<sdist> { | ||||
if (starts_with(p.filename().string(), ".")) { | if (starts_with(p.filename().string(), ".")) { | ||||
return std::nullopt; | return std::nullopt; | ||||
return {writeable, dirpath, std::move(entries)}; | return {writeable, dirpath, std::move(entries)}; | ||||
} | } | ||||
void repository::add_sdist(const sdist& sd, if_exists ife_action) { | |||||
void pkg_cache::add_sdist(const sdist& sd, if_exists ife_action) { | |||||
neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.pkg_id.to_string()); | neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.pkg_id.to_string()); | ||||
if (!_write_enabled) { | if (!_write_enabled) { | ||||
dds_log( | |||||
critical, | |||||
"DDS attempted to write into a repository that wasn't opened with a write-lock. This " | |||||
"is a hard bug and should be reported. For the safety and integrity of the local " | |||||
"repository, we'll hard-exit immediately."); | |||||
dds_log(critical, | |||||
"DDS attempted to write into a cache that wasn't opened with a write-lock. This " | |||||
"is a hard bug and should be reported. For the safety and integrity of the local " | |||||
"cache, we'll hard-exit immediately."); | |||||
std::terminate(); | std::terminate(); | ||||
} | } | ||||
auto sd_dest = _root / sd.manifest.pkg_id.to_string(); | auto sd_dest = _root / sd.manifest.pkg_id.to_string(); | ||||
if (fs::exists(sd_dest)) { | if (fs::exists(sd_dest)) { | ||||
auto msg = fmt:: | auto msg = fmt:: | ||||
format("Package '{}' (Importing from [{}]) is already available in the local repo", | |||||
format("Package '{}' (Importing from [{}]) is already available in the local cache", | |||||
sd.manifest.pkg_id.to_string(), | sd.manifest.pkg_id.to_string(), | ||||
sd.path.string()); | sd.path.string()); | ||||
if (ife_action == if_exists::throw_exc) { | if (ife_action == if_exists::throw_exc) { | ||||
dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string()); | dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.pkg_id.to_string()); | ||||
} | } | ||||
const sdist* repository::find(const package_id& pkg) const noexcept { | |||||
const sdist* pkg_cache::find(const package_id& pkg) const noexcept { | |||||
auto found = _sdists.find(pkg); | auto found = _sdists.find(pkg); | ||||
if (found == _sdists.end()) { | if (found == _sdists.end()) { | ||||
return nullptr; | return nullptr; | ||||
return &*found; | return &*found; | ||||
} | } | ||||
std::vector<package_id> repository::solve(const std::vector<dependency>& deps, | |||||
const catalog& ctlg) const { | |||||
std::vector<package_id> pkg_cache::solve(const std::vector<dependency>& deps, | |||||
const catalog& ctlg) const { | |||||
return dds::solve( | return dds::solve( | ||||
deps, | deps, | ||||
[&](std::string_view name) -> std::vector<package_id> { | [&](std::string_view name) -> std::vector<package_id> { |
namespace dds { | namespace dds { | ||||
enum repo_flags { | |||||
enum pkg_cache_flags { | |||||
none = 0b00, | none = 0b00, | ||||
read = none, | read = none, | ||||
create_if_absent = 0b01, | create_if_absent = 0b01, | ||||
ignore, | ignore, | ||||
}; | }; | ||||
inline repo_flags operator|(repo_flags a, repo_flags b) { | |||||
return static_cast<repo_flags>(int(a) | int(b)); | |||||
inline pkg_cache_flags operator|(pkg_cache_flags a, pkg_cache_flags b) { | |||||
return static_cast<pkg_cache_flags>(int(a) | int(b)); | |||||
} | } | ||||
class repository { | |||||
class pkg_cache { | |||||
using sdist_set = std::set<sdist, sdist_compare_t>; | using sdist_set = std::set<sdist, sdist_compare_t>; | ||||
bool _write_enabled = false; | bool _write_enabled = false; | ||||
fs::path _root; | fs::path _root; | ||||
sdist_set _sdists; | sdist_set _sdists; | ||||
repository(bool writeable, path_ref p, sdist_set sds) | |||||
pkg_cache(bool writeable, path_ref p, sdist_set sds) | |||||
: _write_enabled(writeable) | : _write_enabled(writeable) | ||||
, _root(p) | , _root(p) | ||||
, _sdists(std::move(sds)) {} | , _sdists(std::move(sds)) {} | ||||
static void _log_blocking(path_ref dir) noexcept; | |||||
static void _init_repo_dir(path_ref dir) noexcept; | |||||
static repository _open_for_directory(bool writeable, path_ref); | |||||
static void _log_blocking(path_ref dir) noexcept; | |||||
static void _init_cache_dir(path_ref dir) noexcept; | |||||
static pkg_cache _open_for_directory(bool writeable, path_ref); | |||||
public: | public: | ||||
template <typename Func> | template <typename Func> | ||||
static decltype(auto) with_repository(path_ref dirpath, repo_flags flags, Func&& fn) { | |||||
static decltype(auto) with_cache(path_ref dirpath, pkg_cache_flags flags, Func&& fn) { | |||||
if (!fs::exists(dirpath)) { | if (!fs::exists(dirpath)) { | ||||
if (flags & repo_flags::create_if_absent) { | |||||
_init_repo_dir(dirpath); | |||||
if (flags & pkg_cache_flags::create_if_absent) { | |||||
_init_cache_dir(dirpath); | |||||
} | } | ||||
} | } | ||||
shared_file_mutex mut{dirpath / ".dds-repo-lock"}; | |||||
shared_file_mutex mut{dirpath / ".dds-cache-lock"}; | |||||
std::shared_lock shared_lk{mut, std::defer_lock}; | std::shared_lock shared_lk{mut, std::defer_lock}; | ||||
std::unique_lock excl_lk{mut, std::defer_lock}; | std::unique_lock excl_lk{mut, std::defer_lock}; | ||||
bool writeable = (flags & repo_flags::write_lock) != repo_flags::none; | |||||
bool writeable = (flags & pkg_cache_flags::write_lock) != pkg_cache_flags::none; | |||||
if (writeable) { | if (writeable) { | ||||
if (!excl_lk.try_lock()) { | if (!excl_lk.try_lock()) { | ||||
} | } | ||||
} | } | ||||
auto repo = _open_for_directory(writeable, dirpath); | |||||
return std::invoke(NEO_FWD(fn), std::move(repo)); | |||||
auto cache = _open_for_directory(writeable, dirpath); | |||||
return std::invoke(NEO_FWD(fn), std::move(cache)); | |||||
} | } | ||||
static fs::path default_local_path() noexcept; | static fs::path default_local_path() noexcept; |