#include "./get.hpp" | #include "./get.hpp" | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/pkg/cache.hpp> | #include <dds/pkg/cache.hpp> | ||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/util/log.hpp> | #include <dds/util/log.hpp> | ||||
#include <dds/util/parallel.hpp> | #include <dds/util/parallel.hpp> | ||||
neo_assert_always( | neo_assert_always( | ||||
invariant, | invariant, | ||||
false, | false, | ||||
"A package listing in the catalog has no defined remote from which to pull. This " | |||||
"A package listing in the database has no defined remote from which to pull. This " | |||||
"shouldn't happen in normal usage. This will occur if the database has been " | "shouldn't happen in normal usage. This will occur if the database has been " | ||||
"manually altered, or if DDS has a bug.", | "manually altered, or if DDS has a bug.", | ||||
listing.ident.to_string()); | listing.ident.to_string()); | ||||
return tsd; | return tsd; | ||||
} | } | ||||
void dds::get_all(const std::vector<package_id>& pkgs, pkg_cache& repo, const catalog& cat) { | |||||
void dds::get_all(const std::vector<package_id>& pkgs, pkg_cache& repo, const pkg_db& cat) { | |||||
std::mutex repo_mut; | std::mutex repo_mut; | ||||
auto absent_pkg_infos = pkgs // | auto absent_pkg_infos = pkgs // | ||||
auto info = cat.get(id); | auto info = cat.get(id); | ||||
neo_assert(invariant, | neo_assert(invariant, | ||||
info.has_value(), | info.has_value(), | ||||
"No catalog entry for package id?", | |||||
"No database entry for package id?", | |||||
id.to_string()); | id.to_string()); | ||||
return *info; | return *info; | ||||
}); | }); |
namespace dds { | namespace dds { | ||||
class pkg_cache; | class pkg_cache; | ||||
class catalog; | |||||
class pkg_db; | |||||
struct package_info; | struct package_info; | ||||
temporary_sdist get_package_sdist(const package_info&); | temporary_sdist get_package_sdist(const package_info&); | ||||
void get_all(const std::vector<package_id>& pkgs, dds::pkg_cache& repo, const catalog& cat); | |||||
void get_all(const std::vector<package_id>& pkgs, dds::pkg_cache& repo, const pkg_db& cat); | |||||
} // namespace dds | } // namespace dds |
#pragma once | |||||
#include "./package_info.hpp" | |||||
namespace dds { | |||||
std::vector<package_info> parse_packages_json(std::string_view); | |||||
} // namespace dds |
#include "./build_common.hpp" | #include "./build_common.hpp" | ||||
#include <dds/build/builder.hpp> | #include <dds/build/builder.hpp> | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/remote/remote.hpp> | #include <dds/remote/remote.hpp> | ||||
#include <dds/toolchain/from_json.hpp> | #include <dds/toolchain/from_json.hpp> | ||||
#include "./build_common.hpp" | #include "./build_common.hpp" | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/catalog/get.hpp> | #include <dds/catalog/get.hpp> | ||||
#include <dds/pkg/cache.hpp> | #include <dds/pkg/cache.hpp> | ||||
#include <dds/pkg/db.hpp> | |||||
using namespace dds; | using namespace dds; | ||||
}; | }; | ||||
auto man = package_manifest::load_from_directory(opts.project_dir).value_or(package_manifest{}); | auto man = package_manifest::load_from_directory(opts.project_dir).value_or(package_manifest{}); | ||||
auto cat_path = opts.pkg_db_dir.value_or(catalog::default_path()); | |||||
auto cat_path = opts.pkg_db_dir.value_or(pkg_db::default_path()); | |||||
auto repo_path = opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()); | auto repo_path = opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()); | ||||
builder builder; | builder builder; | ||||
if (!opts.build.lm_index.has_value()) { | if (!opts.build.lm_index.has_value()) { | ||||
auto cat = catalog::open(cat_path); | |||||
auto cat = pkg_db::open(cat_path); | |||||
// Build the dependencies | // Build the dependencies | ||||
pkg_cache::with_cache( // | pkg_cache::with_cache( // | ||||
repo_path, | repo_path, |
#include "../options.hpp" | #include "../options.hpp" | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/catalog/get.hpp> | #include <dds/catalog/get.hpp> | ||||
#include <dds/dym.hpp> | #include <dds/dym.hpp> | ||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/http/session.hpp> | #include <dds/http/session.hpp> | ||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
#include <boost/leaf/handle_exception.hpp> | #include <boost/leaf/handle_exception.hpp> | ||||
auto info = cat.get(id); | auto info = cat.get(id); | ||||
if (!info) { | if (!info) { | ||||
dds::throw_user_error<dds::errc::no_such_catalog_package>( | dds::throw_user_error<dds::errc::no_such_catalog_package>( | ||||
"No package in the catalog matched the ID '{}'.{}", item, dym.sentence_suffix()); | |||||
"No package in the database matched the ID '{}'.{}", item, dym.sentence_suffix()); | |||||
} | } | ||||
auto tsd = get_package_sdist(*info); | auto tsd = get_package_sdist(*info); | ||||
auto dest = opts.out_path.value_or(fs::current_path()) / id.to_string(); | auto dest = opts.out_path.value_or(fs::current_path()) / id.to_string(); |
#include "./pkg_repo_err_handle.hpp" | #include "./pkg_repo_err_handle.hpp" | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/remote/remote.hpp> | #include <dds/remote/remote.hpp> | ||||
namespace dds::cli::cmd { | namespace dds::cli::cmd { |
#include "./pkg_repo_err_handle.hpp" | #include "./pkg_repo_err_handle.hpp" | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/remote/remote.hpp> | #include <dds/remote/remote.hpp> | ||||
namespace dds::cli::cmd { | namespace dds::cli::cmd { |
#include "./error_handler.hpp" | #include "./error_handler.hpp" | ||||
#include "./options.hpp" | #include "./options.hpp" | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/remote/remote.hpp> | #include <dds/remote/remote.hpp> | ||||
#include <dds/util/paths.hpp> | #include <dds/util/paths.hpp> | ||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> |
#include "./options.hpp" | #include "./options.hpp" | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/toolchain/from_json.hpp> | #include <dds/toolchain/from_json.hpp> | ||||
#include <dds/toolchain/toolchain.hpp> | #include <dds/toolchain/toolchain.hpp> | ||||
build_cmd.add_argument({ | build_cmd.add_argument({ | ||||
.long_spellings = {"add-repo"}, | .long_spellings = {"add-repo"}, | ||||
.help = "" | .help = "" | ||||
"Add remote repositories to the package catalog before building\n" | |||||
"Add remote repositories to the package database before building\n" | |||||
"(Implies --update-repos)", | "(Implies --update-repos)", | ||||
.valname = "<repo-url>", | .valname = "<repo-url>", | ||||
.can_repeat = true, | .can_repeat = true, | ||||
setup{*this}.do_setup(parser); | setup{*this}.do_setup(parser); | ||||
} | } | ||||
catalog dds::cli::options::open_catalog() const { | |||||
return catalog::open(this->pkg_db_dir.value_or(catalog::default_path())); | |||||
pkg_db dds::cli::options::open_catalog() const { | |||||
return pkg_db::open(this->pkg_db_dir.value_or(pkg_db::default_path())); | |||||
} | } | ||||
toolchain dds::cli::options::load_toolchain() const { | toolchain dds::cli::options::load_toolchain() const { |
namespace dds { | namespace dds { | ||||
namespace fs = std::filesystem; | namespace fs = std::filesystem; | ||||
class catalog; | |||||
class pkg_db; | |||||
class toolchain; | class toolchain; | ||||
namespace cli { | namespace cli { | ||||
cli::if_exists if_exists = cli::if_exists::fail; | cli::if_exists if_exists = cli::if_exists::fail; | ||||
/** | /** | ||||
* @brief Open the package catalog based on the user-specified options. | |||||
* @return catalog | |||||
* @brief Open the package pkg_db based on the user-specified options. | |||||
* @return pkg_db | |||||
*/ | */ | ||||
catalog open_catalog() const; | |||||
pkg_db open_catalog() const; | |||||
/** | /** | ||||
* @brief Load a dds toolchain as specified by the user, or a default. | * @brief Load a dds toolchain as specified by the user, or a default. | ||||
* @return dds::toolchain | * @return dds::toolchain |
#include "./cache.hpp" | #include "./cache.hpp" | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/solve/solve.hpp> | #include <dds/solve/solve.hpp> | ||||
#include <dds/source/dist.hpp> | #include <dds/source/dist.hpp> | ||||
#include <dds/util/log.hpp> | #include <dds/util/log.hpp> | ||||
} | } | ||||
std::vector<package_id> pkg_cache::solve(const std::vector<dependency>& deps, | std::vector<package_id> pkg_cache::solve(const std::vector<dependency>& deps, | ||||
const catalog& ctlg) const { | |||||
const pkg_db& ctlg) const { | |||||
return dds::solve( | return dds::solve( | ||||
deps, | deps, | ||||
[&](std::string_view name) -> std::vector<package_id> { | [&](std::string_view name) -> std::vector<package_id> { |
#pragma once | #pragma once | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/pkg/db.hpp> | |||||
#include <dds/source/dist.hpp> | #include <dds/source/dist.hpp> | ||||
#include <dds/util/flock.hpp> | #include <dds/util/flock.hpp> | ||||
#include <dds/util/fs.hpp> | #include <dds/util/fs.hpp> | ||||
return r; | return r; | ||||
} | } | ||||
std::vector<package_id> solve(const std::vector<dependency>& deps, const catalog&) const; | |||||
std::vector<package_id> solve(const std::vector<dependency>& deps, const pkg_db&) const; | |||||
}; | }; | ||||
} // namespace dds | } // namespace dds |
#include "./catalog.hpp" | |||||
#include "./import.hpp" | |||||
#include "./db.hpp" | |||||
#include <dds/dym.hpp> | #include <dds/dym.hpp> | ||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
auto meta = nlohmann::json::parse(meta_json); | auto meta = nlohmann::json::parse(meta_json); | ||||
if (!meta.is_object()) { | if (!meta.is_object()) { | ||||
dds_log(critical, "Root of catalog dds_cat_meta cell should be a JSON object"); | |||||
dds_log(critical, "Root of database dds_cat_meta cell should be a JSON object"); | |||||
throw_external_error<errc::corrupted_catalog_db>(); | throw_external_error<errc::corrupted_catalog_db>(); | ||||
} | } | ||||
if (!version_.is_number_integer()) { | if (!version_.is_number_integer()) { | ||||
dds_log(critical, "'version' key in dds_cat_meta is not an integer"); | dds_log(critical, "'version' key in dds_cat_meta is not an integer"); | ||||
throw_external_error<errc::corrupted_catalog_db>( | throw_external_error<errc::corrupted_catalog_db>( | ||||
"The catalog database metadata is invalid [bad dds_meta.version]"); | |||||
"The database metadata is invalid [bad dds_meta.version]"); | |||||
} | } | ||||
constexpr int current_database_version = 3; | constexpr int current_database_version = 3; | ||||
} | } | ||||
if (version < 1) { | if (version < 1) { | ||||
dds_log(debug, "Applying catalog migration 1"); | |||||
dds_log(debug, "Applying pkg_db migration 1"); | |||||
migrate_repodb_1(db); | migrate_repodb_1(db); | ||||
} | } | ||||
if (version < 2) { | if (version < 2) { | ||||
dds_log(debug, "Applying catalog migration 2"); | |||||
dds_log(debug, "Applying pkg_db migration 2"); | |||||
migrate_repodb_2(db); | migrate_repodb_2(db); | ||||
} | } | ||||
if (version < 3) { | if (version < 3) { | ||||
dds_log(debug, "Applying catalog migration 3"); | |||||
dds_log(debug, "Applying pkg_db migration 3"); | |||||
migrate_repodb_3(db); | migrate_repodb_3(db); | ||||
} | } | ||||
meta["version"] = current_database_version; | meta["version"] = current_database_version; | ||||
} // namespace | } // namespace | ||||
fs::path catalog::default_path() noexcept { return dds_data_dir() / "catalog.db"; } | |||||
fs::path pkg_db::default_path() noexcept { return dds_data_dir() / "pkgs.db"; } | |||||
catalog catalog::open(const std::string& db_path) { | |||||
pkg_db pkg_db::open(const std::string& db_path) { | |||||
if (db_path != ":memory:") { | if (db_path != ":memory:") { | ||||
auto pardir = fs::weakly_canonical(db_path).parent_path(); | auto pardir = fs::weakly_canonical(db_path).parent_path(); | ||||
dds_log(trace, "Ensuring parent directory [{}]", pardir.string()); | dds_log(trace, "Ensuring parent directory [{}]", pardir.string()); | ||||
fs::create_directories(pardir); | fs::create_directories(pardir); | ||||
} | } | ||||
dds_log(debug, "Opening package catalog [{}]", db_path); | |||||
dds_log(debug, "Opening package database [{}]", db_path); | |||||
auto db = nsql::database::open(db_path); | auto db = nsql::database::open(db_path); | ||||
try { | try { | ||||
ensure_migrated(db); | ensure_migrated(db); | ||||
e.what()); | e.what()); | ||||
throw_external_error<errc::corrupted_catalog_db>(); | throw_external_error<errc::corrupted_catalog_db>(); | ||||
} | } | ||||
dds_log(trace, "Successfully opened catalog"); | |||||
return catalog(std::move(db)); | |||||
dds_log(trace, "Successfully opened database"); | |||||
return pkg_db(std::move(db)); | |||||
} | } | ||||
catalog::catalog(nsql::database db) | |||||
pkg_db::pkg_db(nsql::database db) | |||||
: _db(std::move(db)) {} | : _db(std::move(db)) {} | ||||
void catalog::store(const package_info& pkg) { | |||||
void pkg_db::store(const package_info& pkg) { | |||||
nsql::transaction_guard tr{_db}; | nsql::transaction_guard tr{_db}; | ||||
do_store_pkg(_db, _stmt_cache, pkg); | do_store_pkg(_db, _stmt_cache, pkg); | ||||
} | } | ||||
std::optional<package_info> catalog::get(const package_id& pk_id) const noexcept { | |||||
std::optional<package_info> pkg_db::get(const package_id& pk_id) const noexcept { | |||||
auto ver_str = pk_id.version.to_string(); | auto ver_str = pk_id.version.to_string(); | ||||
dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str); | dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str); | ||||
auto& st = _stmt_cache(R"( | auto& st = _stmt_cache(R"( | ||||
} | } | ||||
neo_assert_always(invariant, | neo_assert_always(invariant, | ||||
ec == nsql::errc::row, | ec == nsql::errc::row, | ||||
"Failed to pull a package from the catalog database", | |||||
"Failed to pull a package from the database", | |||||
ec, | ec, | ||||
pk_id.to_string(), | pk_id.to_string(), | ||||
nsql::error_category().message(int(ec))); | nsql::error_category().message(int(ec))); | ||||
ec = st.step(std::nothrow); | ec = st.step(std::nothrow); | ||||
if (ec == nsql::errc::row) { | if (ec == nsql::errc::row) { | ||||
dds_log(warn, | dds_log(warn, | ||||
"There is more than one entry for package {} in the catalog database. One will be " | |||||
"There is more than one entry for package {} in the database. One will be " | |||||
"chosen arbitrarily.", | "chosen arbitrarily.", | ||||
pk_id.to_string()); | pk_id.to_string()); | ||||
} | } | ||||
return package_id{name, semver::version::parse(ver)}; | return package_id{name, semver::version::parse(ver)}; | ||||
}; | }; | ||||
std::vector<package_id> catalog::all() const noexcept { | |||||
std::vector<package_id> pkg_db::all() const noexcept { | |||||
return nsql::exec_tuples<std::string, std::string>( | return nsql::exec_tuples<std::string, std::string>( | ||||
_stmt_cache("SELECT name, version FROM dds_cat_pkgs"_sql)) | _stmt_cache("SELECT name, version FROM dds_cat_pkgs"_sql)) | ||||
| neo::lref // | | neo::lref // | ||||
| ranges::to_vector; | | ranges::to_vector; | ||||
} | } | ||||
std::vector<package_id> catalog::by_name(std::string_view sv) const noexcept { | |||||
std::vector<package_id> pkg_db::by_name(std::string_view sv) const noexcept { | |||||
return nsql::exec_tuples<std::string, std::string>( // | return nsql::exec_tuples<std::string, std::string>( // | ||||
_stmt_cache( | _stmt_cache( | ||||
R"( | R"( | ||||
| ranges::to_vector; | | ranges::to_vector; | ||||
} | } | ||||
std::vector<dependency> catalog::dependencies_of(const package_id& pkg) const noexcept { | |||||
std::vector<dependency> pkg_db::dependencies_of(const package_id& pkg) const noexcept { | |||||
dds_log(trace, "Lookup dependencies of {}@{}", pkg.name, pkg.version.to_string()); | dds_log(trace, "Lookup dependencies of {}@{}", pkg.name, pkg.version.to_string()); | ||||
return nsql::exec_tuples<std::string, | return nsql::exec_tuples<std::string, | ||||
std::string, | std::string, |
#include <dds/util/fs.hpp> | #include <dds/util/fs.hpp> | ||||
#include <dds/util/glob.hpp> | #include <dds/util/glob.hpp> | ||||
#include "./package_info.hpp" | |||||
#include <dds/catalog/package_info.hpp> | |||||
#include <neo/sqlite3/database.hpp> | #include <neo/sqlite3/database.hpp> | ||||
#include <neo/sqlite3/statement.hpp> | #include <neo/sqlite3/statement.hpp> | ||||
namespace dds { | namespace dds { | ||||
class catalog { | |||||
class pkg_db { | |||||
neo::sqlite3::database _db; | neo::sqlite3::database _db; | ||||
mutable neo::sqlite3::statement_cache _stmt_cache{_db}; | mutable neo::sqlite3::statement_cache _stmt_cache{_db}; | ||||
explicit catalog(neo::sqlite3::database db); | |||||
catalog(const catalog&) = delete; | |||||
explicit pkg_db(neo::sqlite3::database db); | |||||
pkg_db(const pkg_db&) = delete; | |||||
public: | public: | ||||
catalog(catalog&&) = default; | |||||
catalog& operator=(catalog&&) = default; | |||||
pkg_db(pkg_db&&) = default; | |||||
pkg_db& operator=(pkg_db&&) = default; | |||||
static catalog open(const std::string& db_path); | |||||
static catalog open(path_ref db_path) { return open(db_path.string()); } | |||||
static pkg_db open(const std::string& db_path); | |||||
static pkg_db open(path_ref db_path) { return open(db_path.string()); } | |||||
static fs::path default_path() noexcept; | static fs::path default_path() noexcept; | ||||
#include <dds/catalog/catalog.hpp> | |||||
#include <dds/pkg/db.hpp> | |||||
#include <catch2/catch.hpp> | #include <catch2/catch.hpp> | ||||
TEST_CASE("Create a simple database") { | TEST_CASE("Create a simple database") { | ||||
// Just create and run migrations on an in-memory database | // Just create and run migrations on an in-memory database | ||||
auto repo = dds::catalog::open(":memory:"s); | |||||
auto repo = dds::pkg_db::open(":memory:"s); | |||||
} | } | ||||
TEST_CASE("Open a catalog in a non-ascii path") { | |||||
TEST_CASE("Open a database in a non-ascii path") { | |||||
::setlocale(LC_ALL, ".utf8"); | ::setlocale(LC_ALL, ".utf8"); | ||||
auto THIS_DIR = dds::fs::canonical(__FILE__).parent_path(); | auto THIS_DIR = dds::fs::canonical(__FILE__).parent_path(); | ||||
auto BUILD_DIR | auto BUILD_DIR | ||||
= (THIS_DIR.parent_path().parent_path().parent_path() / "_build").lexically_normal(); | = (THIS_DIR.parent_path().parent_path().parent_path() / "_build").lexically_normal(); | ||||
auto subdir = BUILD_DIR / "Ю́рий Алексе́евич Гага́рин"; | auto subdir = BUILD_DIR / "Ю́рий Алексе́евич Гага́рин"; | ||||
dds::fs::remove_all(subdir); | dds::fs::remove_all(subdir); | ||||
dds::catalog::open(subdir / "test.db"); | |||||
dds::pkg_db::open(subdir / "test.db"); | |||||
dds::fs::remove_all(subdir); | dds::fs::remove_all(subdir); | ||||
} | } | ||||
class catalog_test_case { | class catalog_test_case { | ||||
public: | public: | ||||
dds::catalog db = dds::catalog::open(":memory:"s); | |||||
dds::pkg_db db = dds::pkg_db::open(":memory:"s); | |||||
}; | }; | ||||
TEST_CASE_METHOD(catalog_test_case, "Store a simple package") { | TEST_CASE_METHOD(catalog_test_case, "Store a simple package") { |
#include "./repoman.hpp" | #include "./repoman.hpp" | ||||
#include <dds/catalog/import.hpp> | |||||
#include <dds/package/manifest.hpp> | #include <dds/package/manifest.hpp> | ||||
#include <dds/util/log.hpp> | #include <dds/util/log.hpp> | ||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> |