"range-v3@0.11.0", | "range-v3@0.11.0", | ||||
"nlohmann-json@3.7.1", | "nlohmann-json@3.7.1", | ||||
"neo-sqlite3@0.4.1", | "neo-sqlite3@0.4.1", | ||||
"neo-fun~0.5.4", | |||||
"neo-compress~0.1.1", | |||||
"neo-fun~0.6.0", | |||||
"neo-compress~0.2.0", | |||||
"neo-url~0.2.2", | "neo-url~0.2.2", | ||||
"semver@0.2.2", | "semver@0.2.2", | ||||
"pubgrub@0.2.1", | "pubgrub@0.2.1", | ||||
"ctre@2.8.1", | "ctre@2.8.1", | ||||
"fmt^7.0.3", | "fmt^7.0.3", | ||||
"neo-http^0.1.0", | "neo-http^0.1.0", | ||||
"neo-io^0.1.0", | |||||
"neo-io^0.1.1", | |||||
"boost.leaf~0.3.0", | "boost.leaf~0.3.0", | ||||
], | ], | ||||
"test_driver": "Catch-Main" | "test_driver": "Catch-Main" |
catalog_path_flag cat_path{cmd}; | catalog_path_flag cat_path{cmd}; | ||||
args::Flag import_stdin{cmd, "stdin", "Import JSON from stdin", {"stdin"}}; | args::Flag import_stdin{cmd, "stdin", "Import JSON from stdin", {"stdin"}}; | ||||
args::Flag init{cmd, "initial", "Re-import the initial catalog contents", {"initial"}}; | |||||
args::ValueFlagList<std::string> | args::ValueFlagList<std::string> | ||||
json_paths{cmd, | json_paths{cmd, | ||||
"json", | "json", | ||||
int run() { | int run() { | ||||
auto cat = cat_path.open(); | auto cat = cat_path.open(); | ||||
if (init.Get()) { | |||||
cat.import_initial(); | |||||
} | |||||
for (const auto& json_fpath : json_paths.Get()) { | for (const auto& json_fpath : json_paths.Get()) { | ||||
cat.import_json_file(json_fpath); | cat.import_json_file(json_fpath); | ||||
} | } | ||||
auto repo | auto repo | ||||
= dds::repo_manager::create(where.Get(), | = dds::repo_manager::create(where.Get(), | ||||
name ? std::make_optional(name.Get()) : std::nullopt); | name ? std::make_optional(name.Get()) : std::nullopt); | ||||
dds_log(info, "Created new repository '{}' in {}", repo.root(), repo.name()); | |||||
dds_log(info, "Created new repository '{}' in {}", repo.name(), repo.root()); | |||||
return 0; | return 0; | ||||
} | } | ||||
} init{*this}; | } init{*this}; | ||||
args::Flag update{cmd, "update", "Update catalog contents immediately", {"update", 'U'}}; | args::Flag update{cmd, "update", "Update catalog contents immediately", {"update", 'U'}}; | ||||
int run() { | int run() { | ||||
return boost::leaf::try_handle_all( // | |||||
[&]() -> dds::result<int> { | |||||
try { | |||||
auto cat = cat_path.open(); | |||||
auto repo = dds::remote_repository::connect(url.Get()); | |||||
repo.store(cat.database()); | |||||
if (update) { | |||||
repo.update_catalog(cat.database()); | |||||
} | |||||
} catch (...) { | |||||
return dds::capture_exception(); | |||||
} | |||||
return 0; | |||||
}, | |||||
[&](neo::url_validation_error url_err, dds::e_url_string bad_url) { | |||||
dds_log(error, "Invalid URL [{}]: {}", bad_url.value, url_err.what()); | |||||
return 1; | |||||
}, | |||||
[&](const json5::parse_error& e, dds::e_http_url bad_url) { | |||||
dds_log(error, | |||||
"Error parsing JSON downloaded from URL [{}]: {}", | |||||
bad_url.value, | |||||
e.what()); | |||||
return 1; | |||||
}, | |||||
[](dds::e_sqlite3_error_exc e, dds::e_url_string url) { | |||||
dds_log(error, | |||||
"Error accessing remote database (From {}): {}", | |||||
url.value, | |||||
e.message); | |||||
return 1; | |||||
}, | |||||
[](dds::e_sqlite3_error_exc e) { | |||||
dds_log(error, "Unexpected database error: {}", e.message); | |||||
return 1; | |||||
}, | |||||
[&](dds::e_system_error_exc e, dds::e_http_connect conn) { | |||||
dds_log(error, | |||||
"Error opening connection to [{}:{}]: {}", | |||||
conn.host, | |||||
conn.port, | |||||
e.message); | |||||
return 1; | |||||
}, | |||||
[](const std::exception& e) { | |||||
dds_log(error, "An unknown unhandled exception occurred: {}", e.what()); | |||||
return 1; | |||||
}, | |||||
[](dds::e_system_error_exc e) { | |||||
dds_log(error, "An unknown system_error occurred: {}", e.message); | |||||
return 42; | |||||
}, | |||||
[](boost::leaf::diagnostic_info const& info) { | |||||
dds_log(error, "An unnknown error occurred? {}", info); | |||||
return 42; | |||||
}); | |||||
auto cat = cat_path.open(); | |||||
auto repo = dds::remote_repository::connect(url.Get()); | |||||
repo.store(cat.database()); | |||||
if (update) { | |||||
repo.update_catalog(cat.database()); | |||||
} | |||||
return 0; | |||||
} | } | ||||
} add{*this}; | } add{*this}; | ||||
struct { | |||||
cli_repo& parent; | |||||
args::Command cmd{parent.repo_group, "update", "Update remote package information"}; | |||||
common_flags _flags{cmd}; | |||||
catalog_path_flag cat_path{cmd}; | |||||
int run() { | |||||
auto cat = cat_path.open(); | |||||
dds::update_all_remotes(cat.database()); | |||||
return 0; | |||||
} | |||||
} update{*this}; | |||||
struct { | struct { | ||||
cli_repo& parent; | cli_repo& parent; | ||||
args::Command cmd{parent.repo_group, "init", "Initialize a directory as a repository"}; | args::Command cmd{parent.repo_group, "init", "Initialize a directory as a repository"}; | ||||
} | } | ||||
} init{*this}; | } init{*this}; | ||||
int run() { | |||||
int _run() { | |||||
if (ls.cmd) { | if (ls.cmd) { | ||||
return ls.run(); | return ls.run(); | ||||
} else if (init.cmd) { | } else if (init.cmd) { | ||||
return import_.run(); | return import_.run(); | ||||
} else if (add.cmd) { | } else if (add.cmd) { | ||||
return add.run(); | return add.run(); | ||||
} else if (update.cmd) { | |||||
return update.run(); | |||||
} else { | } else { | ||||
assert(false); | assert(false); | ||||
std::terminate(); | std::terminate(); | ||||
} | } | ||||
} | } | ||||
int run() { | |||||
return boost::leaf::try_handle_all( // | |||||
[&]() -> dds::result<int> { | |||||
try { | |||||
return _run(); | |||||
} catch (...) { | |||||
return dds::capture_exception(); | |||||
} | |||||
return 0; | |||||
}, | |||||
[&](neo::url_validation_error url_err, dds::e_url_string bad_url) { | |||||
dds_log(error, "Invalid URL [{}]: {}", bad_url.value, url_err.what()); | |||||
return 1; | |||||
}, | |||||
[&](const json5::parse_error& e, dds::e_http_url bad_url) { | |||||
dds_log(error, | |||||
"Error parsing JSON downloaded from URL [{}]: {}", | |||||
bad_url.value, | |||||
e.what()); | |||||
return 1; | |||||
}, | |||||
[](dds::e_sqlite3_error_exc e, dds::e_url_string url) { | |||||
dds_log(error, | |||||
"Error accessing remote database (From {}): {}", | |||||
url.value, | |||||
e.message); | |||||
return 1; | |||||
}, | |||||
[](dds::e_sqlite3_error_exc e) { | |||||
dds_log(error, "Unexpected database error: {}", e.message); | |||||
return 1; | |||||
}, | |||||
[&](dds::e_system_error_exc e, dds::e_http_connect conn) { | |||||
dds_log(error, | |||||
"Error opening connection to [{}:{}]: {}", | |||||
conn.host, | |||||
conn.port, | |||||
e.message); | |||||
return 1; | |||||
}, | |||||
[](const std::exception& e) { | |||||
dds_log(error, "An unknown unhandled exception occurred: {}", e.what()); | |||||
return 1; | |||||
}, | |||||
[](dds::e_system_error_exc e) { | |||||
dds_log(error, "An unknown system_error occurred: {}", e.message); | |||||
return 42; | |||||
}, | |||||
[](boost::leaf::diagnostic_info const& info) { | |||||
dds_log(error, "An unnknown error occurred? {}", info); | |||||
return 42; | |||||
}); | |||||
} | |||||
}; | }; | ||||
/* | /* |
#include "./import.hpp" | #include "./import.hpp" | ||||
#include <dds/catalog/init_catalog.hpp> | |||||
#include <dds/dym.hpp> | #include <dds/dym.hpp> | ||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/solve/solve.hpp> | #include <dds/solve/solve.hpp> | ||||
version TEXT NOT NULL, | version TEXT NOT NULL, | ||||
description TEXT NOT NULL, | description TEXT NOT NULL, | ||||
remote_url TEXT NOT NULL, | remote_url TEXT NOT NULL, | ||||
remote_id INTEGER REFERENCES dds_cat_remotes DEFAULT NULL, | |||||
remote_id INTEGER | |||||
REFERENCES dds_cat_remotes | |||||
ON DELETE CASCADE, | |||||
repo_transform TEXT NOT NULL DEFAULT '[]', | repo_transform TEXT NOT NULL DEFAULT '[]', | ||||
UNIQUE (name, version) | |||||
UNIQUE (name, version, remote_id) | |||||
); | ); | ||||
INSERT INTO dds_cat_pkgs_new(pkg_id, | INSERT INTO dds_cat_pkgs_new(pkg_id, | ||||
CREATE TABLE dds_cat_pkg_deps_new ( | CREATE TABLE dds_cat_pkg_deps_new ( | ||||
dep_id INTEGER PRIMARY KEY AUTOINCREMENT, | dep_id INTEGER PRIMARY KEY AUTOINCREMENT, | ||||
pkg_id INTEGER NOT NULL REFERENCES dds_cat_pkgs_new(pkg_id), | |||||
pkg_id INTEGER | |||||
NOT NULL | |||||
REFERENCES dds_cat_pkgs_new(pkg_id) | |||||
ON DELETE CASCADE, | |||||
dep_name TEXT NOT NULL, | dep_name TEXT NOT NULL, | ||||
low TEXT NOT NULL, | low TEXT NOT NULL, | ||||
high TEXT NOT NULL, | high TEXT NOT NULL, | ||||
} | } | ||||
} | } | ||||
void store_init_packages(nsql::database& db, nsql::statement_cache& st_cache) { | |||||
dds_log(debug, "Restoring initial package data"); | |||||
for (auto& pkg : init_catalog_packages()) { | |||||
do_store_pkg(db, st_cache, pkg); | |||||
} | |||||
} | |||||
void ensure_migrated(nsql::database& db) { | void ensure_migrated(nsql::database& db) { | ||||
db.exec(R"( | db.exec(R"( | ||||
PRAGMA foreign_keys = 1; | PRAGMA foreign_keys = 1; | ||||
int version = version_; | int version = version_; | ||||
// If this is the first time we're working here, import the initial | |||||
// catalog with some useful tidbits. | |||||
bool import_init_packages = version == 0; | |||||
if (version > current_database_version) { | if (version > current_database_version) { | ||||
dds_log(critical, | dds_log(critical, | ||||
"Catalog version is {}, but we only support up to {}", | "Catalog version is {}, but we only support up to {}", | ||||
} | } | ||||
meta["version"] = current_database_version; | meta["version"] = current_database_version; | ||||
exec(db.prepare("UPDATE dds_cat_meta SET meta=?"), meta.dump()); | exec(db.prepare("UPDATE dds_cat_meta SET meta=?"), meta.dump()); | ||||
if (import_init_packages) { | |||||
dds_log( | |||||
info, | |||||
"A new catalog database case been created, and has been populated with some initial " | |||||
"contents."); | |||||
neo::sqlite3::statement_cache stmts{db}; | |||||
store_init_packages(db, stmts); | |||||
} | |||||
} | } | ||||
void check_json(bool b, std::string_view what) { | void check_json(bool b, std::string_view what) { | ||||
description, | description, | ||||
repo_transform | repo_transform | ||||
FROM dds_cat_pkgs | FROM dds_cat_pkgs | ||||
WHERE name = ? AND version = ? | |||||
WHERE name = ?1 AND version = ?2 | |||||
ORDER BY pkg_id DESC | |||||
)"_sql); | )"_sql); | ||||
st.reset(); | st.reset(); | ||||
st.bindings() = std::forward_as_tuple(pk_id.name, ver_str); | st.bindings() = std::forward_as_tuple(pk_id.name, ver_str); | ||||
auto opt_tup = nsql::unpack_single_opt<std::int64_t, | |||||
std::string, | |||||
std::string, | |||||
std::string, | |||||
std::string, | |||||
std::string>(st); | |||||
if (!opt_tup) { | |||||
auto ec = st.step(std::nothrow); | |||||
if (ec == nsql::errc::done) { | |||||
dym_target::fill([&] { | dym_target::fill([&] { | ||||
auto all_ids = this->all(); | auto all_ids = this->all(); | ||||
auto id_strings | auto id_strings | ||||
}); | }); | ||||
return std::nullopt; | return std::nullopt; | ||||
} | } | ||||
const auto& [pkg_id, name, version, remote_url, description, repo_transform] = *opt_tup; | |||||
assert(pk_id.name == name); | |||||
assert(pk_id.version == semver::version::parse(version)); | |||||
neo_assert_always(invariant, | |||||
ec == nsql::errc::row, | |||||
"Failed to pull a package from the catalog database", | |||||
ec, | |||||
pk_id.to_string(), | |||||
nsql::error_category().message(int(ec))); | |||||
const auto& [pkg_id, name, version, remote_url, description, repo_transform] | |||||
= st.row() | |||||
.unpack<std::int64_t, | |||||
std::string, | |||||
std::string, | |||||
std::string, | |||||
std::string, | |||||
std::string>(); | |||||
ec = st.step(std::nothrow); | |||||
if (ec == nsql::errc::row) { | |||||
dds_log(warn, | |||||
"There is more than one entry for package {} in the catalog database. One will be " | |||||
"chosen arbitrarily.", | |||||
pk_id.to_string()); | |||||
} | |||||
neo_assert(invariant, | |||||
pk_id.name == name && pk_id.version == semver::version::parse(version), | |||||
"Package metadata does not match", | |||||
pk_id.to_string(), | |||||
name, | |||||
version); | |||||
auto deps = dependencies_of(pk_id); | auto deps = dependencies_of(pk_id); | ||||
SELECT name, version | SELECT name, version | ||||
FROM dds_cat_pkgs | FROM dds_cat_pkgs | ||||
WHERE name = ? | WHERE name = ? | ||||
ORDER BY pkg_id DESC | |||||
)"_sql), | )"_sql), | ||||
sv) // | sv) // | ||||
| neo::lref // | | neo::lref // | ||||
do_store_pkg(_db, _stmt_cache, pkg); | do_store_pkg(_db, _stmt_cache, pkg); | ||||
} | } | ||||
} | } | ||||
void catalog::import_initial() { | |||||
nsql::transaction_guard tr{_db}; | |||||
dds_log(info, "Restoring built-in initial catalog contents"); | |||||
store_init_packages(_db, _stmt_cache); | |||||
} |
std::vector<package_id> by_name(std::string_view sv) const noexcept; | std::vector<package_id> by_name(std::string_view sv) const noexcept; | ||||
std::vector<dependency> dependencies_of(const package_id& pkg) const noexcept; | std::vector<dependency> dependencies_of(const package_id& pkg) const noexcept; | ||||
void import_initial(); | |||||
void import_json_str(std::string_view json_str); | void import_json_str(std::string_view json_str); | ||||
void import_json_file(path_ref json_path) { | void import_json_file(path_ref json_path) { | ||||
auto content = dds::slurp_file(json_path); | auto content = dds::slurp_file(json_path); |
#pragma once | |||||
#include "./package_info.hpp" | |||||
#include <vector> | |||||
namespace dds { | |||||
const std::vector<package_info>& init_catalog_packages() noexcept; | |||||
} // namespace dds |
#include "./package_info.hpp" | #include "./package_info.hpp" | ||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/util/string.hpp> | |||||
#include <neo/url.hpp> | #include <neo/url.hpp> | ||||
#include <neo/utility.hpp> | #include <neo/utility.hpp> | ||||
return git_remote_listing::from_url(sv); | return git_remote_listing::from_url(sv); | ||||
} else if (url.scheme == neo::oper::any_of("http", "https")) { | } else if (url.scheme == neo::oper::any_of("http", "https")) { | ||||
return http_remote_listing::from_url(sv); | return http_remote_listing::from_url(sv); | ||||
} else if (url.scheme == neo::oper::any_of("dds+http", "dds+https", "http+dds", "https+dds")) { | |||||
fs::path path = url.path; | |||||
auto leaf = path.filename().string(); | |||||
auto namever_path = replace(leaf, "@", "/"); | |||||
url.path = (path.parent_path() / "pkg" / namever_path / "sdist.tar.gz").generic_string(); | |||||
return http_remote_listing::from_url(url.to_string()); | |||||
} else { | } else { | ||||
throw_user_error< | throw_user_error< | ||||
errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'", | errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'", |
unsigned int strip_components = 1; | unsigned int strip_components = 1; | ||||
std::optional<lm::usage> auto_lib; | std::optional<lm::usage> auto_lib; | ||||
// IF we are a dds+ URL, strip_components should be zero, and give the url a plain | |||||
// HTTP/HTTPS scheme | |||||
if (url.scheme.starts_with("dds+")) { | |||||
url.scheme = url.scheme.substr(4); | |||||
strip_components = 0; | |||||
} else if (url.scheme.ends_with("+dds")) { | |||||
url.scheme.erase(url.scheme.end() - 3); | |||||
strip_components = 0; | |||||
} else { | |||||
// Leave the URL as-is | |||||
} | |||||
if (url.query) { | if (url.query) { | ||||
neo::basic_query_string_view qsv{*url.query}; | neo::basic_query_string_view qsv{*url.query}; | ||||
for (auto qstr : qsv) { | for (auto qstr : qsv) { |
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/http/session.hpp> | #include <dds/http/session.hpp> | ||||
#include <dds/temp.hpp> | #include <dds/temp.hpp> | ||||
#include <dds/util/log.hpp> | |||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
#include <neo/event.hpp> | |||||
#include <neo/scope.hpp> | |||||
#include <neo/sqlite3/exec.hpp> | #include <neo/sqlite3/exec.hpp> | ||||
#include <neo/sqlite3/iter_tuples.hpp> | |||||
#include <neo/sqlite3/single.hpp> | #include <neo/sqlite3/single.hpp> | ||||
#include <neo/sqlite3/transaction.hpp> | #include <neo/sqlite3/transaction.hpp> | ||||
#include <neo/url.hpp> | #include <neo/url.hpp> | ||||
#include <neo/utility.hpp> | #include <neo/utility.hpp> | ||||
#include <range/v3/range/conversion.hpp> | |||||
using namespace dds; | using namespace dds; | ||||
namespace nsql = neo::sqlite3; | namespace nsql = neo::sqlite3; | ||||
url.host.has_value(), | url.host.has_value(), | ||||
"URL does not have a hostname??", | "URL does not have a hostname??", | ||||
url.to_string()); | url.to_string()); | ||||
auto sess = url.scheme == "https" | |||||
? http_session::connect_ssl(*url.host, url.port_or_default_port_or(443)) | |||||
: http_session::connect(*url.host, url.port_or_default_port_or(80)); | |||||
auto sess = http_session::connect_for(url); | |||||
auto tempdir = temporary_dir::create(); | auto tempdir = temporary_dir::create(); | ||||
auto repo_db_dl = tempdir.path() / "repo.db"; | auto repo_db_dl = tempdir.path() / "repo.db"; | ||||
auto name_st = db.db.prepare("SELECT name FROM dds_repo_meta"); | auto name_st = db.db.prepare("SELECT name FROM dds_repo_meta"); | ||||
auto [name] = nsql::unpack_single<std::string>(name_st); | auto [name] = nsql::unpack_single<std::string>(name_st); | ||||
remote_repository ret; | |||||
ret._base_url = url; | |||||
ret._name = name; | |||||
return ret; | |||||
return {name, url}; | |||||
} | } | ||||
void remote_repository::store(nsql::database_ref db) { | void remote_repository::store(nsql::database_ref db) { | ||||
auto st = db.prepare(R"( | auto st = db.prepare(R"( | ||||
INSERT INTO dds_cat_remotes (name, gen_ident, remote_url) | INSERT INTO dds_cat_remotes (name, gen_ident, remote_url) | ||||
VALUES (?, ?, ?) | VALUES (?, ?, ?) | ||||
ON CONFLICT (name) DO | |||||
UPDATE SET gen_ident = ?2, remote_url = ?3 | |||||
)"); | )"); | ||||
nsql::exec(st, _name, "[placeholder]", _base_url.to_string()); | nsql::exec(st, _name, "[placeholder]", _base_url.to_string()); | ||||
} | } | ||||
void remote_repository::update_catalog(nsql::database_ref db) { | void remote_repository::update_catalog(nsql::database_ref db) { | ||||
dds_log(info, "Pulling repository contents for {} [{}]", _name, _base_url.to_string()); | |||||
auto rdb = remote_db::download_and_open_for_base(_base_url); | auto rdb = remote_db::download_and_open_for_base(_base_url); | ||||
auto base_url_str = _base_url.to_string(); | |||||
while (base_url_str.ends_with("/")) { | |||||
base_url_str.pop_back(); | |||||
} | |||||
auto db_path = rdb._tempdir.path() / "repo.db"; | auto db_path = rdb._tempdir.path() / "repo.db"; | ||||
auto rid_st = db.prepare("SELECT remote_id FROM dds_cat_remotes WHERE name = ?"); | auto rid_st = db.prepare("SELECT remote_id FROM dds_cat_remotes WHERE name = ?"); | ||||
rid_st.bindings()[1] = _name; | rid_st.bindings()[1] = _name; | ||||
auto [remote_id] = nsql::unpack_single<std::int64_t>(rid_st); | auto [remote_id] = nsql::unpack_single<std::int64_t>(rid_st); | ||||
rid_st.reset(); | |||||
nsql::transaction_guard tr{db}; | |||||
nsql::exec(db.prepare("ATTACH DATABASE ? AS remote"), db_path.string()); | nsql::exec(db.prepare("ATTACH DATABASE ? AS remote"), db_path.string()); | ||||
neo_defer { db.exec("DETACH DATABASE remote"); }; | |||||
nsql::transaction_guard tr{db}; | |||||
nsql::exec( // | nsql::exec( // | ||||
db.prepare(R"( | db.prepare(R"( | ||||
DELETE FROM dds_cat_pkgs | DELETE FROM dds_cat_pkgs | ||||
name, | name, | ||||
version, | version, | ||||
description, | description, | ||||
printf('dds:%s/%s', name, version), | |||||
CASE | |||||
WHEN url LIKE 'dds:%@%' THEN | |||||
-- Convert 'dds:name@ver' to 'dds+<base-repo-url>/name@ver' | |||||
-- This will later resolve to the actual package URL | |||||
printf('dds+%s/%s', ?2, substr(url, 5)) | |||||
ELSE | |||||
-- Non-'dds:' URLs are kept as-is | |||||
url | |||||
END, | |||||
?1 | ?1 | ||||
FROM remote.dds_repo_packages | FROM remote.dds_repo_packages | ||||
)"), | )"), | ||||
remote_id); | |||||
remote_id, | |||||
base_url_str); | |||||
db.exec(R"( | |||||
INSERT OR REPLACE INTO dds_cat_pkg_deps (pkg_id, dep_name, low, high) | |||||
SELECT | |||||
local_pkgs.pkg_id AS pkg_id, | |||||
dep_name, | |||||
low, | |||||
high | |||||
FROM remote.dds_repo_package_deps AS deps, | |||||
remote.dds_repo_packages AS pkgs USING(package_id), | |||||
dds_cat_pkgs AS local_pkgs USING(name, version) | |||||
)"); | |||||
// Validate our database | |||||
auto fk_check = db.prepare("PRAGMA foreign_key_check"); | |||||
auto rows = nsql::iter_tuples<std::string, std::int64_t, std::string, std::string>(fk_check); | |||||
bool any_failed = false; | |||||
for (auto [child_table, rowid, parent_table, failed_idx] : rows) { | |||||
dds_log( | |||||
critical, | |||||
"Database foreign_key error after import: {0}.{3} referencing {2} violated at row {1}", | |||||
child_table, | |||||
rowid, | |||||
parent_table, | |||||
failed_idx); | |||||
any_failed = true; | |||||
} | |||||
auto int_check = db.prepare("PRAGMA main.integrity_check"); | |||||
for (auto [error] : nsql::iter_tuples<std::string>(int_check)) { | |||||
if (error == "ok") { | |||||
continue; | |||||
} | |||||
dds_log(critical, "Database errors after import: {}", error); | |||||
any_failed = true; | |||||
} | |||||
if (any_failed) { | |||||
throw_external_error<errc::corrupted_catalog_db>( | |||||
"Database update failed due to data integrity errors"); | |||||
} | |||||
} | |||||
void dds::update_all_remotes(nsql::database_ref db) { | |||||
dds_log(info, "Updating catalog from all remotes"); | |||||
auto repos_st = db.prepare("SELECT name, remote_url FROM dds_cat_remotes"); | |||||
auto tups = nsql::iter_tuples<std::string, std::string>(repos_st) | ranges::to_vector; | |||||
for (const auto& [name, remote_url] : tups) { | |||||
DDS_E_SCOPE(e_url_string{remote_url}); | |||||
remote_repository repo{name, neo::url::parse(remote_url)}; | |||||
repo.update_catalog(db); | |||||
} | |||||
dds_log(info, "Recompacting database..."); | |||||
db.exec("VACUUM"); | |||||
} | } |
std::string _name; | std::string _name; | ||||
neo::url _base_url; | neo::url _base_url; | ||||
remote_repository(std::string name, neo::url url) | |||||
: _name(std::move(name)) | |||||
, _base_url(std::move(url)) {} | |||||
remote_repository() = default; | remote_repository() = default; | ||||
public: | public: | ||||
static remote_repository connect(std::string_view url); | static remote_repository connect(std::string_view url); | ||||
// const repository_manifest& manifest() const noexcept; | |||||
void store(neo::sqlite3::database_ref); | void store(neo::sqlite3::database_ref); | ||||
void update_catalog(neo::sqlite3::database_ref); | void update_catalog(neo::sqlite3::database_ref); | ||||
}; | }; | ||||
void update_all_remotes(neo::sqlite3::database_ref); | |||||
} // namespace dds | } // namespace dds |
#include "./repoman.hpp" | #include "./repoman.hpp" | ||||
#include <dds/catalog/import.hpp> | |||||
#include <dds/package/manifest.hpp> | #include <dds/package/manifest.hpp> | ||||
#include <dds/util/log.hpp> | #include <dds/util/log.hpp> | ||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
name TEXT NOT NULL, | name TEXT NOT NULL, | ||||
version TEXT NOT NULL, | version TEXT NOT NULL, | ||||
description TEXT NOT NULL, | description TEXT NOT NULL, | ||||
url TEXT NOT NULL, | |||||
UNIQUE (name, version) | UNIQUE (name, version) | ||||
); | ); | ||||
void ensure_migrated(nsql::database_ref db, std::optional<std::string_view> name) { | void ensure_migrated(nsql::database_ref db, std::optional<std::string_view> name) { | ||||
db.exec(R"( | db.exec(R"( | ||||
PRAGMA busy_timeout = 6000; | |||||
PRAGMA foreign_keys = 1; | PRAGMA foreign_keys = 1; | ||||
CREATE TABLE IF NOT EXISTS dds_repo_meta ( | CREATE TABLE IF NOT EXISTS dds_repo_meta ( | ||||
meta_version INTEGER DEFAULT 1, | meta_version INTEGER DEFAULT 1, | ||||
DDS_E_SCOPE(e_init_repo_db{db_path}); | DDS_E_SCOPE(e_init_repo_db{db_path}); | ||||
DDS_E_SCOPE(e_open_repo_db{db_path}); | DDS_E_SCOPE(e_open_repo_db{db_path}); | ||||
ensure_migrated(db, name); | ensure_migrated(db, name); | ||||
fs::create_directories(directory / "data"); | |||||
fs::create_directories(directory / "pkg"); | |||||
} | } | ||||
return open(directory); | return open(directory); | ||||
} | } | ||||
dds_log(debug, "Recording package {}@{}", man->pkg_id.name, man->pkg_id.version.to_string()); | dds_log(debug, "Recording package {}@{}", man->pkg_id.name, man->pkg_id.version.to_string()); | ||||
nsql::exec( // | nsql::exec( // | ||||
_stmts(R"( | _stmts(R"( | ||||
INSERT INTO dds_repo_packages (name, version, description) | |||||
VALUES (?, ?, 'No description') | |||||
INSERT INTO dds_repo_packages (name, version, description, url) | |||||
VALUES ( | |||||
?1, | |||||
?2, | |||||
'No description', | |||||
printf('dds:%s@%s', ?1, ?2) | |||||
) | |||||
)"_sql), | )"_sql), | ||||
man->pkg_id.name, | man->pkg_id.name, | ||||
man->pkg_id.version.to_string()); | man->pkg_id.version.to_string()); | ||||
iv_1.high.to_string()); | iv_1.high.to_string()); | ||||
} | } | ||||
auto dest_dir = data_dir() / man->pkg_id.name; | |||||
auto dest_path = dest_dir / fmt::format("{}.tar.gz", man->pkg_id.version.to_string()); | |||||
fs::create_directories(dest_dir); | |||||
auto dest_path | |||||
= pkg_dir() / man->pkg_id.name / man->pkg_id.version.to_string() / "sdist.tar.gz"; | |||||
fs::create_directories(dest_path.parent_path()); | |||||
fs::copy(tgz_file, dest_path); | fs::copy(tgz_file, dest_path); | ||||
tr.commit(); | tr.commit(); | ||||
pkg_id.version.to_string()); | pkg_id.version.to_string()); | ||||
/// XXX: Verify with _db.changes() that we actually deleted one row | /// XXX: Verify with _db.changes() that we actually deleted one row | ||||
auto name_dir = data_dir() / pkg_id.name; | |||||
auto ver_file = name_dir / fmt::format("{}.tar.gz", pkg_id.version.to_string()); | |||||
auto name_dir = pkg_dir() / pkg_id.name; | |||||
auto ver_dir = name_dir / pkg_id.version.to_string(); | |||||
DDS_E_SCOPE(e_repo_delete_targz{ver_file}); | |||||
DDS_E_SCOPE(e_repo_delete_targz{ver_dir}); | |||||
if (!fs::is_regular_file(ver_file)) { | |||||
if (!fs::is_directory(ver_dir)) { | |||||
throw std::system_error(std::make_error_code(std::errc::no_such_file_or_directory), | throw std::system_error(std::make_error_code(std::errc::no_such_file_or_directory), | ||||
"No source archive for the requested package"); | "No source archive for the requested package"); | ||||
} | } | ||||
fs::remove(ver_file); | |||||
fs::remove_all(ver_dir); | |||||
tr.commit(); | tr.commit(); | ||||
static repo_manager create(path_ref directory, std::optional<std::string_view> name); | static repo_manager create(path_ref directory, std::optional<std::string_view> name); | ||||
static repo_manager open(path_ref directory); | static repo_manager open(path_ref directory); | ||||
auto data_dir() const noexcept { return _root / "data"; } | |||||
auto pkg_dir() const noexcept { return _root / "pkg"; } | |||||
path_ref root() const noexcept { return _root; } | path_ref root() const noexcept { return _root; } | ||||
std::string name() const noexcept; | std::string name() const noexcept; | ||||
} // namespace | } // namespace | ||||
TEST_CASE("Open a repository") { | |||||
TEST_CASE("Open and import into a repository") { | |||||
auto tdir = dds::temporary_dir::create(); | auto tdir = dds::temporary_dir::create(); | ||||
auto repo = dds::repo_manager::create(tdir.path(), "test-repo"); | auto repo = dds::repo_manager::create(tdir.path(), "test-repo"); | ||||
auto neo_url_tgz = DATA_DIR / "neo-url@0.2.1.tar.gz"; | auto neo_url_tgz = DATA_DIR / "neo-url@0.2.1.tar.gz"; | ||||
repo.import_targz(neo_url_tgz); | repo.import_targz(neo_url_tgz); | ||||
CHECK(dds::fs::is_directory(repo.data_dir() / "neo-url/")); | |||||
CHECK(dds::fs::is_regular_file(repo.data_dir() / "neo-url/0.2.1.tar.gz")); | |||||
CHECK(dds::fs::is_directory(repo.pkg_dir() / "neo-url/")); | |||||
CHECK(dds::fs::is_regular_file(repo.pkg_dir() / "neo-url/0.2.1/sdist.tar.gz")); | |||||
CHECK_THROWS_AS(repo.import_targz(neo_url_tgz), neo::sqlite3::constraint_unique_error); | CHECK_THROWS_AS(repo.import_targz(neo_url_tgz), neo::sqlite3::constraint_unique_error); | ||||
repo.delete_package(dds::package_id::parse("neo-url@0.2.1")); | repo.delete_package(dds::package_id::parse("neo-url@0.2.1")); | ||||
CHECK_FALSE(dds::fs::is_regular_file(repo.data_dir() / "neo-url/0.2.1.tar.gz")); | |||||
CHECK_FALSE(dds::fs::is_directory(repo.data_dir() / "neo-url")); | |||||
CHECK_FALSE(dds::fs::is_regular_file(repo.pkg_dir() / "neo-url/0.2.1/sdist.tar.gz")); | |||||
CHECK_FALSE(dds::fs::is_directory(repo.pkg_dir() / "neo-url")); | |||||
CHECK_THROWS_AS(repo.delete_package(dds::package_id::parse("neo-url@0.2.1")), | CHECK_THROWS_AS(repo.delete_package(dds::package_id::parse("neo-url@0.2.1")), | ||||
std::system_error); | std::system_error); | ||||
CHECK_NOTHROW(repo.import_targz(neo_url_tgz)); | CHECK_NOTHROW(repo.import_targz(neo_url_tgz)); |
{ | { | ||||
"version": 2, | "version": 2, | ||||
"packages": { | "packages": { | ||||
"neo-sqlite3": { | |||||
"0.1.0": { | |||||
"url": "git+https://github.com/vector-of-bool/neo-sqlite3.git#0.1.0" | |||||
}, | |||||
"0.2.2": { | |||||
"url": "git+https://github.com/vector-of-bool/neo-sqlite3.git#0.2.2" | |||||
}, | |||||
"neo-fun": { | |||||
"0.3.0": { | "0.3.0": { | ||||
"url": "git+https://github.com/vector-of-bool/neo-sqlite3.git#0.3.0" | |||||
"url": "git+https://github.com/vector-of-bool/neo-fun.git#0.3.0" | |||||
} | } | ||||
} | } | ||||
} | } |