#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/pkg/get/get.hpp> | #include <dds/pkg/get/get.hpp> | ||||
#include <dds/pkg/info.hpp> | |||||
#include <dds/pkg/listing.hpp> | |||||
#include <dds/repoman/repoman.hpp> | #include <dds/repoman/repoman.hpp> | ||||
#include <dds/util/http/pool.hpp> | #include <dds/util/http/pool.hpp> | ||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
namespace dds::cli::cmd { | namespace dds::cli::cmd { | ||||
static int _repoman_add(const options& opts) { | static int _repoman_add(const options& opts) { | ||||
auto pkg_id = dds::pkg_id::parse(opts.repoman.add.pkg_id_str); | |||||
auto listing = parse_remote_url(opts.repoman.add.url_str); | |||||
dds::pkg_info add_info{ | |||||
auto pkg_id = dds::pkg_id::parse(opts.repoman.add.pkg_id_str); | |||||
auto listing = parse_remote_url(opts.repoman.add.url_str); | |||||
dds::pkg_listing add_info{ | |||||
.ident = pkg_id, | .ident = pkg_id, | ||||
.deps = {}, | .deps = {}, | ||||
.description = opts.repoman.add.description, | .description = opts.repoman.add.description, |
)"); | )"); | ||||
} | } | ||||
void store_with_remote(const neo::sqlite3::statement_cache&, const pkg_info& pkg, std::monostate) { | |||||
void store_with_remote(const neo::sqlite3::statement_cache&, | |||||
const pkg_listing& pkg, | |||||
std::monostate) { | |||||
neo_assert_always( | neo_assert_always( | ||||
invariant, | invariant, | ||||
false, | false, | ||||
} | } | ||||
void store_with_remote(neo::sqlite3::statement_cache& stmts, | void store_with_remote(neo::sqlite3::statement_cache& stmts, | ||||
const pkg_info& pkg, | |||||
const pkg_listing& pkg, | |||||
const http_remote_listing& http) { | const http_remote_listing& http) { | ||||
nsql::exec( // | nsql::exec( // | ||||
stmts(R"( | stmts(R"( | ||||
} | } | ||||
void store_with_remote(neo::sqlite3::statement_cache& stmts, | void store_with_remote(neo::sqlite3::statement_cache& stmts, | ||||
const pkg_info& pkg, | |||||
const pkg_listing& pkg, | |||||
const git_remote_listing& git) { | const git_remote_listing& git) { | ||||
std::string url = git.url; | std::string url = git.url; | ||||
if (url.starts_with("https://") || url.starts_with("http://")) { | if (url.starts_with("https://") || url.starts_with("http://")) { | ||||
void do_store_pkg(neo::sqlite3::database& db, | void do_store_pkg(neo::sqlite3::database& db, | ||||
neo::sqlite3::statement_cache& st_cache, | neo::sqlite3::statement_cache& st_cache, | ||||
const pkg_info& pkg) { | |||||
const pkg_listing& pkg) { | |||||
dds_log(debug, "Recording package {}@{}", pkg.ident.name, pkg.ident.version.to_string()); | dds_log(debug, "Recording package {}@{}", pkg.ident.name, pkg.ident.version.to_string()); | ||||
std::visit([&](auto&& remote) { store_with_remote(st_cache, pkg, remote); }, pkg.remote); | std::visit([&](auto&& remote) { store_with_remote(st_cache, pkg, remote); }, pkg.remote); | ||||
auto db_pkg_id = db.last_insert_rowid(); | auto db_pkg_id = db.last_insert_rowid(); | ||||
pkg_db::pkg_db(nsql::database db) | pkg_db::pkg_db(nsql::database db) | ||||
: _db(std::move(db)) {} | : _db(std::move(db)) {} | ||||
void pkg_db::store(const pkg_info& pkg) { | |||||
void pkg_db::store(const pkg_listing& pkg) { | |||||
nsql::transaction_guard tr{_db}; | nsql::transaction_guard tr{_db}; | ||||
do_store_pkg(_db, _stmt_cache, pkg); | do_store_pkg(_db, _stmt_cache, pkg); | ||||
} | } | ||||
std::optional<pkg_info> pkg_db::get(const pkg_id& pk_id) const noexcept { | |||||
std::optional<pkg_listing> pkg_db::get(const pkg_id& pk_id) const noexcept { | |||||
auto ver_str = pk_id.version.to_string(); | auto ver_str = pk_id.version.to_string(); | ||||
dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str); | dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str); | ||||
auto& st = _stmt_cache(R"( | auto& st = _stmt_cache(R"( | ||||
auto deps = dependencies_of(pk_id); | auto deps = dependencies_of(pk_id); | ||||
auto info = pkg_info{ | |||||
auto info = pkg_listing{ | |||||
pk_id, | pk_id, | ||||
std::move(deps), | std::move(deps), | ||||
std::move(description), | std::move(description), |
#pragma once | #pragma once | ||||
#include "./info.hpp" | |||||
#include "./listing.hpp" | |||||
#include <dds/util/fs.hpp> | #include <dds/util/fs.hpp> | ||||
static fs::path default_path() noexcept; | static fs::path default_path() noexcept; | ||||
void store(const pkg_info& info); | |||||
std::optional<pkg_info> get(const pkg_id& id) const noexcept; | |||||
void store(const pkg_listing& info); | |||||
std::optional<pkg_listing> get(const pkg_id& id) const noexcept; | |||||
std::vector<pkg_id> all() const noexcept; | std::vector<pkg_id> all() const noexcept; | ||||
std::vector<pkg_id> by_name(std::string_view sv) const noexcept; | std::vector<pkg_id> by_name(std::string_view sv) const noexcept; |
}; | }; | ||||
TEST_CASE_METHOD(catalog_test_case, "Store a simple package") { | TEST_CASE_METHOD(catalog_test_case, "Store a simple package") { | ||||
db.store(dds::pkg_info{ | |||||
db.store(dds::pkg_listing{ | |||||
dds::pkg_id("foo", semver::version::parse("1.2.3")), | dds::pkg_id("foo", semver::version::parse("1.2.3")), | ||||
{}, | {}, | ||||
"example", | "example", | ||||
CHECK(std::get<dds::git_remote_listing>(info->remote).ref == "master"); | CHECK(std::get<dds::git_remote_listing>(info->remote).ref == "master"); | ||||
// Update the entry with a new git remote ref | // Update the entry with a new git remote ref | ||||
CHECK_NOTHROW(db.store(dds::pkg_info{ | |||||
CHECK_NOTHROW(db.store(dds::pkg_listing{ | |||||
dds::pkg_id("foo", semver::version::parse("1.2.3")), | dds::pkg_id("foo", semver::version::parse("1.2.3")), | ||||
{}, | {}, | ||||
"example", | "example", | ||||
} | } | ||||
TEST_CASE_METHOD(catalog_test_case, "Package requirements") { | TEST_CASE_METHOD(catalog_test_case, "Package requirements") { | ||||
db.store(dds::pkg_info{ | |||||
db.store(dds::pkg_listing{ | |||||
dds::pkg_id{"foo", semver::version::parse("1.2.3")}, | dds::pkg_id{"foo", semver::version::parse("1.2.3")}, | ||||
{ | { | ||||
{"bar", {semver::version::parse("1.2.3"), semver::version::parse("1.4.0")}}, | {"bar", {semver::version::parse("1.2.3"), semver::version::parse("1.4.0")}}, |
namespace { | namespace { | ||||
temporary_sdist do_pull_sdist(const pkg_info& listing, std::monostate) { | |||||
temporary_sdist do_pull_sdist(const pkg_listing& listing, std::monostate) { | |||||
neo_assert_always( | neo_assert_always( | ||||
invariant, | invariant, | ||||
false, | false, | ||||
} | } | ||||
template <remote_listing R> | template <remote_listing R> | ||||
temporary_sdist do_pull_sdist(const pkg_info& listing, const R& remote) { | |||||
temporary_sdist do_pull_sdist(const pkg_listing& listing, const R& remote) { | |||||
auto tmpdir = dds::temporary_dir::create(); | auto tmpdir = dds::temporary_dir::create(); | ||||
remote.pull_source(tmpdir.path()); | remote.pull_source(tmpdir.path()); | ||||
} // namespace | } // namespace | ||||
temporary_sdist dds::get_package_sdist(const pkg_info& pkg) { | |||||
temporary_sdist dds::get_package_sdist(const pkg_listing& pkg) { | |||||
auto tsd = std::visit([&](auto&& remote) { return do_pull_sdist(pkg, remote); }, pkg.remote); | auto tsd = std::visit([&](auto&& remote) { return do_pull_sdist(pkg, remote); }, pkg.remote); | ||||
if (!(tsd.sdist.manifest.id == pkg.ident)) { | if (!(tsd.sdist.manifest.id == pkg.ident)) { | ||||
throw_external_error<errc::sdist_ident_mismatch>( | throw_external_error<errc::sdist_ident_mismatch>( | ||||
return *info; | return *info; | ||||
}); | }); | ||||
auto okay = parallel_run(absent_pkg_infos, 8, [&](pkg_info inf) { | |||||
auto okay = parallel_run(absent_pkg_infos, 8, [&](pkg_listing inf) { | |||||
dds_log(info, "Download package: {}", inf.ident.to_string()); | dds_log(info, "Download package: {}", inf.ident.to_string()); | ||||
auto tsd = get_package_sdist(inf); | auto tsd = get_package_sdist(inf); | ||||
std::scoped_lock lk{repo_mut}; | std::scoped_lock lk{repo_mut}; |
class pkg_cache; | class pkg_cache; | ||||
class pkg_db; | class pkg_db; | ||||
struct pkg_info; | |||||
struct pkg_listing; | |||||
temporary_sdist get_package_sdist(const pkg_info&); | |||||
temporary_sdist get_package_sdist(const pkg_listing&); | |||||
void get_all(const std::vector<pkg_id>& pkgs, dds::pkg_cache& repo, const pkg_db& cat); | void get_all(const std::vector<pkg_id>& pkgs, dds::pkg_cache& repo, const pkg_db& cat); | ||||
#include "./info.hpp" | |||||
#include "./listing.hpp" | |||||
#include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
#include <dds/util/string.hpp> | #include <dds/util/string.hpp> |
remote_listing_var parse_remote_url(std::string_view url); | remote_listing_var parse_remote_url(std::string_view url); | ||||
struct pkg_info { | |||||
struct pkg_listing { | |||||
pkg_id ident; | pkg_id ident; | ||||
std::vector<dependency> deps; | std::vector<dependency> deps; | ||||
std::string description; | std::string description; |
#include "./repoman.hpp" | #include "./repoman.hpp" | ||||
#include <dds/pkg/info.hpp> | |||||
#include <dds/pkg/listing.hpp> | |||||
#include <dds/sdist/package.hpp> | #include <dds/sdist/package.hpp> | ||||
#include <dds/util/log.hpp> | #include <dds/util/log.hpp> | ||||
#include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
neo::sqlite3::transaction_guard tr{_db}; | neo::sqlite3::transaction_guard tr{_db}; | ||||
dds_log(debug, "Recording package {}@{}", man->id.name, man->id.version.to_string()); | dds_log(debug, "Recording package {}@{}", man->id.name, man->id.version.to_string()); | ||||
dds::pkg_info info{.ident = man->id, | |||||
.deps = man->dependencies, | |||||
.description = "[No description]", | |||||
.remote = {}}; | |||||
auto rel_url = fmt::format("dds:{}", man->id.to_string()); | |||||
dds::pkg_listing info{.ident = man->id, | |||||
.deps = man->dependencies, | |||||
.description = "[No description]", | |||||
.remote = {}}; | |||||
auto rel_url = fmt::format("dds:{}", man->id.to_string()); | |||||
add_pkg(info, rel_url); | add_pkg(info, rel_url); | ||||
auto dest_path = pkg_dir() / man->id.name / man->id.version.to_string() / "sdist.tar.gz"; | auto dest_path = pkg_dir() / man->id.name / man->id.version.to_string() / "sdist.tar.gz"; | ||||
} | } | ||||
} | } | ||||
void repo_manager::add_pkg(const pkg_info& info, std::string_view url) { | |||||
void repo_manager::add_pkg(const pkg_listing& info, std::string_view url) { | |||||
dds_log(info, "Directly add an entry for {}", info.ident.to_string()); | dds_log(info, "Directly add an entry for {}", info.ident.to_string()); | ||||
DDS_E_SCOPE(info.ident); | DDS_E_SCOPE(info.ident); | ||||
nsql::recursive_transaction_guard tr{_db}; | nsql::recursive_transaction_guard tr{_db}; |
namespace dds { | namespace dds { | ||||
struct pkg_info; | |||||
struct pkg_listing; | |||||
struct e_init_repo { | struct e_init_repo { | ||||
fs::path path; | fs::path path; | ||||
void import_targz(path_ref tgz_path); | void import_targz(path_ref tgz_path); | ||||
void delete_package(pkg_id id); | void delete_package(pkg_id id); | ||||
void add_pkg(const pkg_info& info, std::string_view url); | |||||
void add_pkg(const pkg_listing& info, std::string_view url); | |||||
auto all_packages() const noexcept { | auto all_packages() const noexcept { | ||||
using namespace neo::sqlite3::literals; | using namespace neo::sqlite3::literals; |
#include <dds/repoman/repoman.hpp> | #include <dds/repoman/repoman.hpp> | ||||
#include <dds/pkg/info.hpp> | |||||
#include <dds/pkg/listing.hpp> | |||||
#include <dds/temp.hpp> | #include <dds/temp.hpp> | ||||
#include <neo/sqlite3/error.hpp> | #include <neo/sqlite3/error.hpp> | ||||
} | } | ||||
TEST_CASE_METHOD(tmp_repo, "Add a package directly") { | TEST_CASE_METHOD(tmp_repo, "Add a package directly") { | ||||
dds::pkg_info info{ | |||||
dds::pkg_listing info{ | |||||
.ident = dds::pkg_id::parse("foo@1.2.3"), | .ident = dds::pkg_id::parse("foo@1.2.3"), | ||||
.deps = {}, | .deps = {}, | ||||
.description = "Something", | .description = "Something", |