| #include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
| #include <dds/pkg/get/get.hpp> | #include <dds/pkg/get/get.hpp> | ||||
| #include <dds/pkg/info.hpp> | |||||
| #include <dds/pkg/listing.hpp> | |||||
| #include <dds/repoman/repoman.hpp> | #include <dds/repoman/repoman.hpp> | ||||
| #include <dds/util/http/pool.hpp> | #include <dds/util/http/pool.hpp> | ||||
| #include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
| namespace dds::cli::cmd { | namespace dds::cli::cmd { | ||||
| static int _repoman_add(const options& opts) { | static int _repoman_add(const options& opts) { | ||||
| auto pkg_id = dds::pkg_id::parse(opts.repoman.add.pkg_id_str); | |||||
| auto listing = parse_remote_url(opts.repoman.add.url_str); | |||||
| dds::pkg_info add_info{ | |||||
| auto pkg_id = dds::pkg_id::parse(opts.repoman.add.pkg_id_str); | |||||
| auto listing = parse_remote_url(opts.repoman.add.url_str); | |||||
| dds::pkg_listing add_info{ | |||||
| .ident = pkg_id, | .ident = pkg_id, | ||||
| .deps = {}, | .deps = {}, | ||||
| .description = opts.repoman.add.description, | .description = opts.repoman.add.description, |
| )"); | )"); | ||||
| } | } | ||||
| void store_with_remote(const neo::sqlite3::statement_cache&, const pkg_info& pkg, std::monostate) { | |||||
| void store_with_remote(const neo::sqlite3::statement_cache&, | |||||
| const pkg_listing& pkg, | |||||
| std::monostate) { | |||||
| neo_assert_always( | neo_assert_always( | ||||
| invariant, | invariant, | ||||
| false, | false, | ||||
| } | } | ||||
| void store_with_remote(neo::sqlite3::statement_cache& stmts, | void store_with_remote(neo::sqlite3::statement_cache& stmts, | ||||
| const pkg_info& pkg, | |||||
| const pkg_listing& pkg, | |||||
| const http_remote_listing& http) { | const http_remote_listing& http) { | ||||
| nsql::exec( // | nsql::exec( // | ||||
| stmts(R"( | stmts(R"( | ||||
| } | } | ||||
| void store_with_remote(neo::sqlite3::statement_cache& stmts, | void store_with_remote(neo::sqlite3::statement_cache& stmts, | ||||
| const pkg_info& pkg, | |||||
| const pkg_listing& pkg, | |||||
| const git_remote_listing& git) { | const git_remote_listing& git) { | ||||
| std::string url = git.url; | std::string url = git.url; | ||||
| if (url.starts_with("https://") || url.starts_with("http://")) { | if (url.starts_with("https://") || url.starts_with("http://")) { | ||||
| void do_store_pkg(neo::sqlite3::database& db, | void do_store_pkg(neo::sqlite3::database& db, | ||||
| neo::sqlite3::statement_cache& st_cache, | neo::sqlite3::statement_cache& st_cache, | ||||
| const pkg_info& pkg) { | |||||
| const pkg_listing& pkg) { | |||||
| dds_log(debug, "Recording package {}@{}", pkg.ident.name, pkg.ident.version.to_string()); | dds_log(debug, "Recording package {}@{}", pkg.ident.name, pkg.ident.version.to_string()); | ||||
| std::visit([&](auto&& remote) { store_with_remote(st_cache, pkg, remote); }, pkg.remote); | std::visit([&](auto&& remote) { store_with_remote(st_cache, pkg, remote); }, pkg.remote); | ||||
| auto db_pkg_id = db.last_insert_rowid(); | auto db_pkg_id = db.last_insert_rowid(); | ||||
| pkg_db::pkg_db(nsql::database db) | pkg_db::pkg_db(nsql::database db) | ||||
| : _db(std::move(db)) {} | : _db(std::move(db)) {} | ||||
| void pkg_db::store(const pkg_info& pkg) { | |||||
| void pkg_db::store(const pkg_listing& pkg) { | |||||
| nsql::transaction_guard tr{_db}; | nsql::transaction_guard tr{_db}; | ||||
| do_store_pkg(_db, _stmt_cache, pkg); | do_store_pkg(_db, _stmt_cache, pkg); | ||||
| } | } | ||||
| std::optional<pkg_info> pkg_db::get(const pkg_id& pk_id) const noexcept { | |||||
| std::optional<pkg_listing> pkg_db::get(const pkg_id& pk_id) const noexcept { | |||||
| auto ver_str = pk_id.version.to_string(); | auto ver_str = pk_id.version.to_string(); | ||||
| dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str); | dds_log(trace, "Lookup package {}@{}", pk_id.name, ver_str); | ||||
| auto& st = _stmt_cache(R"( | auto& st = _stmt_cache(R"( | ||||
| auto deps = dependencies_of(pk_id); | auto deps = dependencies_of(pk_id); | ||||
| auto info = pkg_info{ | |||||
| auto info = pkg_listing{ | |||||
| pk_id, | pk_id, | ||||
| std::move(deps), | std::move(deps), | ||||
| std::move(description), | std::move(description), |
| #pragma once | #pragma once | ||||
| #include "./info.hpp" | |||||
| #include "./listing.hpp" | |||||
| #include <dds/util/fs.hpp> | #include <dds/util/fs.hpp> | ||||
| static fs::path default_path() noexcept; | static fs::path default_path() noexcept; | ||||
| void store(const pkg_info& info); | |||||
| std::optional<pkg_info> get(const pkg_id& id) const noexcept; | |||||
| void store(const pkg_listing& info); | |||||
| std::optional<pkg_listing> get(const pkg_id& id) const noexcept; | |||||
| std::vector<pkg_id> all() const noexcept; | std::vector<pkg_id> all() const noexcept; | ||||
| std::vector<pkg_id> by_name(std::string_view sv) const noexcept; | std::vector<pkg_id> by_name(std::string_view sv) const noexcept; |
| }; | }; | ||||
| TEST_CASE_METHOD(catalog_test_case, "Store a simple package") { | TEST_CASE_METHOD(catalog_test_case, "Store a simple package") { | ||||
| db.store(dds::pkg_info{ | |||||
| db.store(dds::pkg_listing{ | |||||
| dds::pkg_id("foo", semver::version::parse("1.2.3")), | dds::pkg_id("foo", semver::version::parse("1.2.3")), | ||||
| {}, | {}, | ||||
| "example", | "example", | ||||
| CHECK(std::get<dds::git_remote_listing>(info->remote).ref == "master"); | CHECK(std::get<dds::git_remote_listing>(info->remote).ref == "master"); | ||||
| // Update the entry with a new git remote ref | // Update the entry with a new git remote ref | ||||
| CHECK_NOTHROW(db.store(dds::pkg_info{ | |||||
| CHECK_NOTHROW(db.store(dds::pkg_listing{ | |||||
| dds::pkg_id("foo", semver::version::parse("1.2.3")), | dds::pkg_id("foo", semver::version::parse("1.2.3")), | ||||
| {}, | {}, | ||||
| "example", | "example", | ||||
| } | } | ||||
| TEST_CASE_METHOD(catalog_test_case, "Package requirements") { | TEST_CASE_METHOD(catalog_test_case, "Package requirements") { | ||||
| db.store(dds::pkg_info{ | |||||
| db.store(dds::pkg_listing{ | |||||
| dds::pkg_id{"foo", semver::version::parse("1.2.3")}, | dds::pkg_id{"foo", semver::version::parse("1.2.3")}, | ||||
| { | { | ||||
| {"bar", {semver::version::parse("1.2.3"), semver::version::parse("1.4.0")}}, | {"bar", {semver::version::parse("1.2.3"), semver::version::parse("1.4.0")}}, |
| namespace { | namespace { | ||||
| temporary_sdist do_pull_sdist(const pkg_info& listing, std::monostate) { | |||||
| temporary_sdist do_pull_sdist(const pkg_listing& listing, std::monostate) { | |||||
| neo_assert_always( | neo_assert_always( | ||||
| invariant, | invariant, | ||||
| false, | false, | ||||
| } | } | ||||
| template <remote_listing R> | template <remote_listing R> | ||||
| temporary_sdist do_pull_sdist(const pkg_info& listing, const R& remote) { | |||||
| temporary_sdist do_pull_sdist(const pkg_listing& listing, const R& remote) { | |||||
| auto tmpdir = dds::temporary_dir::create(); | auto tmpdir = dds::temporary_dir::create(); | ||||
| remote.pull_source(tmpdir.path()); | remote.pull_source(tmpdir.path()); | ||||
| } // namespace | } // namespace | ||||
| temporary_sdist dds::get_package_sdist(const pkg_info& pkg) { | |||||
| temporary_sdist dds::get_package_sdist(const pkg_listing& pkg) { | |||||
| auto tsd = std::visit([&](auto&& remote) { return do_pull_sdist(pkg, remote); }, pkg.remote); | auto tsd = std::visit([&](auto&& remote) { return do_pull_sdist(pkg, remote); }, pkg.remote); | ||||
| if (!(tsd.sdist.manifest.id == pkg.ident)) { | if (!(tsd.sdist.manifest.id == pkg.ident)) { | ||||
| throw_external_error<errc::sdist_ident_mismatch>( | throw_external_error<errc::sdist_ident_mismatch>( | ||||
| return *info; | return *info; | ||||
| }); | }); | ||||
| auto okay = parallel_run(absent_pkg_infos, 8, [&](pkg_info inf) { | |||||
| auto okay = parallel_run(absent_pkg_infos, 8, [&](pkg_listing inf) { | |||||
| dds_log(info, "Download package: {}", inf.ident.to_string()); | dds_log(info, "Download package: {}", inf.ident.to_string()); | ||||
| auto tsd = get_package_sdist(inf); | auto tsd = get_package_sdist(inf); | ||||
| std::scoped_lock lk{repo_mut}; | std::scoped_lock lk{repo_mut}; |
| class pkg_cache; | class pkg_cache; | ||||
| class pkg_db; | class pkg_db; | ||||
| struct pkg_info; | |||||
| struct pkg_listing; | |||||
| temporary_sdist get_package_sdist(const pkg_info&); | |||||
| temporary_sdist get_package_sdist(const pkg_listing&); | |||||
| void get_all(const std::vector<pkg_id>& pkgs, dds::pkg_cache& repo, const pkg_db& cat); | void get_all(const std::vector<pkg_id>& pkgs, dds::pkg_cache& repo, const pkg_db& cat); | ||||
| #include "./info.hpp" | |||||
| #include "./listing.hpp" | |||||
| #include <dds/error/errors.hpp> | #include <dds/error/errors.hpp> | ||||
| #include <dds/util/string.hpp> | #include <dds/util/string.hpp> |
| remote_listing_var parse_remote_url(std::string_view url); | remote_listing_var parse_remote_url(std::string_view url); | ||||
| struct pkg_info { | |||||
| struct pkg_listing { | |||||
| pkg_id ident; | pkg_id ident; | ||||
| std::vector<dependency> deps; | std::vector<dependency> deps; | ||||
| std::string description; | std::string description; |
| #include "./repoman.hpp" | #include "./repoman.hpp" | ||||
| #include <dds/pkg/info.hpp> | |||||
| #include <dds/pkg/listing.hpp> | |||||
| #include <dds/sdist/package.hpp> | #include <dds/sdist/package.hpp> | ||||
| #include <dds/util/log.hpp> | #include <dds/util/log.hpp> | ||||
| #include <dds/util/result.hpp> | #include <dds/util/result.hpp> | ||||
| neo::sqlite3::transaction_guard tr{_db}; | neo::sqlite3::transaction_guard tr{_db}; | ||||
| dds_log(debug, "Recording package {}@{}", man->id.name, man->id.version.to_string()); | dds_log(debug, "Recording package {}@{}", man->id.name, man->id.version.to_string()); | ||||
| dds::pkg_info info{.ident = man->id, | |||||
| .deps = man->dependencies, | |||||
| .description = "[No description]", | |||||
| .remote = {}}; | |||||
| auto rel_url = fmt::format("dds:{}", man->id.to_string()); | |||||
| dds::pkg_listing info{.ident = man->id, | |||||
| .deps = man->dependencies, | |||||
| .description = "[No description]", | |||||
| .remote = {}}; | |||||
| auto rel_url = fmt::format("dds:{}", man->id.to_string()); | |||||
| add_pkg(info, rel_url); | add_pkg(info, rel_url); | ||||
| auto dest_path = pkg_dir() / man->id.name / man->id.version.to_string() / "sdist.tar.gz"; | auto dest_path = pkg_dir() / man->id.name / man->id.version.to_string() / "sdist.tar.gz"; | ||||
| } | } | ||||
| } | } | ||||
| void repo_manager::add_pkg(const pkg_info& info, std::string_view url) { | |||||
| void repo_manager::add_pkg(const pkg_listing& info, std::string_view url) { | |||||
| dds_log(info, "Directly add an entry for {}", info.ident.to_string()); | dds_log(info, "Directly add an entry for {}", info.ident.to_string()); | ||||
| DDS_E_SCOPE(info.ident); | DDS_E_SCOPE(info.ident); | ||||
| nsql::recursive_transaction_guard tr{_db}; | nsql::recursive_transaction_guard tr{_db}; |
| namespace dds { | namespace dds { | ||||
| struct pkg_info; | |||||
| struct pkg_listing; | |||||
| struct e_init_repo { | struct e_init_repo { | ||||
| fs::path path; | fs::path path; | ||||
| void import_targz(path_ref tgz_path); | void import_targz(path_ref tgz_path); | ||||
| void delete_package(pkg_id id); | void delete_package(pkg_id id); | ||||
| void add_pkg(const pkg_info& info, std::string_view url); | |||||
| void add_pkg(const pkg_listing& info, std::string_view url); | |||||
| auto all_packages() const noexcept { | auto all_packages() const noexcept { | ||||
| using namespace neo::sqlite3::literals; | using namespace neo::sqlite3::literals; |
| #include <dds/repoman/repoman.hpp> | #include <dds/repoman/repoman.hpp> | ||||
| #include <dds/pkg/info.hpp> | |||||
| #include <dds/pkg/listing.hpp> | |||||
| #include <dds/temp.hpp> | #include <dds/temp.hpp> | ||||
| #include <neo/sqlite3/error.hpp> | #include <neo/sqlite3/error.hpp> | ||||
| } | } | ||||
| TEST_CASE_METHOD(tmp_repo, "Add a package directly") { | TEST_CASE_METHOD(tmp_repo, "Add a package directly") { | ||||
| dds::pkg_info info{ | |||||
| dds::pkg_listing info{ | |||||
| .ident = dds::pkg_id::parse("foo@1.2.3"), | .ident = dds::pkg_id::parse("foo@1.2.3"), | ||||
| .deps = {}, | .deps = {}, | ||||
| .description = "Something", | .description = "Something", |