Browse Source

Pulling catalogs from repositories!

default_compile_flags
vector-of-bool 4 years ago
parent
commit
b7df47661a
14 changed files with 258 additions and 189 deletions
  1. +3
    -3
      package.jsonc
  2. +79
    -62
      src/dds.main.cpp
  3. +43
    -41
      src/dds/catalog/catalog.cpp
  4. +0
    -1
      src/dds/catalog/catalog.hpp
  5. +0
    -34
      src/dds/catalog/init_catalog.cpp
  6. +0
    -11
      src/dds/catalog/init_catalog.hpp
  7. +7
    -0
      src/dds/catalog/package_info.cpp
  8. +12
    -0
      src/dds/catalog/remote/http.cpp
  9. +82
    -10
      src/dds/remote/remote.cpp
  10. +5
    -2
      src/dds/remote/remote.hpp
  11. +19
    -11
      src/dds/repoman/repoman.cpp
  12. +1
    -1
      src/dds/repoman/repoman.hpp
  13. +5
    -5
      src/dds/repoman/repoman.test.cpp
  14. +2
    -8
      tests/deps/build-deps/project/catalog.json

+ 3
- 3
package.jsonc View File

"range-v3@0.11.0", "range-v3@0.11.0",
"nlohmann-json@3.7.1", "nlohmann-json@3.7.1",
"neo-sqlite3@0.4.1", "neo-sqlite3@0.4.1",
"neo-fun~0.5.4",
"neo-compress~0.1.1",
"neo-fun~0.6.0",
"neo-compress~0.2.0",
"neo-url~0.2.2", "neo-url~0.2.2",
"semver@0.2.2", "semver@0.2.2",
"pubgrub@0.2.1", "pubgrub@0.2.1",
"ctre@2.8.1", "ctre@2.8.1",
"fmt^7.0.3", "fmt^7.0.3",
"neo-http^0.1.0", "neo-http^0.1.0",
"neo-io^0.1.0",
"neo-io^0.1.1",
"boost.leaf~0.3.0", "boost.leaf~0.3.0",
], ],
"test_driver": "Catch-Main" "test_driver": "Catch-Main"

+ 79
- 62
src/dds.main.cpp View File

catalog_path_flag cat_path{cmd}; catalog_path_flag cat_path{cmd};


args::Flag import_stdin{cmd, "stdin", "Import JSON from stdin", {"stdin"}}; args::Flag import_stdin{cmd, "stdin", "Import JSON from stdin", {"stdin"}};
args::Flag init{cmd, "initial", "Re-import the initial catalog contents", {"initial"}};
args::ValueFlagList<std::string> args::ValueFlagList<std::string>
json_paths{cmd, json_paths{cmd,
"json", "json",


int run() { int run() {
auto cat = cat_path.open(); auto cat = cat_path.open();
if (init.Get()) {
cat.import_initial();
}
for (const auto& json_fpath : json_paths.Get()) { for (const auto& json_fpath : json_paths.Get()) {
cat.import_json_file(json_fpath); cat.import_json_file(json_fpath);
} }
auto repo auto repo
= dds::repo_manager::create(where.Get(), = dds::repo_manager::create(where.Get(),
name ? std::make_optional(name.Get()) : std::nullopt); name ? std::make_optional(name.Get()) : std::nullopt);
dds_log(info, "Created new repository '{}' in {}", repo.root(), repo.name());
dds_log(info, "Created new repository '{}' in {}", repo.name(), repo.root());
return 0; return 0;
} }
} init{*this}; } init{*this};
args::Flag update{cmd, "update", "Update catalog contents immediately", {"update", 'U'}}; args::Flag update{cmd, "update", "Update catalog contents immediately", {"update", 'U'}};


int run() { int run() {
return boost::leaf::try_handle_all( //
[&]() -> dds::result<int> {
try {
auto cat = cat_path.open();
auto repo = dds::remote_repository::connect(url.Get());
repo.store(cat.database());
if (update) {
repo.update_catalog(cat.database());
}
} catch (...) {
return dds::capture_exception();
}
return 0;
},
[&](neo::url_validation_error url_err, dds::e_url_string bad_url) {
dds_log(error, "Invalid URL [{}]: {}", bad_url.value, url_err.what());
return 1;
},
[&](const json5::parse_error& e, dds::e_http_url bad_url) {
dds_log(error,
"Error parsing JSON downloaded from URL [{}]: {}",
bad_url.value,
e.what());
return 1;
},
[](dds::e_sqlite3_error_exc e, dds::e_url_string url) {
dds_log(error,
"Error accessing remote database (From {}): {}",
url.value,
e.message);
return 1;
},
[](dds::e_sqlite3_error_exc e) {
dds_log(error, "Unexpected database error: {}", e.message);
return 1;
},
[&](dds::e_system_error_exc e, dds::e_http_connect conn) {
dds_log(error,
"Error opening connection to [{}:{}]: {}",
conn.host,
conn.port,
e.message);
return 1;
},
[](const std::exception& e) {
dds_log(error, "An unknown unhandled exception occurred: {}", e.what());
return 1;
},
[](dds::e_system_error_exc e) {
dds_log(error, "An unknown system_error occurred: {}", e.message);
return 42;
},
[](boost::leaf::diagnostic_info const& info) {
dds_log(error, "An unnknown error occurred? {}", info);
return 42;
});
auto cat = cat_path.open();
auto repo = dds::remote_repository::connect(url.Get());
repo.store(cat.database());
if (update) {
repo.update_catalog(cat.database());
}
return 0;
} }
} add{*this}; } add{*this};


struct {
cli_repo& parent;
args::Command cmd{parent.repo_group, "update", "Update remote package information"};
common_flags _flags{cmd};

catalog_path_flag cat_path{cmd};

int run() {
auto cat = cat_path.open();
dds::update_all_remotes(cat.database());
return 0;
}
} update{*this};

struct { struct {
cli_repo& parent; cli_repo& parent;
args::Command cmd{parent.repo_group, "init", "Initialize a directory as a repository"}; args::Command cmd{parent.repo_group, "init", "Initialize a directory as a repository"};
} }
} init{*this}; } init{*this};


int run() {
int _run() {
if (ls.cmd) { if (ls.cmd) {
return ls.run(); return ls.run();
} else if (init.cmd) { } else if (init.cmd) {
return import_.run(); return import_.run();
} else if (add.cmd) { } else if (add.cmd) {
return add.run(); return add.run();
} else if (update.cmd) {
return update.run();
} else { } else {
assert(false); assert(false);
std::terminate(); std::terminate();
} }
} }

int run() {
return boost::leaf::try_handle_all( //
[&]() -> dds::result<int> {
try {
return _run();
} catch (...) {
return dds::capture_exception();
}
return 0;
},
[&](neo::url_validation_error url_err, dds::e_url_string bad_url) {
dds_log(error, "Invalid URL [{}]: {}", bad_url.value, url_err.what());
return 1;
},
[&](const json5::parse_error& e, dds::e_http_url bad_url) {
dds_log(error,
"Error parsing JSON downloaded from URL [{}]: {}",
bad_url.value,
e.what());
return 1;
},
[](dds::e_sqlite3_error_exc e, dds::e_url_string url) {
dds_log(error,
"Error accessing remote database (From {}): {}",
url.value,
e.message);
return 1;
},
[](dds::e_sqlite3_error_exc e) {
dds_log(error, "Unexpected database error: {}", e.message);
return 1;
},
[&](dds::e_system_error_exc e, dds::e_http_connect conn) {
dds_log(error,
"Error opening connection to [{}:{}]: {}",
conn.host,
conn.port,
e.message);
return 1;
},
[](const std::exception& e) {
dds_log(error, "An unknown unhandled exception occurred: {}", e.what());
return 1;
},
[](dds::e_system_error_exc e) {
dds_log(error, "An unknown system_error occurred: {}", e.message);
return 42;
},
[](boost::leaf::diagnostic_info const& info) {
dds_log(error, "An unnknown error occurred? {}", info);
return 42;
});
}
}; };


/* /*

+ 43
- 41
src/dds/catalog/catalog.cpp View File



#include "./import.hpp" #include "./import.hpp"


#include <dds/catalog/init_catalog.hpp>
#include <dds/dym.hpp> #include <dds/dym.hpp>
#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>
#include <dds/solve/solve.hpp> #include <dds/solve/solve.hpp>
version TEXT NOT NULL, version TEXT NOT NULL,
description TEXT NOT NULL, description TEXT NOT NULL,
remote_url TEXT NOT NULL, remote_url TEXT NOT NULL,
remote_id INTEGER REFERENCES dds_cat_remotes DEFAULT NULL,
remote_id INTEGER
REFERENCES dds_cat_remotes
ON DELETE CASCADE,
repo_transform TEXT NOT NULL DEFAULT '[]', repo_transform TEXT NOT NULL DEFAULT '[]',
UNIQUE (name, version)
UNIQUE (name, version, remote_id)
); );


INSERT INTO dds_cat_pkgs_new(pkg_id, INSERT INTO dds_cat_pkgs_new(pkg_id,


CREATE TABLE dds_cat_pkg_deps_new ( CREATE TABLE dds_cat_pkg_deps_new (
dep_id INTEGER PRIMARY KEY AUTOINCREMENT, dep_id INTEGER PRIMARY KEY AUTOINCREMENT,
pkg_id INTEGER NOT NULL REFERENCES dds_cat_pkgs_new(pkg_id),
pkg_id INTEGER
NOT NULL
REFERENCES dds_cat_pkgs_new(pkg_id)
ON DELETE CASCADE,
dep_name TEXT NOT NULL, dep_name TEXT NOT NULL,
low TEXT NOT NULL, low TEXT NOT NULL,
high TEXT NOT NULL, high TEXT NOT NULL,
} }
} }


void store_init_packages(nsql::database& db, nsql::statement_cache& st_cache) {
dds_log(debug, "Restoring initial package data");
for (auto& pkg : init_catalog_packages()) {
do_store_pkg(db, st_cache, pkg);
}
}

void ensure_migrated(nsql::database& db) { void ensure_migrated(nsql::database& db) {
db.exec(R"( db.exec(R"(
PRAGMA foreign_keys = 1; PRAGMA foreign_keys = 1;


int version = version_; int version = version_;


// If this is the first time we're working here, import the initial
// catalog with some useful tidbits.
bool import_init_packages = version == 0;

if (version > current_database_version) { if (version > current_database_version) {
dds_log(critical, dds_log(critical,
"Catalog version is {}, but we only support up to {}", "Catalog version is {}, but we only support up to {}",
} }
meta["version"] = current_database_version; meta["version"] = current_database_version;
exec(db.prepare("UPDATE dds_cat_meta SET meta=?"), meta.dump()); exec(db.prepare("UPDATE dds_cat_meta SET meta=?"), meta.dump());

if (import_init_packages) {
dds_log(
info,
"A new catalog database case been created, and has been populated with some initial "
"contents.");
neo::sqlite3::statement_cache stmts{db};
store_init_packages(db, stmts);
}
} }


void check_json(bool b, std::string_view what) { void check_json(bool b, std::string_view what) {
description, description,
repo_transform repo_transform
FROM dds_cat_pkgs FROM dds_cat_pkgs
WHERE name = ? AND version = ?
WHERE name = ?1 AND version = ?2
ORDER BY pkg_id DESC
)"_sql); )"_sql);
st.reset(); st.reset();
st.bindings() = std::forward_as_tuple(pk_id.name, ver_str); st.bindings() = std::forward_as_tuple(pk_id.name, ver_str);
auto opt_tup = nsql::unpack_single_opt<std::int64_t,
std::string,
std::string,
std::string,
std::string,
std::string>(st);
if (!opt_tup) {
auto ec = st.step(std::nothrow);
if (ec == nsql::errc::done) {
dym_target::fill([&] { dym_target::fill([&] {
auto all_ids = this->all(); auto all_ids = this->all();
auto id_strings auto id_strings
}); });
return std::nullopt; return std::nullopt;
} }
const auto& [pkg_id, name, version, remote_url, description, repo_transform] = *opt_tup;
assert(pk_id.name == name);
assert(pk_id.version == semver::version::parse(version));
neo_assert_always(invariant,
ec == nsql::errc::row,
"Failed to pull a package from the catalog database",
ec,
pk_id.to_string(),
nsql::error_category().message(int(ec)));

const auto& [pkg_id, name, version, remote_url, description, repo_transform]
= st.row()
.unpack<std::int64_t,
std::string,
std::string,
std::string,
std::string,
std::string>();

ec = st.step(std::nothrow);
if (ec == nsql::errc::row) {
dds_log(warn,
"There is more than one entry for package {} in the catalog database. One will be "
"chosen arbitrarily.",
pk_id.to_string());
}

neo_assert(invariant,
pk_id.name == name && pk_id.version == semver::version::parse(version),
"Package metadata does not match",
pk_id.to_string(),
name,
version);


auto deps = dependencies_of(pk_id); auto deps = dependencies_of(pk_id);


SELECT name, version SELECT name, version
FROM dds_cat_pkgs FROM dds_cat_pkgs
WHERE name = ? WHERE name = ?
ORDER BY pkg_id DESC
)"_sql), )"_sql),
sv) // sv) //
| neo::lref // | neo::lref //
do_store_pkg(_db, _stmt_cache, pkg); do_store_pkg(_db, _stmt_cache, pkg);
} }
} }

void catalog::import_initial() {
nsql::transaction_guard tr{_db};
dds_log(info, "Restoring built-in initial catalog contents");
store_init_packages(_db, _stmt_cache);
}

+ 0
- 1
src/dds/catalog/catalog.hpp View File

std::vector<package_id> by_name(std::string_view sv) const noexcept; std::vector<package_id> by_name(std::string_view sv) const noexcept;
std::vector<dependency> dependencies_of(const package_id& pkg) const noexcept; std::vector<dependency> dependencies_of(const package_id& pkg) const noexcept;


void import_initial();
void import_json_str(std::string_view json_str); void import_json_str(std::string_view json_str);
void import_json_file(path_ref json_path) { void import_json_file(path_ref json_path) {
auto content = dds::slurp_file(json_path); auto content = dds::slurp_file(json_path);

+ 0
- 34
src/dds/catalog/init_catalog.cpp
File diff suppressed because it is too large
View File


+ 0
- 11
src/dds/catalog/init_catalog.hpp View File

#pragma once

#include "./package_info.hpp"

#include <vector>

namespace dds {

const std::vector<package_info>& init_catalog_packages() noexcept;

} // namespace dds

+ 7
- 0
src/dds/catalog/package_info.cpp View File

#include "./package_info.hpp" #include "./package_info.hpp"


#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>
#include <dds/util/string.hpp>


#include <neo/url.hpp> #include <neo/url.hpp>
#include <neo/utility.hpp> #include <neo/utility.hpp>
return git_remote_listing::from_url(sv); return git_remote_listing::from_url(sv);
} else if (url.scheme == neo::oper::any_of("http", "https")) { } else if (url.scheme == neo::oper::any_of("http", "https")) {
return http_remote_listing::from_url(sv); return http_remote_listing::from_url(sv);
} else if (url.scheme == neo::oper::any_of("dds+http", "dds+https", "http+dds", "https+dds")) {
fs::path path = url.path;
auto leaf = path.filename().string();
auto namever_path = replace(leaf, "@", "/");
url.path = (path.parent_path() / "pkg" / namever_path / "sdist.tar.gz").generic_string();
return http_remote_listing::from_url(url.to_string());
} else { } else {
throw_user_error< throw_user_error<
errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'", errc::invalid_remote_url>("Unknown scheme '{}' for remote package URL '{}'",

+ 12
- 0
src/dds/catalog/remote/http.cpp View File

unsigned int strip_components = 1; unsigned int strip_components = 1;
std::optional<lm::usage> auto_lib; std::optional<lm::usage> auto_lib;


// IF we are a dds+ URL, strip_components should be zero, and give the url a plain
// HTTP/HTTPS scheme
if (url.scheme.starts_with("dds+")) {
url.scheme = url.scheme.substr(4);
strip_components = 0;
} else if (url.scheme.ends_with("+dds")) {
url.scheme.erase(url.scheme.end() - 3);
strip_components = 0;
} else {
// Leave the URL as-is
}

if (url.query) { if (url.query) {
neo::basic_query_string_view qsv{*url.query}; neo::basic_query_string_view qsv{*url.query};
for (auto qstr : qsv) { for (auto qstr : qsv) {

+ 82
- 10
src/dds/remote/remote.cpp View File

#include <dds/error/errors.hpp> #include <dds/error/errors.hpp>
#include <dds/http/session.hpp> #include <dds/http/session.hpp>
#include <dds/temp.hpp> #include <dds/temp.hpp>
#include <dds/util/log.hpp>
#include <dds/util/result.hpp> #include <dds/util/result.hpp>


#include <neo/event.hpp>
#include <neo/scope.hpp>
#include <neo/sqlite3/exec.hpp> #include <neo/sqlite3/exec.hpp>
#include <neo/sqlite3/iter_tuples.hpp>
#include <neo/sqlite3/single.hpp> #include <neo/sqlite3/single.hpp>
#include <neo/sqlite3/transaction.hpp> #include <neo/sqlite3/transaction.hpp>
#include <neo/url.hpp> #include <neo/url.hpp>
#include <neo/utility.hpp> #include <neo/utility.hpp>
#include <range/v3/range/conversion.hpp>


using namespace dds; using namespace dds;
namespace nsql = neo::sqlite3; namespace nsql = neo::sqlite3;
url.host.has_value(), url.host.has_value(),
"URL does not have a hostname??", "URL does not have a hostname??",
url.to_string()); url.to_string());
auto sess = url.scheme == "https"
? http_session::connect_ssl(*url.host, url.port_or_default_port_or(443))
: http_session::connect(*url.host, url.port_or_default_port_or(80));
auto sess = http_session::connect_for(url);


auto tempdir = temporary_dir::create(); auto tempdir = temporary_dir::create();
auto repo_db_dl = tempdir.path() / "repo.db"; auto repo_db_dl = tempdir.path() / "repo.db";
auto name_st = db.db.prepare("SELECT name FROM dds_repo_meta"); auto name_st = db.db.prepare("SELECT name FROM dds_repo_meta");
auto [name] = nsql::unpack_single<std::string>(name_st); auto [name] = nsql::unpack_single<std::string>(name_st);


remote_repository ret;
ret._base_url = url;
ret._name = name;
return ret;
return {name, url};
} }


void remote_repository::store(nsql::database_ref db) { void remote_repository::store(nsql::database_ref db) {
auto st = db.prepare(R"( auto st = db.prepare(R"(
INSERT INTO dds_cat_remotes (name, gen_ident, remote_url) INSERT INTO dds_cat_remotes (name, gen_ident, remote_url)
VALUES (?, ?, ?) VALUES (?, ?, ?)
ON CONFLICT (name) DO
UPDATE SET gen_ident = ?2, remote_url = ?3
)"); )");
nsql::exec(st, _name, "[placeholder]", _base_url.to_string()); nsql::exec(st, _name, "[placeholder]", _base_url.to_string());
} }


void remote_repository::update_catalog(nsql::database_ref db) { void remote_repository::update_catalog(nsql::database_ref db) {
dds_log(info, "Pulling repository contents for {} [{}]", _name, _base_url.to_string());

auto rdb = remote_db::download_and_open_for_base(_base_url); auto rdb = remote_db::download_and_open_for_base(_base_url);


auto base_url_str = _base_url.to_string();
while (base_url_str.ends_with("/")) {
base_url_str.pop_back();
}

auto db_path = rdb._tempdir.path() / "repo.db"; auto db_path = rdb._tempdir.path() / "repo.db";


auto rid_st = db.prepare("SELECT remote_id FROM dds_cat_remotes WHERE name = ?"); auto rid_st = db.prepare("SELECT remote_id FROM dds_cat_remotes WHERE name = ?");
rid_st.bindings()[1] = _name; rid_st.bindings()[1] = _name;
auto [remote_id] = nsql::unpack_single<std::int64_t>(rid_st); auto [remote_id] = nsql::unpack_single<std::int64_t>(rid_st);
rid_st.reset();


nsql::transaction_guard tr{db};
nsql::exec(db.prepare("ATTACH DATABASE ? AS remote"), db_path.string()); nsql::exec(db.prepare("ATTACH DATABASE ? AS remote"), db_path.string());
neo_defer { db.exec("DETACH DATABASE remote"); };
nsql::transaction_guard tr{db};
nsql::exec( // nsql::exec( //
db.prepare(R"( db.prepare(R"(
DELETE FROM dds_cat_pkgs DELETE FROM dds_cat_pkgs
name, name,
version, version,
description, description,
printf('dds:%s/%s', name, version),
CASE
WHEN url LIKE 'dds:%@%' THEN
-- Convert 'dds:name@ver' to 'dds+<base-repo-url>/name@ver'
-- This will later resolve to the actual package URL
printf('dds+%s/%s', ?2, substr(url, 5))
ELSE
-- Non-'dds:' URLs are kept as-is
url
END,
?1 ?1
FROM remote.dds_repo_packages FROM remote.dds_repo_packages
)"), )"),
remote_id);
remote_id,
base_url_str);
db.exec(R"(
INSERT OR REPLACE INTO dds_cat_pkg_deps (pkg_id, dep_name, low, high)
SELECT
local_pkgs.pkg_id AS pkg_id,
dep_name,
low,
high
FROM remote.dds_repo_package_deps AS deps,
remote.dds_repo_packages AS pkgs USING(package_id),
dds_cat_pkgs AS local_pkgs USING(name, version)
)");
// Validate our database
auto fk_check = db.prepare("PRAGMA foreign_key_check");
auto rows = nsql::iter_tuples<std::string, std::int64_t, std::string, std::string>(fk_check);
bool any_failed = false;
for (auto [child_table, rowid, parent_table, failed_idx] : rows) {
dds_log(
critical,
"Database foreign_key error after import: {0}.{3} referencing {2} violated at row {1}",
child_table,
rowid,
parent_table,
failed_idx);
any_failed = true;
}
auto int_check = db.prepare("PRAGMA main.integrity_check");
for (auto [error] : nsql::iter_tuples<std::string>(int_check)) {
if (error == "ok") {
continue;
}
dds_log(critical, "Database errors after import: {}", error);
any_failed = true;
}
if (any_failed) {
throw_external_error<errc::corrupted_catalog_db>(
"Database update failed due to data integrity errors");
}
}

void dds::update_all_remotes(nsql::database_ref db) {
dds_log(info, "Updating catalog from all remotes");
auto repos_st = db.prepare("SELECT name, remote_url FROM dds_cat_remotes");
auto tups = nsql::iter_tuples<std::string, std::string>(repos_st) | ranges::to_vector;

for (const auto& [name, remote_url] : tups) {
DDS_E_SCOPE(e_url_string{remote_url});
remote_repository repo{name, neo::url::parse(remote_url)};
repo.update_catalog(db);
}

dds_log(info, "Recompacting database...");
db.exec("VACUUM");
} }

+ 5
- 2
src/dds/remote/remote.hpp View File

std::string _name; std::string _name;
neo::url _base_url; neo::url _base_url;


remote_repository(std::string name, neo::url url)
: _name(std::move(name))
, _base_url(std::move(url)) {}
remote_repository() = default; remote_repository() = default;


public: public:
static remote_repository connect(std::string_view url); static remote_repository connect(std::string_view url);


// const repository_manifest& manifest() const noexcept;

void store(neo::sqlite3::database_ref); void store(neo::sqlite3::database_ref);
void update_catalog(neo::sqlite3::database_ref); void update_catalog(neo::sqlite3::database_ref);
}; };


void update_all_remotes(neo::sqlite3::database_ref);

} // namespace dds } // namespace dds

+ 19
- 11
src/dds/repoman/repoman.cpp View File

#include "./repoman.hpp" #include "./repoman.hpp"


#include <dds/catalog/import.hpp>
#include <dds/package/manifest.hpp> #include <dds/package/manifest.hpp>
#include <dds/util/log.hpp> #include <dds/util/log.hpp>
#include <dds/util/result.hpp> #include <dds/util/result.hpp>
name TEXT NOT NULL, name TEXT NOT NULL,
version TEXT NOT NULL, version TEXT NOT NULL,
description TEXT NOT NULL, description TEXT NOT NULL,
url TEXT NOT NULL,
UNIQUE (name, version) UNIQUE (name, version)
); );




void ensure_migrated(nsql::database_ref db, std::optional<std::string_view> name) { void ensure_migrated(nsql::database_ref db, std::optional<std::string_view> name) {
db.exec(R"( db.exec(R"(
PRAGMA busy_timeout = 6000;
PRAGMA foreign_keys = 1; PRAGMA foreign_keys = 1;
CREATE TABLE IF NOT EXISTS dds_repo_meta ( CREATE TABLE IF NOT EXISTS dds_repo_meta (
meta_version INTEGER DEFAULT 1, meta_version INTEGER DEFAULT 1,
DDS_E_SCOPE(e_init_repo_db{db_path}); DDS_E_SCOPE(e_init_repo_db{db_path});
DDS_E_SCOPE(e_open_repo_db{db_path}); DDS_E_SCOPE(e_open_repo_db{db_path});
ensure_migrated(db, name); ensure_migrated(db, name);
fs::create_directories(directory / "data");
fs::create_directories(directory / "pkg");
} }
return open(directory); return open(directory);
} }
dds_log(debug, "Recording package {}@{}", man->pkg_id.name, man->pkg_id.version.to_string()); dds_log(debug, "Recording package {}@{}", man->pkg_id.name, man->pkg_id.version.to_string());
nsql::exec( // nsql::exec( //
_stmts(R"( _stmts(R"(
INSERT INTO dds_repo_packages (name, version, description)
VALUES (?, ?, 'No description')
INSERT INTO dds_repo_packages (name, version, description, url)
VALUES (
?1,
?2,
'No description',
printf('dds:%s@%s', ?1, ?2)
)
)"_sql), )"_sql),
man->pkg_id.name, man->pkg_id.name,
man->pkg_id.version.to_string()); man->pkg_id.version.to_string());
iv_1.high.to_string()); iv_1.high.to_string());
} }


auto dest_dir = data_dir() / man->pkg_id.name;
auto dest_path = dest_dir / fmt::format("{}.tar.gz", man->pkg_id.version.to_string());
fs::create_directories(dest_dir);
auto dest_path
= pkg_dir() / man->pkg_id.name / man->pkg_id.version.to_string() / "sdist.tar.gz";
fs::create_directories(dest_path.parent_path());
fs::copy(tgz_file, dest_path); fs::copy(tgz_file, dest_path);


tr.commit(); tr.commit();
pkg_id.version.to_string()); pkg_id.version.to_string());
/// XXX: Verify with _db.changes() that we actually deleted one row /// XXX: Verify with _db.changes() that we actually deleted one row


auto name_dir = data_dir() / pkg_id.name;
auto ver_file = name_dir / fmt::format("{}.tar.gz", pkg_id.version.to_string());
auto name_dir = pkg_dir() / pkg_id.name;
auto ver_dir = name_dir / pkg_id.version.to_string();


DDS_E_SCOPE(e_repo_delete_targz{ver_file});
DDS_E_SCOPE(e_repo_delete_targz{ver_dir});


if (!fs::is_regular_file(ver_file)) {
if (!fs::is_directory(ver_dir)) {
throw std::system_error(std::make_error_code(std::errc::no_such_file_or_directory), throw std::system_error(std::make_error_code(std::errc::no_such_file_or_directory),
"No source archive for the requested package"); "No source archive for the requested package");
} }


fs::remove(ver_file);
fs::remove_all(ver_dir);


tr.commit(); tr.commit();



+ 1
- 1
src/dds/repoman/repoman.hpp View File

static repo_manager create(path_ref directory, std::optional<std::string_view> name); static repo_manager create(path_ref directory, std::optional<std::string_view> name);
static repo_manager open(path_ref directory); static repo_manager open(path_ref directory);


auto data_dir() const noexcept { return _root / "data"; }
auto pkg_dir() const noexcept { return _root / "pkg"; }
path_ref root() const noexcept { return _root; } path_ref root() const noexcept { return _root; }
std::string name() const noexcept; std::string name() const noexcept;



+ 5
- 5
src/dds/repoman/repoman.test.cpp View File



} // namespace } // namespace


TEST_CASE("Open a repository") {
TEST_CASE("Open and import into a repository") {
auto tdir = dds::temporary_dir::create(); auto tdir = dds::temporary_dir::create();
auto repo = dds::repo_manager::create(tdir.path(), "test-repo"); auto repo = dds::repo_manager::create(tdir.path(), "test-repo");
auto neo_url_tgz = DATA_DIR / "neo-url@0.2.1.tar.gz"; auto neo_url_tgz = DATA_DIR / "neo-url@0.2.1.tar.gz";
repo.import_targz(neo_url_tgz); repo.import_targz(neo_url_tgz);
CHECK(dds::fs::is_directory(repo.data_dir() / "neo-url/"));
CHECK(dds::fs::is_regular_file(repo.data_dir() / "neo-url/0.2.1.tar.gz"));
CHECK(dds::fs::is_directory(repo.pkg_dir() / "neo-url/"));
CHECK(dds::fs::is_regular_file(repo.pkg_dir() / "neo-url/0.2.1/sdist.tar.gz"));
CHECK_THROWS_AS(repo.import_targz(neo_url_tgz), neo::sqlite3::constraint_unique_error); CHECK_THROWS_AS(repo.import_targz(neo_url_tgz), neo::sqlite3::constraint_unique_error);
repo.delete_package(dds::package_id::parse("neo-url@0.2.1")); repo.delete_package(dds::package_id::parse("neo-url@0.2.1"));
CHECK_FALSE(dds::fs::is_regular_file(repo.data_dir() / "neo-url/0.2.1.tar.gz"));
CHECK_FALSE(dds::fs::is_directory(repo.data_dir() / "neo-url"));
CHECK_FALSE(dds::fs::is_regular_file(repo.pkg_dir() / "neo-url/0.2.1/sdist.tar.gz"));
CHECK_FALSE(dds::fs::is_directory(repo.pkg_dir() / "neo-url"));
CHECK_THROWS_AS(repo.delete_package(dds::package_id::parse("neo-url@0.2.1")), CHECK_THROWS_AS(repo.delete_package(dds::package_id::parse("neo-url@0.2.1")),
std::system_error); std::system_error);
CHECK_NOTHROW(repo.import_targz(neo_url_tgz)); CHECK_NOTHROW(repo.import_targz(neo_url_tgz));

+ 2
- 8
tests/deps/build-deps/project/catalog.json View File

{ {
"version": 2, "version": 2,
"packages": { "packages": {
"neo-sqlite3": {
"0.1.0": {
"url": "git+https://github.com/vector-of-bool/neo-sqlite3.git#0.1.0"
},
"0.2.2": {
"url": "git+https://github.com/vector-of-bool/neo-sqlite3.git#0.2.2"
},
"neo-fun": {
"0.3.0": { "0.3.0": {
"url": "git+https://github.com/vector-of-bool/neo-sqlite3.git#0.3.0"
"url": "git+https://github.com/vector-of-bool/neo-fun.git#0.3.0"
} }
} }
} }

Loading…
Cancel
Save