visus
/
dds
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

'repoman' for managing static package repositories

default_compile_flags
vector-of-bool 4 years ago
parent
5f57cca29e
commit
a0a8bdf1c2
11 changed files with 595 additions and 2 deletions
Split View Show Diff Stats
  1. BIN
      data/neo-url@0.2.1.tar.gz
  2. +1
    -0
      library.jsonc
  3. +1
    -0
      package.jsonc
  4. +204
    -0
      src/dds.main.cpp
  5. +7
    -2
      src/dds/package/manifest.cpp
  6. +4
    -0
      src/dds/package/manifest.hpp
  7. +216
    -0
      src/dds/repoman/repoman.cpp
  8. +70
    -0
      src/dds/repoman/repoman.hpp
  9. +31
    -0
      src/dds/repoman/repoman.test.cpp
  10. +15
    -0
      src/dds/util/result.cpp
  11. +46
    -0
      src/dds/util/result.hpp

BIN
data/neo-url@0.2.1.tar.gz View File


+ 1
- 0
library.jsonc View File

@@ -16,6 +16,7 @@
"neo/io",
"neo/http",
"neo/url",
"boost/leaf",
// Explicit zlib link is required due to linker input order bug.
// Can be removed after alpha.5
"zlib/zlib",

+ 1
- 0
package.jsonc View File

@@ -20,6 +20,7 @@
"fmt^7.0.3",
"neo-http^0.1.0",
"neo-io^0.1.0",
"boost.leaf~0.3.0",
],
"test_driver": "Catch-Main"
}

+ 204
- 0
src/dds.main.cpp View File

@@ -4,14 +4,22 @@
#include <dds/dym.hpp>
#include <dds/error/errors.hpp>
#include <dds/repo/repo.hpp>
#include <dds/repoman/repoman.hpp>
#include <dds/source/dist.hpp>
#include <dds/toolchain/from_json.hpp>
#include <dds/util/fs.hpp>
#include <dds/util/log.hpp>
#include <dds/util/paths.hpp>
#include <dds/util/result.hpp>
#include <dds/util/signal.hpp>

#include <boost/leaf/handle_error.hpp>
#include <boost/leaf/handle_exception.hpp>
#include <fmt/ostream.h>
#include <json5/parse_data.hpp>
#include <neo/assert.hpp>
#include <neo/sqlite3/error.hpp>
#include <nlohmann/json.hpp>
#include <range/v3/action/join.hpp>
#include <range/v3/range/conversion.hpp>
#include <range/v3/view/concat.hpp>
@@ -396,6 +404,199 @@ struct cli_catalog {
}
};

/*
######## ######## ######## ####### ## ## ### ## ##
## ## ## ## ## ## ## ### ### ## ## ### ##
## ## ## ## ## ## ## #### #### ## ## #### ##
######## ###### ######## ## ## ## ### ## ## ## ## ## ##
## ## ## ## ## ## ## ## ######### ## ####
## ## ## ## ## ## ## ## ## ## ## ###
## ## ######## ## ####### ## ## ## ## ## ##
*/

struct cli_repoman {
cli_base& base;
args::Command cmd{base.cmd_group, "repoman", "Manage a package package repository"};
common_flags _common{cmd};

args::Group repoman_group{cmd, "Repoman subcommand"};

struct {
cli_repoman& parent;
args::Command cmd{parent.repoman_group, "init", "Initialize a new repository directory"};
common_flags _common{cmd};

args::Positional<dds::fs::path> where{cmd,
"<repo-path>",
"Directory where the repository will be created",
args::Options::Required};

string_flag name{cmd,
"<name>",
"Give the repository a name (should be GLOBALLY unique). If not provided, "
"a new random one will be generated.",
{"name"}};

int run() {
auto repo
= dds::repo_manager::create(where.Get(),
name ? std::make_optional(name.Get()) : std::nullopt);
dds_log(info, "Created new repository '{}' in {}", repo.root(), repo.name());
return 0;
}
} init{*this};

struct {
cli_repoman& parent;
args::Command cmd{parent.repoman_group, "import", "Import packages into a repository"};
common_flags _common{cmd};

args::Positional<dds::fs::path> where{cmd,
"<repo-path>",
"Directory of the repository to import",
args::Options::Required};

args::PositionalList<dds::fs::path> files{cmd,
"<targz-path>",
"Path to one or more sdist archives to import"};

int run() {
auto repo = dds::repo_manager::open(where.Get());
for (auto pkg : files.Get()) {
repo.import_targz(pkg);
}
return 0;
}
} import{*this};

struct {
cli_repoman& parent;
args::Command cmd{parent.repoman_group, "remove", "Remove packages from the repository"};
common_flags _common{cmd};

args::Positional<dds::fs::path> where{cmd,
"<repo-path>",
"Directory of the repository to import",
args::Options::Required};

args::PositionalList<std::string> packages{cmd,
"<package-id>",
"One or more identifiers of packages to remove"};

int run() {
auto repo = dds::repo_manager::open(where.Get());
for (auto& str : packages) {
auto pkg_id = dds::package_id::parse(str);
repo.delete_package(pkg_id);
}
return 0;
}
} remove{*this};

struct {
cli_repoman& parent;
args::Command cmd{parent.repoman_group, "ls", "List packages in the repository"};
common_flags _common{cmd};

args::Positional<dds::fs::path> where{cmd,
"<repo-path>",
"Directory of the repository to inspect",
args::Options::Required};

int run() {
auto repo = dds::repo_manager::open(where.Get());
for (auto pkg_id : repo.all_packages()) {
std::cout << pkg_id.to_string() << '\n';
}
return 0;
}
} ls{*this};

dds::result<int> _run() {
if (init.cmd) {
return init.run();
} else if (import.cmd) {
return import.run();
} else if (remove.cmd) {
return remove.run();
} else if (ls.cmd) {
return ls.run();
}
return 66;
}

int run() {
return boost::leaf::try_handle_all( //
[&]() -> dds::result<int> {
try {
return _run();
} catch (...) {
return dds::capture_exception();
}
},
[](dds::e_sqlite3_error_exc,
boost::leaf::match<neo::sqlite3::errc, neo::sqlite3::errc::constraint_unique>,
dds::e_repo_import_targz tgz,
dds::package_id pkg_id) {
dds_log(error,
"Package {} (from {}) is already present in the repository",
pkg_id.to_string(),
tgz.path);
return 1;
},
[](dds::e_sqlite3_error_exc e, dds::e_repo_import_targz tgz) {
dds_log(error,
"Database error while importing tar file {}: {}",
tgz.path,
e.message);
return 1;
},
[](dds::e_sqlite3_error_exc e, dds::e_init_repo init, dds::e_init_repo_db init_db) {
dds_log(
error,
"SQLite error while initializing repository in [{}] (SQlite database {}): {}",
init.path,
init_db.path,
e.message);
return 1;
},
[](dds::e_system_error_exc e, dds::e_repo_import_targz tgz) {
dds_log(error, "Failed to import package archive {}: {}", tgz.path, e.message);
return 1;
},
[](dds::e_system_error_exc e, dds::e_open_repo_db db) {
dds_log(error,
"Error while opening repository database {}: {}",
db.path,
e.message);
return 1;
},
[](dds::e_sqlite3_error_exc e, dds::e_init_repo init) {
dds_log(error,
"SQLite error while initializing repository in [{}]: {}",
init.path,
e.message);
return 1;
},
[](dds::e_system_error_exc e, dds::e_repo_delete_targz tgz, dds::package_id pkg_id) {
dds_log(error,
"Cannot delete requested package '{}' from repository (Archive {}): {}",
pkg_id.to_string(),
tgz.path,
e.message);
return 1;
},
[](dds::e_system_error_exc e) {
dds_log(error, "Unhandled system_error: {}", e.message);
return 1;
},
[](boost::leaf::diagnostic_info const& info) {
dds_log(error, "Unknown error: {}", info);
return 42;
});
}
};

/*
######## ######## ######## #######
## ## ## ## ## ## ##
@@ -910,6 +1111,7 @@ int main_fn(const std::vector<std::string>& argv) {
cli_build build{cli};
cli_sdist sdist{cli};
cli_repo repo{cli};
cli_repoman repoman{cli};
cli_catalog catalog{cli};
cli_build_deps build_deps{cli};

@@ -939,6 +1141,8 @@ int main_fn(const std::vector<std::string>& argv) {
return sdist.run();
} else if (repo.cmd) {
return repo.run();
} else if (repoman.cmd) {
return repoman.run();
} else if (catalog.cmd) {
return catalog.run();
} else if (build_deps.cmd) {

+ 7
- 2
src/dds/package/manifest.cpp View File

@@ -103,9 +103,14 @@ package_manifest parse_json(const json5::data& data, std::string_view fpath) {

package_manifest package_manifest::load_from_file(const fs::path& fpath) {
auto content = slurp_file(fpath);
auto data = json5::parse_data(content);
return load_from_json5_str(content, fpath.string());
}

package_manifest package_manifest::load_from_json5_str(std::string_view content,
std::string_view input_name) {
auto data = json5::parse_data(content);
try {
return parse_json(data, fpath.string());
return parse_json(data, input_name);
} catch (const semester::walk_error& e) {
throw_user_error<errc::invalid_pkg_manifest>(e.what());
}

+ 4
- 0
src/dds/package/manifest.hpp View File

@@ -35,6 +35,10 @@ struct package_manifest {
* Load a package manifest from a file on disk.
*/
static package_manifest load_from_file(path_ref);
/**
* @brief Load a package manifest from an in-memory string
*/
static package_manifest load_from_json5_str(std::string_view, std::string_view input_name);

/**
* Find a package manifest contained within a directory. This will search

+ 216
- 0
src/dds/repoman/repoman.cpp View File

@@ -0,0 +1,216 @@
#include "./repoman.hpp"

#include <dds/package/manifest.hpp>
#include <dds/util/log.hpp>
#include <dds/util/result.hpp>

#include <neo/gzip.hpp>
#include <neo/inflate.hpp>
#include <neo/io/stream/buffers.hpp>
#include <neo/io/stream/file.hpp>
#include <neo/sqlite3/exec.hpp>
#include <neo/sqlite3/single.hpp>
#include <neo/sqlite3/transaction.hpp>
#include <neo/tar/ustar.hpp>
#include <neo/transform_io.hpp>
#include <neo/utility.hpp>
#include <nlohmann/json.hpp>

using namespace dds;

namespace nsql = neo::sqlite3;
using namespace nsql::literals;

namespace {

void migrate_db_1(nsql::database_ref db) {
db.exec(R"(
CREATE TABLE dds_repo_packages (
package_id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
version TEXT NOT NULL,
description TEXT NOT NULL,
UNIQUE (name, version)
);

CREATE TABLE dds_repo_package_deps (
dep_id INTEGER PRIMARY KEY,
package_id INTEGER NOT NULL
REFERENCES dds_repo_packages
ON DELETE CASCADE,
dep_name TEXT NOT NULL,
low TEXT NOT NULL,
high TEXT NOT NULL,
UNIQUE(package_id, dep_name)
);
)");
}

void ensure_migrated(nsql::database_ref db, std::optional<std::string_view> name) {
db.exec(R"(
PRAGMA foreign_keys = 1;
CREATE TABLE IF NOT EXISTS dds_repo_meta (
meta_version INTEGER DEFAULT 1,
version INTEGER NOT NULL,
name TEXT NOT NULL
);

-- Insert the initial metadata
INSERT INTO dds_repo_meta (version, name)
SELECT 0, 'dds-repo-' || lower(hex(randomblob(6)))
WHERE NOT EXISTS (SELECT 1 FROM dds_repo_meta);
)");
nsql::transaction_guard tr{db};

auto meta_st = db.prepare("SELECT version FROM dds_repo_meta");
auto [version] = nsql::unpack_single<int>(meta_st);

constexpr int current_database_version = 1;
if (version < 1) {
migrate_db_1(db);
}

nsql::exec(db.prepare("UPDATE dds_repo_meta SET version=?"), current_database_version);
if (name) {
nsql::exec(db.prepare("UPDATE dds_repo_meta SET name=?"), *name);
}
}

} // namespace

repo_manager repo_manager::create(path_ref directory, std::optional<std::string_view> name) {
{
DDS_E_SCOPE(e_init_repo{directory});
fs::create_directories(directory);
auto db_path = directory / "repo.db";
auto db = nsql::database::open(db_path.string());
DDS_E_SCOPE(e_init_repo_db{db_path});
DDS_E_SCOPE(e_open_repo_db{db_path});
ensure_migrated(db, name);
fs::create_directories(directory / "data");
}
return open(directory);
}

repo_manager repo_manager::open(path_ref directory) {
DDS_E_SCOPE(e_open_repo{directory});
auto db_path = directory / "repo.db";
DDS_E_SCOPE(e_open_repo_db{db_path});
if (!fs::is_regular_file(db_path)) {
throw std::system_error(make_error_code(std::errc::no_such_file_or_directory),
"The database file does not exist");
}
auto db = nsql::database::open(db_path.string());
ensure_migrated(db, std::nullopt);
return repo_manager{fs::canonical(directory), std::move(db)};
}

std::string repo_manager::name() const noexcept {
auto [name] = nsql::unpack_single<std::string>(_stmts("SELECT name FROM dds_repo_meta"_sql));
return name;
}

void repo_manager::import_targz(path_ref tgz_file) {
neo_assertion_breadcrumbs("Importing targz file", tgz_file.string());
DDS_E_SCOPE(e_repo_import_targz{tgz_file});
dds_log(info, "Importing sdist archive [{}]", tgz_file.string());
neo::ustar_reader tar{
neo::buffer_transform_source{neo::stream_io_buffers{
neo::file_stream::open(tgz_file, neo::open_mode::read)},
neo::gzip_decompressor{neo::inflate_decompressor{}}}};

std::optional<package_manifest> man;

for (auto mem : tar) {
if (fs::path(mem.filename_str()).lexically_normal()
== neo::oper::none_of("package.jsonc", "package.json5", "package.json")) {
continue;
}

auto content = tar.all_data();
auto synth_filename = tgz_file / mem.filename_str();
man = package_manifest::load_from_json5_str(std::string_view(content),
synth_filename.string());
break;
}

if (!man) {
dds_log(critical,
"Given archive [{}] does not contain a package manifest file",
tgz_file.string());
throw std::runtime_error("Invalid package archive");
}

DDS_E_SCOPE(man->pkg_id);

neo::sqlite3::transaction_guard tr{_db};

dds_log(debug, "Recording package {}@{}", man->pkg_id.name, man->pkg_id.version.to_string());
nsql::exec( //
_stmts(R"(
INSERT INTO dds_repo_packages (name, version, description)
VALUES (?, ?, 'No description')
)"_sql),
man->pkg_id.name,
man->pkg_id.version.to_string());

auto package_id = _db.last_insert_rowid();

auto& insert_dep_st = _stmts(R"(
INSERT INTO dds_repo_package_deps(package_id, dep_name, low, high)
VALUES (?, ?, ?, ?)
)"_sql);
for (auto& dep : man->dependencies) {
assert(dep.versions.num_intervals() == 1);
auto iv_1 = *dep.versions.iter_intervals().begin();
dds_log(trace, " Depends on: {}", dep.to_string());
nsql::exec(insert_dep_st,
package_id,
dep.name,
iv_1.low.to_string(),
iv_1.high.to_string());
}

auto dest_dir = data_dir() / man->pkg_id.name;
auto dest_path = dest_dir / fmt::format("{}.tar.gz", man->pkg_id.version.to_string());
fs::create_directories(dest_dir);
fs::copy(tgz_file, dest_path);

tr.commit();
}

void repo_manager::delete_package(package_id pkg_id) {
neo::sqlite3::transaction_guard tr{_db};

DDS_E_SCOPE(pkg_id);

nsql::exec( //
_stmts(R"(
DELETE FROM dds_repo_packages
WHERE name = ?
AND version = ?
)"_sql),
pkg_id.name,
pkg_id.version.to_string());
/// XXX: Verify with _db.changes() that we actually deleted one row

auto name_dir = data_dir() / pkg_id.name;
auto ver_file = name_dir / fmt::format("{}.tar.gz", pkg_id.version.to_string());

DDS_E_SCOPE(e_repo_delete_targz{ver_file});

if (!fs::is_regular_file(ver_file)) {
throw std::system_error(std::make_error_code(std::errc::no_such_file_or_directory),
"No source archive for the requested package");
}

fs::remove(ver_file);

tr.commit();

std::error_code ec;
fs::remove(name_dir, ec);
if (ec && ec != std::errc::directory_not_empty) {
throw std::system_error(ec, "Failed to delete package name directory");
}
}

+ 70
- 0
src/dds/repoman/repoman.hpp View File

@@ -0,0 +1,70 @@
#pragma once

#include <dds/package/id.hpp>
#include <dds/util/fs.hpp>

#include <neo/sqlite3/database.hpp>
#include <neo/sqlite3/iter_tuples.hpp>
#include <neo/sqlite3/statement_cache.hpp>
#include <range/v3/view/transform.hpp>

namespace dds {

struct e_init_repo {
fs::path path;
};

struct e_open_repo {
fs::path path;
};

struct e_init_repo_db {
fs::path path;
};

struct e_open_repo_db {
fs::path path;
};

struct e_repo_import_targz {
fs::path path;
};

struct e_repo_delete_targz {
fs::path path;
};

class repo_manager {
neo::sqlite3::database _db;
mutable neo::sqlite3::statement_cache _stmts{_db};
fs::path _root;

explicit repo_manager(path_ref root, neo::sqlite3::database db)
: _db(std::move(db))
, _root(root) {}

public:
repo_manager(repo_manager&&) = default;

static repo_manager create(path_ref directory, std::optional<std::string_view> name);
static repo_manager open(path_ref directory);

auto data_dir() const noexcept { return _root / "data"; }
path_ref root() const noexcept { return _root; }
std::string name() const noexcept;

void import_targz(path_ref tgz_path);
void delete_package(package_id id);

auto all_packages() const noexcept {
using namespace neo::sqlite3::literals;
auto& st = _stmts("SELECT name, version FROM dds_repo_packages"_sql);
auto tups = neo::sqlite3::iter_tuples<std::string, std::string>(st);
return tups | ranges::views::transform([](auto&& pair) {
auto [name, version] = pair;
return package_id{name, semver::version::parse(version)};
});
}
};

} // namespace dds

+ 31
- 0
src/dds/repoman/repoman.test.cpp View File

@@ -0,0 +1,31 @@
#include <dds/repoman/repoman.hpp>

#include <dds/temp.hpp>
#include <neo/sqlite3/error.hpp>

#include <catch2/catch.hpp>

namespace {

const auto THIS_FILE = dds::fs::canonical(__FILE__);
const auto THIS_DIR = THIS_FILE.parent_path();
const auto REPO_ROOT = (THIS_DIR / "../../../").lexically_normal();
const auto DATA_DIR = REPO_ROOT / "data";

} // namespace

TEST_CASE("Open a repository") {
auto tdir = dds::temporary_dir::create();
auto repo = dds::repo_manager::create(tdir.path(), "test-repo");
auto neo_url_tgz = DATA_DIR / "neo-url@0.2.1.tar.gz";
repo.import_targz(neo_url_tgz);
CHECK(dds::fs::is_directory(repo.data_dir() / "neo-url/"));
CHECK(dds::fs::is_regular_file(repo.data_dir() / "neo-url/0.2.1.tar.gz"));
CHECK_THROWS_AS(repo.import_targz(neo_url_tgz), neo::sqlite3::constraint_unique_error);
repo.delete_package(dds::package_id::parse("neo-url@0.2.1"));
CHECK_FALSE(dds::fs::is_regular_file(repo.data_dir() / "neo-url/0.2.1.tar.gz"));
CHECK_FALSE(dds::fs::is_directory(repo.data_dir() / "neo-url"));
CHECK_THROWS_AS(repo.delete_package(dds::package_id::parse("neo-url@0.2.1")),
std::system_error);
CHECK_NOTHROW(repo.import_targz(neo_url_tgz));
}

+ 15
- 0
src/dds/util/result.cpp View File

@@ -0,0 +1,15 @@
#include "./result.hpp"

#include <neo/sqlite3/error.hpp>

dds::error_id dds::capture_exception() {
try {
throw;
} catch (const neo::sqlite3::sqlite3_error& e) {
return current_error().load(e_sqlite3_error_exc{std::string(e.what()), e.code()},
e.code(),
neo::sqlite3::errc{e.code().value()});
} catch (const std::system_error& e) {
return current_error().load(e_system_error_exc{std::string(e.what()), e.code()}, e.code());
}
}

+ 46
- 0
src/dds/util/result.hpp View File

@@ -0,0 +1,46 @@
#pragma once

#include <neo/pp.hpp>
#include <boost/leaf/on_error.hpp>
#include <boost/leaf/result.hpp>

#include <exception>
#include <string>

namespace dds {

using boost::leaf::current_error;
using boost::leaf::error_id;
using boost::leaf::new_error;
using boost::leaf::result;

/**
* @brief Error object representing a captured system_error exception
*/
struct e_system_error_exc {
std::string message;
std::error_code code;
};

/**
* @brief Error object representing a captured neo::sqlite3::sqlite3_error
*/
struct e_sqlite3_error_exc {
std::string message;
std::error_code code;
};

/**
* @brief Capture currently in-flight special exceptions as new error object. Works around a bug in
* Boost.LEAF when catching std::system error.
*/
error_id capture_exception();

/**
* @brief Generate a leaf::on_error object that loads the given expression into the currently
* in-flight error if the current scope is exitted via exception or a bad result<>
*/
#define DDS_E_SCOPE(...) \
auto NEO_CONCAT(_err_info_, __LINE__) = boost::leaf::on_error([&] { return __VA_ARGS__; })

} // namespace dds

Write Preview
Loading…
Cancel
Save