visus
/
dds
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

Merge branch 'feature/repodb' into develop

default_compile_flags
vector-of-bool 5 years ago
parent
1588722c84 f9b4d5f85c
commit
b9d438ab86
26 changed files with 935 additions and 250 deletions
Split View Show Diff Stats
  1. +2
    -2
      package.dds
  2. +2
    -2
      remote.dds
  3. +336
    -0
      src/dds/catalog/catalog.cpp
  4. +54
    -0
      src/dds/catalog/catalog.hpp
  5. +76
    -0
      src/dds/catalog/catalog.test.cpp
  6. +63
    -0
      src/dds/catalog/get.cpp
  7. +17
    -0
      src/dds/catalog/get.hpp
  8. +20
    -0
      src/dds/catalog/git.hpp
  9. +203
    -29
      src/dds/dds.main.cpp
  10. +0
    -135
      src/dds/repo/remote.cpp
  11. +0
    -75
      src/dds/repo/remote.hpp
  12. +2
    -2
      tests/basics/test_app_only.py
  13. +8
    -0
      tests/catalog/create_test.py
  14. +34
    -0
      tests/catalog/get_test.py
  15. +29
    -0
      tests/catalog/import_test.py
  16. +29
    -1
      tests/dds.py
  17. +2
    -0
      tests/deps/do_test.py
  18. +24
    -0
      tests/deps/git-remote/catalog.json
  19. +0
    -2
      tests/deps/git-remote/remote.dds
  20. +4
    -0
      tests/deps/no-deps/catalog.json
  21. +0
    -0
      tests/deps/no-deps/remote.dds
  22. +14
    -0
      tests/deps/use-remote/catalog.json
  23. +0
    -1
      tests/deps/use-remote/remote.dds
  24. +15
    -0
      tests/deps/use-spdlog/project/catalog.json
  25. +0
    -1
      tests/deps/use-spdlog/project/remote.dds
  26. +1
    -0
      tests/deps/use-spdlog/use_spdlog_test.py

+ 2
- 2
package.dds View File

@@ -6,7 +6,7 @@ Depends: spdlog 1.4.2
Depends: ms-wil 2019.11.10
Depends: range-v3 0.9.1
Depends: nlohmann-json 3.7.1
Depends: neo-sqlite3 0.2.0
Depends: semver 0.1.0
Depends: neo-sqlite3 0.2.2
Depends: semver 0.2.0

Test-Driver: Catch-Main

+ 2
- 2
remote.dds View File

@@ -10,5 +10,5 @@ Remote-Package: ms-wil 2019.11.10; git url=https://github.com/vector-of-bool/wil
# XXX: Don't depend on a moving revision!
Remote-Package: neo-buffer 0.1.0; git url=https://github.com/vector-of-bool/neo-buffer.git ref=develop

Remote-Package: neo-sqlite3 0.2.0; git url=https://github.com/vector-of-bool/neo-sqlite3.git ref=0.2.0
Remote-Package: semver 0.1.0; git url=https://github.com/vector-of-bool/semver.git ref=0.1.0
Remote-Package: neo-sqlite3 0.2.2; git url=https://github.com/vector-of-bool/neo-sqlite3.git ref=0.2.2
Remote-Package: semver 0.2.0; git url=https://github.com/vector-of-bool/semver.git ref=0.2.0

+ 336
- 0
src/dds/catalog/catalog.cpp View File

@@ -0,0 +1,336 @@
#include "./catalog.hpp"

#include <neo/sqlite3/exec.hpp>
#include <neo/sqlite3/iter_tuples.hpp>
#include <neo/sqlite3/single.hpp>
#include <nlohmann/json.hpp>
#include <range/v3/range/conversion.hpp>
#include <range/v3/view/join.hpp>
#include <range/v3/view/transform.hpp>

#include <spdlog/spdlog.h>

using namespace dds;

namespace sqlite3 = neo::sqlite3;
using namespace sqlite3::literals;

namespace {

void migrate_repodb_1(sqlite3::database& db) {
db.exec(R"(
CREATE TABLE dds_cat_pkgs (
pkg_id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
version TEXT NOT NULL,
git_url TEXT,
git_ref TEXT,
lm_name TEXT,
lm_namespace TEXT,
UNIQUE(name, version),
CONSTRAINT has_remote_info CHECK(
(
git_url NOT NULL
AND git_ref NOT NULL
)
),
CONSTRAINT valid_lm_info CHECK(
(
lm_name NOT NULL
AND lm_namespace NOT NULL
)
+
(
lm_name ISNULL
AND lm_namespace ISNULL
)
= 1
)
);

CREATE TABLE dds_cat_pkg_deps (
dep_id INTEGER PRIMARY KEY AUTOINCREMENT,
pkg_id INTEGER NOT NULL REFERENCES dds_cat_pkgs(pkg_id),
dep_name TEXT NOT NULL,
low TEXT NOT NULL,
high TEXT NOT NULL,
UNIQUE(pkg_id, dep_name)
);
)");
}

void ensure_migrated(sqlite3::database& db) {
sqlite3::transaction_guard tr{db};
db.exec(R"(
PRAGMA foreign_keys = 1;
CREATE TABLE IF NOT EXISTS dds_cat_meta AS
WITH init(meta) AS (VALUES ('{"version": 0}'))
SELECT * FROM init;
)");
auto meta_st = db.prepare("SELECT meta FROM dds_cat_meta");
auto [meta_json] = sqlite3::unpack_single<std::string>(meta_st);

auto meta = nlohmann::json::parse(meta_json);
if (!meta.is_object()) {
throw std::runtime_error("Corrupted repository database file.");
}

auto version_ = meta["version"];
if (!version_.is_number_integer()) {
throw std::runtime_error("Corrupted repository database file [bad dds_meta.version]");
}
int version = version_;
if (version < 1) {
migrate_repodb_1(db);
}
meta["version"] = 1;
exec(db, "UPDATE dds_cat_meta SET meta=?", std::forward_as_tuple(meta.dump()));
}

} // namespace

catalog catalog::open(const std::string& db_path) {
auto db = sqlite3::database::open(db_path);
try {
ensure_migrated(db);
} catch (const sqlite3::sqlite3_error& e) {
spdlog::critical(
"Failed to load the repository databsae. It appears to be invalid/corrupted. The "
"exception message is: {}",
e.what());
throw;
}
return catalog(std::move(db));
}

catalog::catalog(sqlite3::database db)
: _db(std::move(db)) {}

void catalog::_store_pkg(const package_info& pkg, const git_remote_listing& git) {
auto lm_usage = git.auto_lib.value_or(lm::usage{});
sqlite3::exec( //
_stmt_cache,
R"(
INSERT INTO dds_cat_pkgs (
name,
version,
git_url,
git_ref,
lm_name,
lm_namespace
) VALUES (
?1,
?2,
?3,
?4,
CASE WHEN ?5 = '' THEN NULL ELSE ?5 END,
CASE WHEN ?6 = '' THEN NULL ELSE ?6 END
)
)"_sql,
std::forward_as_tuple( //
pkg.ident.name,
pkg.ident.version.to_string(),
git.url,
git.ref,
lm_usage.name,
lm_usage.namespace_));
}

void catalog::store(const package_info& pkg) {
sqlite3::transaction_guard tr{_db};

std::visit([&](auto&& remote) { _store_pkg(pkg, remote); }, pkg.remote);

auto db_pkg_id = _db.last_insert_rowid();
auto& new_dep_st = _stmt_cache(R"(
INSERT INTO dds_cat_pkg_deps (
pkg_id,
dep_name,
low,
high
) VALUES (
?,
?,
?,
?
)
)"_sql);
for (const auto& dep : pkg.deps) {
new_dep_st.reset();
sqlite3::exec(new_dep_st,
std::forward_as_tuple(db_pkg_id,
dep.name,
dep.version.to_string(),
"[placeholder]"));
}
}

std::optional<package_info> catalog::get(const package_id& pk_id) const noexcept {
auto& st = _stmt_cache(R"(
SELECT
pkg_id,
name,
version,
git_url,
git_ref,
lm_name,
lm_namespace
FROM dds_cat_pkgs
WHERE name = ? AND version = ?
)"_sql);
st.reset();
st.bindings = std::forward_as_tuple(pk_id.name, pk_id.version.to_string());
auto opt_tup = sqlite3::unpack_single_opt<std::int64_t,
std::string,
std::string,
std::optional<std::string>,
std::optional<std::string>,
std::optional<std::string>,
std::optional<std::string>>(st);
if (!opt_tup) {
return std::nullopt;
}
const auto& [pkg_id, name, version, git_url, git_ref, lm_name, lm_namespace] = *opt_tup;
assert(pk_id.name == name);
assert(pk_id.version == semver::version::parse(version));
assert(git_url);
assert(git_ref);

auto deps = sqlite3::exec_iter<std::string, std::string>( //
_stmt_cache,
R"(
SELECT dep_name, low
FROM dds_cat_pkg_deps
WHERE pkg_id = ?
)"_sql,
std::tie(pkg_id))
| ranges::views::transform([](auto&& pair) {
const auto& [name, ver] = pair;
return dependency{name, semver::version::parse(ver)};
}) //
| ranges::to_vector;

return package_info{
pk_id,
deps,
git_remote_listing{
*git_url,
*git_ref,
lm_name ? std::make_optional(lm::usage{*lm_name, *lm_namespace}) : std::nullopt,
},
};
}

std::vector<package_id> catalog::by_name(std::string_view sv) const noexcept {
return sqlite3::exec_iter<std::string, std::string>( //
_stmt_cache,
R"(
SELECT name, version
FROM dds_cat_pkgs
WHERE name = ?
)"_sql,
std::tie(sv)) //
| ranges::views::transform([](auto& pair) {
auto& [name, ver] = pair;
return package_id{name, semver::version::parse(ver)};
})
| ranges::to_vector;
}

std::vector<dependency> catalog::dependencies_of(const package_id& pkg) const noexcept {
return sqlite3::exec_iter<std::string,
std::string>( //
_stmt_cache,
R"(
WITH this_pkg_id AS (
SELECT pkg_id
FROM dds_cat_pkgs
WHERE name = ? AND version = ?
)
SELECT dep_name, low
FROM dds_cat_pkg_deps
WHERE pkg_id IN this_pkg_id
ORDER BY dep_name
)"_sql,
std::forward_as_tuple(pkg.name, pkg.version.to_string())) //
| ranges::views::transform([](auto&& pair) {
auto& [name, ver] = pair;
return dependency{name, semver::version::parse(ver)};
}) //
| ranges::to_vector;
}

namespace {

void check_json(bool b, std::string_view what) {
if (!b) {
throw std::runtime_error("Unable to read repository JSON: " + std::string(what));
}
}

} // namespace

void catalog::import_json_str(std::string_view content) {
using nlohmann::json;

auto root = json::parse(content);
check_json(root.is_object(), "Root of JSON must be an object (key-value mapping)");

auto version = root["version"];
check_json(version.is_number_integer(), "/version must be an integral value");
check_json(version <= 1, "/version is too new. We don't know how to parse this.");

auto packages = root["packages"];
check_json(packages.is_object(), "/packages must be an object");

sqlite3::transaction_guard tr{_db};

for (const auto& [pkg_name_, versions_map] : packages.items()) {
std::string pkg_name = pkg_name_;
check_json(versions_map.is_object(),
fmt::format("/packages/{} must be an object", pkg_name));

for (const auto& [version_, pkg_info] : versions_map.items()) {
auto version = semver::version::parse(version_);
check_json(pkg_info.is_object(),
fmt::format("/packages/{}/{} must be an object", pkg_name, version_));

auto deps = pkg_info["depends"];
check_json(deps.is_object(),
fmt::format("/packages/{}/{}/depends must be an object",
pkg_name,
version_));

package_info info{{pkg_name, version}, {}, {}};
for (const auto& [dep_name, dep_version] : deps.items()) {
check_json(dep_version.is_string(),
fmt::format("/packages/{}/{}/depends/{} must be a string",
pkg_name,
version_,
dep_name));
info.deps.push_back({
std::string(dep_name),
semver::version::parse(std::string(dep_version)),
});
}

auto git_remote = pkg_info["git"];
if (!git_remote.is_null()) {
check_json(git_remote.is_object(), "`git` must be an object");
std::string url = git_remote["url"];
std::string ref = git_remote["ref"];
auto lm_usage = git_remote["auto-lib"];
std::optional<lm::usage> autolib;
if (!lm_usage.is_null()) {
autolib = lm::split_usage_string(std::string(lm_usage));
}
info.remote = git_remote_listing{url, ref, autolib};
} else {
throw std::runtime_error(
fmt::format("No remote info for /packages/{}/{}", pkg_name, version_));
}

store(info);
}
}
}

+ 54
- 0
src/dds/catalog/catalog.hpp View File

@@ -0,0 +1,54 @@
#pragma once

#include <dds/deps.hpp>
#include <dds/package_id.hpp>
#include <dds/util/fs.hpp>
#include <dds/catalog/git.hpp>

#include <neo/sqlite3/database.hpp>
#include <neo/sqlite3/statement.hpp>
#include <neo/sqlite3/statement_cache.hpp>
#include <neo/sqlite3/transaction.hpp>

#include <variant>
#include <vector>

namespace dds {

struct package_info {
package_id ident;
std::vector<dependency> deps;

std::variant<git_remote_listing> remote;
};

class catalog {
neo::sqlite3::database _db;
mutable neo::sqlite3::statement_cache _stmt_cache{_db};

explicit catalog(neo::sqlite3::database db);
catalog(const catalog&) = delete;

void _store_pkg(const package_info&, const git_remote_listing&);

public:
catalog(catalog&&) = default;
catalog& operator=(catalog&&) = default;

static catalog open(const std::string& db_path);
static catalog open(path_ref db_path) { return open(db_path.string()); }

void store(const package_info& info);
std::optional<package_info> get(const package_id& id) const noexcept;

std::vector<package_id> by_name(std::string_view sv) const noexcept;
std::vector<dependency> dependencies_of(const package_id& pkg) const noexcept;

void import_json_str(std::string_view json_str);
void import_json_file(path_ref json_path) {
auto content = dds::slurp_file(json_path);
import_json_str(content);
}
};

} // namespace dds

+ 76
- 0
src/dds/catalog/catalog.test.cpp View File

@@ -0,0 +1,76 @@
#include <dds/catalog/catalog.hpp>

#include <catch2/catch.hpp>

using namespace std::literals;

TEST_CASE("Create a simple database") {
// Just create and run migrations on an in-memory database
auto repo = dds::catalog::open(":memory:"s);
}

class catalog_test_case {
public:
dds::catalog db = dds::catalog::open(":memory:"s);
};

TEST_CASE_METHOD(catalog_test_case, "Store a simple package") {
db.store(dds::package_info{
dds::package_id("foo", semver::version::parse("1.2.3")),
{},
dds::git_remote_listing{"http://example.com", "master", std::nullopt},
});
CHECK_THROWS(db.store(dds::package_info{
dds::package_id("foo", semver::version::parse("1.2.3")),
{},
dds::git_remote_listing{"http://example.com", "master", std::nullopt},
}));

auto pkgs = db.by_name("foo");
REQUIRE(pkgs.size() == 1);
}

TEST_CASE_METHOD(catalog_test_case, "Package requirements") {
db.store(dds::package_info{
dds::package_id{"foo", semver::version::parse("1.2.3")},
{
{"bar", semver::version::parse("1.2.5")},
{"baz", semver::version::parse("5.3.2")},
},
dds::git_remote_listing{"http://example.com", "master", std::nullopt},
});
auto pkgs = db.by_name("foo");
REQUIRE(pkgs.size() == 1);
CHECK(pkgs[0].name == "foo");
auto deps = db.dependencies_of(pkgs[0]);
CHECK(deps.size() == 2);
CHECK(deps[0].name == "bar");
CHECK(deps[1].name == "baz");
}

TEST_CASE_METHOD(catalog_test_case, "Parse JSON repo") {
db.import_json_str(R"({
"version": 1,
"packages": {
"foo": {
"1.2.3": {
"depends": {
"bar": "4.2.1"
},
"git": {
"url": "http://example.com",
"ref": "master"
}
}
}
}
})");
auto pkgs = db.by_name("foo");
REQUIRE(pkgs.size() == 1);
CHECK(pkgs[0].name == "foo");
CHECK(pkgs[0].version == semver::version::parse("1.2.3"));
auto deps = db.dependencies_of(pkgs[0]);
REQUIRE(deps.size() == 1);
CHECK(deps[0].name == "bar");
CHECK(deps[0].version == semver::version::parse("4.2.1"));
}

+ 63
- 0
src/dds/catalog/get.cpp View File

@@ -0,0 +1,63 @@
#include "./get.hpp"

#include <dds/catalog/catalog.hpp>
#include <dds/proc.hpp>

#include <spdlog/spdlog.h>

using namespace dds;

namespace {

temporary_sdist do_pull_sdist(const package_info& listing, const git_remote_listing& git) {
auto tmpdir = dds::temporary_dir::create();
using namespace std::literals;
spdlog::info("Cloning Git repository: {} [{}] ...", git.url, git.ref);
auto command = {"git"s,
"clone"s,
"--depth=1"s,
"--branch"s,
git.ref,
git.url,
tmpdir.path().generic_string()};
auto git_res = run_proc(command);
if (!git_res.okay()) {
throw std::runtime_error(
fmt::format("Git clone operation failed [Git command: {}] [Exitted {}]:\n{}",
quote_command(command),
git_res.retc,
git_res.output));
}
spdlog::info("Create sdist from clone ...");
if (git.auto_lib.has_value()) {
spdlog::info("Generating library data automatically");
auto pkg_strm = dds::open(tmpdir.path() / "package.dds", std::ios::binary | std::ios::out);
pkg_strm << "Name: " << listing.ident.name << '\n' //
<< "Version: " << listing.ident.version.to_string() << '\n' //
<< "Namespace: " << git.auto_lib->namespace_;
auto lib_strm = dds::open(tmpdir.path() / "library.dds", std::ios::binary | std::ios::out);
lib_strm << "Name: " << git.auto_lib->name;
}

sdist_params params;
params.project_dir = tmpdir.path();
auto sd_tmp_dir = dds::temporary_dir::create();
params.dest_path = sd_tmp_dir.path();
params.force = true;
auto sd = create_sdist(params);
return {sd_tmp_dir, sd};
}

} // namespace

temporary_sdist dds::get_package_sdist(const package_info& pkg) {
auto tsd = std::visit([&](auto&& remote) { return do_pull_sdist(pkg, remote); }, pkg.remote);
if (!(tsd.sdist.manifest.pk_id == pkg.ident)) {
throw std::runtime_error(fmt::format(
"The package name@version in the generated sdist does not match the name listed in "
"the remote listing file (expected '{}', but got '{}')",
pkg.ident.to_string(),
tsd.sdist.manifest.pk_id.to_string()));
}
return tsd;
}

+ 17
- 0
src/dds/catalog/get.hpp View File

@@ -0,0 +1,17 @@
#pragma once

#include <dds/sdist.hpp>
#include <dds/temp.hpp>

namespace dds {

struct package_info;

struct temporary_sdist {
temporary_dir tmpdir;
dds::sdist sdist;
};

temporary_sdist get_package_sdist(const package_info&);

} // namespace dds

+ 20
- 0
src/dds/catalog/git.hpp View File

@@ -0,0 +1,20 @@
#pragma once

#include <dds/util/fs.hpp>

#include <libman/package.hpp>

#include <optional>
#include <string>

namespace dds {

struct git_remote_listing {
std::string url;
std::string ref;
std::optional<lm::usage> auto_lib;

void clone(path_ref path) const;
};

} // namespace dds

+ 203
- 29
src/dds/dds.main.cpp View File

@@ -1,5 +1,6 @@
#include <dds/build.hpp>
#include <dds/repo/remote.hpp>
#include <dds/catalog/catalog.hpp>
#include <dds/catalog/get.hpp>
#include <dds/repo/repo.hpp>
#include <dds/sdist.hpp>
#include <dds/toolchain/from_dds.hpp>
@@ -45,8 +46,8 @@ struct toolchain_flag : string_flag {
}
};

struct repo_where_flag : path_flag {
repo_where_flag(args::Group& grp)
struct repo_path_flag : path_flag {
repo_path_flag(args::Group& grp)
: path_flag{grp,
"dir",
"Path to the DDS repository directory",
@@ -54,6 +55,17 @@ struct repo_where_flag : path_flag {
dds::repository::default_local_path()} {}
};

struct catalog_path_flag : path_flag {
catalog_path_flag(args::Group& cmd)
: path_flag(cmd,
"catalog-path",
"Override the path to the catalog database",
{"catalog", 'c'},
dds::dds_data_dir() / "catalog.db") {}

dds::catalog open() { return dds::catalog::open(Get()); }
};

/**
* Base class holds the actual argument parser
*/
@@ -92,6 +104,167 @@ struct common_project_flags {
dds::fs::current_path()};
};

/*
###### ### ######## ### ## ####### ######
## ## ## ## ## ## ## ## ## ## ## ##
## ## ## ## ## ## ## ## ## ##
## ## ## ## ## ## ## ## ## ## ####
## ######### ## ######### ## ## ## ## ##
## ## ## ## ## ## ## ## ## ## ## ##
###### ## ## ## ## ## ######## ####### ######
*/

struct cli_catalog {
cli_base& base;
args::Command cmd{base.cmd_group, "catalog", "Manage the package catalog"};
common_flags _common{cmd};

args::Group cat_group{cmd, "Catalog subcommands"};

struct {
cli_catalog& parent;
args::Command cmd{parent.cat_group, "create", "Create a catalog database"};
common_flags _common{cmd};

catalog_path_flag cat_path{cmd};

int run() {
// Simply opening the DB will initialize the catalog
cat_path.open();
return 0;
}
} create{*this};

struct {
cli_catalog& parent;
args::Command cmd{parent.cat_group, "import", "Import entries into a catalog"};
common_flags _common{cmd};

catalog_path_flag cat_path{cmd};
args::ValueFlagList<std::string>
json_paths{cmd,
"json",
"Import catalog entries from the given JSON files",
{"json", 'j'}};

int run() {
auto cat = cat_path.open();
for (const auto& json_fpath : json_paths.Get()) {
cat.import_json_file(json_fpath);
}
return 0;
}
} import{*this};

struct {
cli_catalog& parent;
args::Command cmd{parent.cat_group, "get", "Obtain an sdist from a catalog listing"};
common_flags _common{cmd};

catalog_path_flag cat_path{cmd};

path_flag out{cmd,
"out",
"The directory where the source distributions will be placed",
{"out-dir", 'o'},
dds::fs::current_path()};

args::PositionalList<std::string> requirements{cmd,
"requirement",
"The package IDs to obtain"};

int run() {
auto cat = cat_path.open();
for (const auto& req : requirements.Get()) {
auto id = dds::package_id::parse(req);
auto info = cat.get(id);
if (!info) {
throw std::runtime_error(
fmt::format("No package in the catalog matched the ID '{}'", req));
}
auto tsd = dds::get_package_sdist(*info);
auto out_path = out.Get();
auto dest = out_path / id.to_string();
spdlog::info("Create sdist at {}", dest.string());
dds::fs::remove_all(dest);
dds::safe_rename(tsd.sdist.path, dest);
}
return 0;
}
} get{*this};

struct {
cli_catalog& parent;
args::Command cmd{parent.cat_group, "add", "Manually add an entry to the catalog database"};
common_flags _common{cmd};

catalog_path_flag cat_path{cmd};

args::Positional<std::string> pkg_id{cmd,
"id",
"The name@version ID of the package to add",
args::Options::Required};

string_flag auto_lib{cmd,
"auto-lib",
"Set the auto-library information for this package",
{"auto-lib"}};

args::ValueFlagList<std::string> deps{cmd,
"depends",
"The dependencies of this package",
{"depends", 'd'}};

string_flag git_url{cmd, "git-url", "The Git url for the package", {"git-url"}};
string_flag git_ref{cmd,
"git-ref",
"The Git ref to from which the source distribution should be created",
{"git-ref"}};

int run() {
auto ident = dds::package_id::parse(pkg_id.Get());

std::vector<dds::dependency> deps;
for (const auto& dep : this->deps.Get()) {
auto dep_id = dds::package_id::parse(dep);
deps.push_back({dep_id.name, dep_id.version});
}

dds::package_info info{ident, std::move(deps), {}};

if (git_url) {
if (!git_ref) {
throw std::runtime_error(
"`--git-ref` must be specified when using `--git-url`");
}
auto git = dds::git_remote_listing{git_url.Get(), git_ref.Get(), std::nullopt};
if (auto_lib) {
git.auto_lib = lm::split_usage_string(auto_lib.Get());
}
info.remote = std::move(git);
}

cat_path.open().store(info);
return 0;
}
} add{*this};

int run() {
if (create.cmd) {
return create.run();
} else if (import.cmd) {
return import.run();
} else if (get.cmd) {
return get.run();
} else if (add.cmd) {
return add.run();
} else {
assert(false);
std::terminate();
}
}
};

/*
######## ######## ######## #######
## ## ## ## ## ## ##
@@ -107,7 +280,7 @@ struct cli_repo {
args::Command cmd{base.cmd_group, "repo", "Manage the package repository"};
common_flags _common{cmd};

repo_where_flag where{cmd};
repo_path_flag where{cmd};

args::Group repo_group{cmd, "Repo subcommands"};

@@ -118,8 +291,9 @@ struct cli_repo {

int run() {
auto list_contents = [&](dds::repository repo) {
auto same_name
= [](auto&& a, auto&& b) { return a.manifest.pk_id.name == b.manifest.pk_id.name; };
auto same_name = [](auto&& a, auto&& b) {
return a.manifest.pk_id.name == b.manifest.pk_id.name;
};

auto all = repo.iter_sdists();
auto grp_by_name = all //
@@ -226,8 +400,8 @@ struct cli_sdist {

common_project_flags project{cmd};

repo_where_flag repo_where{cmd};
args::Flag force{cmd,
repo_path_flag repo_where{cmd};
args::Flag force{cmd,
"replace-if-exists",
"Replace an existing export in the repository",
{"replace"}};
@@ -290,6 +464,7 @@ struct cli_build {
args::Flag build_apps{cmd, "build_apps", "Build applications", {"apps", 'A'}};
args::Flag export_{cmd, "export", "Generate a library export", {"export", 'E'}};
toolchain_flag tc_filepath{cmd};

args::Flag enable_warnings{cmd,
"enable_warnings",
"Enable compiler warnings",
@@ -378,18 +553,14 @@ struct cli_deps {
"Ensure we have local copies of the project dependencies"};
common_flags _common{cmd};

repo_where_flag repo_where{cmd};
path_flag remote_listing_file{
cmd,
"remote-listing",
"Path to a file containing listing of remote sdists and how to obtain them",
{'R', "remote-list"},
"remote.dds"};
repo_path_flag repo_where{cmd};
catalog_path_flag catalog_path{cmd};


int run() {
auto man = parent.load_package_manifest();
auto rd = dds::remote_directory::load_from_file(remote_listing_file.Get());
bool failed = false;
auto man = parent.load_package_manifest();
auto catalog = catalog_path.open();
bool failed = false;
dds::repository::with_repository( //
repo_where.Get(),
dds::repo_flags::write_lock | dds::repo_flags::create_if_absent,
@@ -397,13 +568,13 @@ struct cli_deps {
for (auto& dep : man.dependencies) {
auto exists = !!repo.find(dep.name, dep.version);
if (!exists) {
spdlog::info("Pull remote: {} {}", dep.name, dep.version.to_string());
auto opt_remote = rd.find(dep.name, dep.version);
if (opt_remote) {
auto tsd = opt_remote->pull_sdist();
spdlog::info("Pull remote: {}@{}", dep.name, dep.version.to_string());
auto opt_pkg = catalog.get(dds::package_id{dep.name, dep.version});
if (opt_pkg) {
auto tsd = dds::get_package_sdist(*opt_pkg);
repo.add_sdist(tsd.sdist, dds::if_exists::ignore);
} else {
spdlog::error("No remote listing for {} {}",
spdlog::error("No remote listing for {}@{}",
dep.name,
dep.version.to_string());
failed = true;
@@ -440,7 +611,7 @@ struct cli_deps {
"If specified, will not generate an INDEX.lmi",
{"skip-lmi"}};

repo_where_flag repo_where{cmd};
repo_path_flag repo_where{cmd};

toolchain_flag tc_filepath{cmd};

@@ -502,11 +673,12 @@ int main(int argc, char** argv) {
spdlog::set_pattern("[%H:%M:%S] [%^%-5l%$] %v");
args::ArgumentParser parser("DDS - The drop-dead-simple library manager");

cli_base cli{parser};
cli_build build{cli};
cli_sdist sdist{cli};
cli_repo repo{cli};
cli_deps deps{cli};
cli_base cli{parser};
cli_build build{cli};
cli_sdist sdist{cli};
cli_repo repo{cli};
cli_deps deps{cli};
cli_catalog catalog{cli};
try {
parser.ParseCLI(argc, argv);
} catch (const args::Help&) {
@@ -532,6 +704,8 @@ int main(int argc, char** argv) {
return repo.run();
} else if (deps.cmd) {
return deps.run();
} else if (catalog.cmd) {
return catalog.run();
} else {
assert(false);
std::terminate();

+ 0
- 135
src/dds/repo/remote.cpp View File

@@ -1,135 +0,0 @@
#include "./remote.hpp"

#include <dds/deps.hpp>
#include <dds/proc.hpp>
#include <dds/repo/repo.hpp>
#include <dds/sdist.hpp>
#include <dds/temp.hpp>
#include <dds/toolchain/toolchain.hpp>
#include <dds/util/shlex.hpp>

#include <spdlog/spdlog.h>

#include <libman/parse.hpp>

#include <algorithm>

using namespace dds;

namespace {

struct read_listing_item {
std::string_view _key;
std::set<remote_listing, remote_listing_compare_t>& out;

bool operator()(std::string_view context, std::string_view key, std::string_view value) {
if (key != _key) {
return false;
}

auto nested = lm::nested_kvlist::parse(value);
auto pk_id = package_id::parse(nested.primary);
put_listing(context, std::move(pk_id), nested.pairs);
return true;
}

void put_listing(std::string_view context, package_id pk_id, const lm::pair_list& pairs) {
if (pairs.find("git")) {
std::string url;
std::string ref;
std::optional<lm::usage> auto_id;
lm::read(fmt::format("{}: Parsing Git remote listing", context),
pairs,
lm::read_required("url", url),
lm::read_required("ref", ref),
lm::read_check_eq("git", ""),
lm::read_opt("auto", auto_id, &lm::split_usage_string),
lm::reject_unknown());
auto did_insert = out.emplace(remote_listing{std::move(pk_id),
git_remote_listing{url, ref, auto_id}})
.second;
if (!did_insert) {
spdlog::warn("Duplicate remote package defintion for {}", pk_id.to_string());
}
} else {
throw std::runtime_error(
fmt::format("Unable to determine remote type of package {}", pk_id.to_string()));
}
}
};

temporary_sdist do_pull_sdist(const remote_listing& listing, const git_remote_listing& git) {
auto tmpdir = dds::temporary_dir::create();
using namespace std::literals;
spdlog::info("Cloning repository: {} [{}] ...", git.url, git.ref);
auto command = {"git"s,
"clone"s,
"--depth=1"s,
"--branch"s,
git.ref,
git.url,
tmpdir.path().generic_string()};
auto git_res = run_proc(command);
if (!git_res.okay()) {
throw std::runtime_error(
fmt::format("Git clone operation failed [Git command: {}] [Exitted {}]:\n{}",
quote_command(command),
git_res.retc,
git_res.output));
}
spdlog::info("Create sdist from clone ...");
if (git.auto_lib.has_value()) {
spdlog::info("Generating library data automatically");
auto pkg_strm = dds::open(tmpdir.path() / "package.dds", std::ios::binary | std::ios::out);
pkg_strm << "Name: " << listing.pk_id.name << '\n' //
<< "Version: " << listing.pk_id.version.to_string() << '\n' //
<< "Namespace: " << git.auto_lib->namespace_;
auto lib_strm = dds::open(tmpdir.path() / "library.dds", std::ios::binary | std::ios::out);
lib_strm << "Name: " << git.auto_lib->name;
}

sdist_params params;
params.project_dir = tmpdir.path();
auto sd_tmp_dir = dds::temporary_dir::create();
params.dest_path = sd_tmp_dir.path();
params.force = true;
auto sd = create_sdist(params);
return {sd_tmp_dir, sd};
}

} // namespace

temporary_sdist remote_listing::pull_sdist() const {
auto tsd = visit([&](auto&& actual) { return do_pull_sdist(*this, actual); });
if (!(tsd.sdist.manifest.pk_id == pk_id)) {
throw std::runtime_error(fmt::format(
"The package name@version in the generated sdist does not match the name listed in "
"the remote listing file (expected '{}', but got '{}')",
pk_id.to_string(),
tsd.sdist.manifest.pk_id.to_string()));
}
return tsd;
}

remote_directory remote_directory::load_from_file(path_ref filepath) {
auto kvs = lm::parse_file(filepath);
listing_set listings;
lm::read(fmt::format("Loading remote package listing from {}", filepath.string()),
kvs,
read_listing_item{"Remote-Package", listings},
lm::reject_unknown());
return {std::move(listings)};
}

const remote_listing* remote_directory::find(std::string_view name, semver::version ver) const
noexcept {
auto found = _remotes.find(std::tie(name, ver));
if (found == _remotes.end()) {
return nullptr;
}
return &*found;
}

void remote_directory::ensure_all_local(const repository&) const {
spdlog::critical("Dependency download is not fully implemented!");
}

+ 0
- 75
src/dds/repo/remote.hpp View File

@@ -1,75 +0,0 @@
#pragma once

#include <dds/util/fs.hpp>

#include <dds/sdist.hpp>
#include <dds/temp.hpp>
#include <libman/library.hpp>
#include <semver/version.hpp>

#include <set>
#include <string>
#include <tuple>
#include <utility>
#include <variant>

namespace dds {

struct temporary_sdist {
temporary_dir tmpdir;
dds::sdist sdist;
};

struct git_remote_listing {
std::string url;
std::string ref;
std::optional<lm::usage> auto_lib;

void clone(path_ref path) const;
};

struct remote_listing {
package_id pk_id;
std::variant<git_remote_listing> remote;

template <typename Func>
decltype(auto) visit(Func&& fn) const {
return std::visit(std::forward<Func>(fn), remote);
}

temporary_sdist pull_sdist() const;
};

inline constexpr struct remote_listing_compare_t {
using is_transparent = int;
bool operator()(const remote_listing& lhs, const remote_listing& rhs) const {
return lhs.pk_id < rhs.pk_id;
}
template <typename Name, typename Version>
bool operator()(const remote_listing& lhs, const std::tuple<Name, Version>& rhs) const {
auto&& [name, ver] = rhs;
return lhs.pk_id < package_id{name, ver};
}
template <typename Name, typename Version>
bool operator()(const std::tuple<Name, Version>& lhs, const remote_listing& rhs) const {
auto&& [name, ver] = lhs;
return package_id{name, ver} < rhs.pk_id;
}
} remote_listing_compare;

class remote_directory {
using listing_set = std::set<remote_listing, remote_listing_compare_t>;
listing_set _remotes;

remote_directory(listing_set s)
: _remotes(std::move(s)) {}

public:
static remote_directory load_from_file(path_ref);

void ensure_all_local(const class repository& repo) const;

const remote_listing* find(std::string_view name, semver::version ver) const noexcept;
};

} // namespace dds

+ 2
- 2
tests/basics/test_app_only.py View File

@@ -3,8 +3,8 @@ from tests import DDS
from tests.fileutil import set_contents


def test_lib_with_just_app(dds: DDS, scope: ExitStack):
scope.enter_context(
def test_lib_with_just_app(dds: DDS):
dds.scope.enter_context(
set_contents(
dds.source_root / 'src/foo.main.cpp',
b'int main() {}',

+ 8
- 0
tests/catalog/create_test.py View File

@@ -0,0 +1,8 @@
from tests import dds, DDS
from tests.fileutil import ensure_dir


def test_create_catalog(dds: DDS):
dds.scope.enter_context(ensure_dir(dds.build_dir))
dds.catalog_create()
assert dds.catalog_path.is_file()

+ 34
- 0
tests/catalog/get_test.py View File

@@ -0,0 +1,34 @@
import json

from tests import dds, DDS
from tests.fileutil import ensure_dir


def test_get(dds: DDS):
dds.scope.enter_context(ensure_dir(dds.build_dir))
dds.catalog_create()

json_path = dds.build_dir / 'catalog.json'
import_data = {
'version': 1,
'packages': {
'neo-sqlite3': {
'0.2.2': {
'depends': {},
'git': {
'url': 'https://github.com/vector-of-bool/neo-sqlite3.git',
'ref': '0.2.2',
},
},
},
},
}
dds.scope.enter_context(
dds.set_contents(json_path,
json.dumps(import_data).encode()))

dds.catalog_import(json_path)

dds.catalog_get('neo-sqlite3@0.2.2')
assert (dds.source_root / 'neo-sqlite3@0.2.2').is_dir()
assert (dds.source_root / 'neo-sqlite3@0.2.2/package.dds').is_file()

+ 29
- 0
tests/catalog/import_test.py View File

@@ -0,0 +1,29 @@
import json

from tests import dds, DDS
from tests.fileutil import ensure_dir


def test_import_json(dds: DDS):
dds.scope.enter_context(ensure_dir(dds.build_dir))
dds.catalog_create()

json_fpath = dds.build_dir / 'data.json'
import_data = {
'version': 1,
'packages': {
'foo': {
'1.2.4': {
'git': {
'url': 'http://example.com',
'ref': 'master',
},
'depends': {},
},
},
},
}
dds.scope.enter_context(
dds.set_contents(json_fpath,
json.dumps(import_data).encode()))
dds.catalog_import(json_fpath)

+ 29
- 1
tests/dds.py View File

@@ -27,6 +27,10 @@ class DDS:
def repo_dir(self) -> Path:
return self.scratch_dir / 'repo'

@property
def catalog_path(self) -> Path:
return self.scratch_dir / 'catalog.db'

@property
def deps_build_dir(self) -> Path:
return self.scratch_dir / 'deps-build'
@@ -51,7 +55,7 @@ class DDS:
def run(self, cmd: proc.CommandLine, *,
cwd: Path = None) -> subprocess.CompletedProcess:
cmdline = list(proc.flatten_cmd(cmd))
res = self.run_unchecked(cmd)
res = self.run_unchecked(cmd, cwd=cwd)
if res.returncode != 0:
raise subprocess.CalledProcessError(
res.returncode, [self.dds_exe] + cmdline, res.stdout)
@@ -72,6 +76,7 @@ class DDS:
return self.run([
'deps',
'get',
f'--catalog={self.catalog_path}',
self.repo_dir_arg,
])

@@ -143,6 +148,29 @@ class DDS:
f'We don\'t know the executable suffix for the platform "{os.name}"'
)

def catalog_create(self) -> subprocess.CompletedProcess:
self.scratch_dir.mkdir(parents=True, exist_ok=True)
return self.run(
['catalog', 'create', f'--catalog={self.catalog_path}'],
cwd=self.test_dir)

def catalog_import(self, json_path: Path) -> subprocess.CompletedProcess:
self.scratch_dir.mkdir(parents=True, exist_ok=True)
return self.run([
'catalog',
'import',
f'--catalog={self.catalog_path}',
f'--json={json_path}',
])

def catalog_get(self, req: str) -> subprocess.CompletedProcess:
return self.run([
'catalog',
'get',
f'--catalog={self.catalog_path}',
req,
])

def set_contents(self, path: Union[str, Path],
content: bytes) -> ContextManager[Path]:
return fileutil.set_contents(self.source_root / path, content)

+ 2
- 0
tests/deps/do_test.py View File

@@ -16,6 +16,7 @@ def test_ls(dds: DDS):

@dds_conf
def test_deps_build(dds: DDS):
dds.catalog_import(dds.source_root / 'catalog.json')
assert not dds.repo_dir.exists()
dds.deps_get()
assert dds.repo_dir.exists(), '`deps get` did not generate a repo directory'
@@ -27,6 +28,7 @@ def test_deps_build(dds: DDS):

@dds_fixture_conf_1('use-remote')
def test_use_nlohmann_json_remote(dds: DDS):
dds.catalog_import(dds.source_root / 'catalog.json')
dds.deps_get()
dds.deps_build()
dds.build(apps=True)

+ 24
- 0
tests/deps/git-remote/catalog.json View File

@@ -0,0 +1,24 @@
{
"version": 1,
"packages": {
"neo-buffer": {
"0.1.0": {
"git": {
"url": "https://github.com/vector-of-bool/neo-buffer.git",
"ref": "develop"
},
"depends": {}
}
},
"range-v3": {
"0.9.1": {
"git": {
"url": "https://github.com/ericniebler/range-v3.git",
"ref": "0.9.1",
"auto-lib": "Niebler/range-v3"
},
"depends": {}
}
}
}
}

+ 0
- 2
tests/deps/git-remote/remote.dds View File

@@ -1,2 +0,0 @@
Remote-Package: neo-buffer@0.1.0; git url=https://github.com/vector-of-bool/neo-buffer.git ref=develop
Remote-Package: range-v3@0.9.1; git url=https://github.com/ericniebler/range-v3.git ref=0.9.1 auto=Niebler/range-v3

+ 4
- 0
tests/deps/no-deps/catalog.json View File

@@ -0,0 +1,4 @@
{
"version": 1,
"packages": {}
}

+ 0
- 0
tests/deps/no-deps/remote.dds View File


+ 14
- 0
tests/deps/use-remote/catalog.json View File

@@ -0,0 +1,14 @@
{
"version": 1,
"packages": {
"nlohmann-json": {
"3.7.1": {
"git": {
"url": "https://github.com/vector-of-bool/json.git",
"ref": "dds/3.7.1"
},
"depends": {}
}
}
}
}

+ 0
- 1
tests/deps/use-remote/remote.dds View File

@@ -1 +0,0 @@
Remote-Package: nlohmann-json@3.7.1; git url=https://github.com/vector-of-bool/json.git ref=dds/3.7.1

+ 15
- 0
tests/deps/use-spdlog/project/catalog.json View File

@@ -0,0 +1,15 @@
{
"version": 1,
"packages": {
"spdlog": {
"1.4.2": {
"git": {
"url": "https://github.com/gabime/spdlog.git",
"ref": "v1.4.2",
"auto-lib": "spdlog/spdlog"
},
"depends": {}
}
}
}
}

+ 0
- 1
tests/deps/use-spdlog/project/remote.dds View File

@@ -1 +0,0 @@
Remote-Package: spdlog@1.4.2; git url=https://github.com/gabime/spdlog.git ref=v1.4.2 auto=spdlog/spdlog

+ 1
- 0
tests/deps/use-spdlog/use_spdlog_test.py View File

@@ -4,6 +4,7 @@ from dds_ci import proc


def test_get_build_use_spdlog(dds: DDS):
dds.catalog_import(dds.source_root / 'catalog.json')
dds.deps_get()
tc_fname = 'gcc.tc.dds' if 'gcc' in dds.default_builtin_toolchain else 'msvc.tc.dds'
tc = str(dds.test_dir / tc_fname)

Write Preview
Loading…
Cancel
Save