visus
/
dds
보기 1
좋아요 0
포크 0
코드 이슈 0 풀 리퀘스트 0 릴리즈 0 위키 Activity
소스 검색

Initial version of dependency loading. Doesn't all work yet

default_compile_flags
vector-of-bool 5 년 전
부모
ebbda2040d
커밋
bba80a3348
9개의 변경된 파일과 469개의 추가작업 그리고 73개의 파일을 삭제
분할 보기 변경상태 보기
  1. +234
    -34
      src/dds/ddslim.main.cpp
  2. +65
    -0
      src/dds/deps.cpp
  3. +43
    -0
      src/dds/deps.hpp
  4. +25
    -4
      src/dds/package_manifest.cpp
  5. +8
    -4
      src/dds/package_manifest.hpp
  6. +59
    -3
      src/dds/repo/repo.cpp
  7. +8
    -1
      src/dds/repo/repo.hpp
  8. +11
    -11
      src/dds/sdist.cpp
  9. +16
    -16
      src/dds/sdist.hpp

+ 234
- 34
src/dds/ddslim.main.cpp 파일 보기

@@ -6,6 +6,9 @@
#include <dds/util/paths.hpp>
#include <dds/util/signal.hpp>

#include <range/v3/view/group_by.hpp>
#include <range/v3/view/transform.hpp>

#include <libman/parse.hpp>

#include <args.hxx>
@@ -18,35 +21,93 @@ namespace {
using string_flag = args::ValueFlag<std::string>;
using path_flag = args::ValueFlag<dds::fs::path>;

struct toolchain_flag : string_flag {
toolchain_flag(args::Group& grp)
: string_flag{grp,
"toolchain_file",
"Path/ident of the toolchain file to use",
{"toolchain", 'T'},
(dds::fs::current_path() / "toolchain.dds").string()} {}

dds::toolchain get_toolchain() {
const auto tc_path = this->Get();
if (tc_path.find(":") == 0) {
auto default_tc = tc_path.substr(1);
auto tc = dds::toolchain::get_builtin(default_tc);
if (!tc.has_value()) {
throw std::runtime_error(
fmt::format("Invalid default toolchain name '{}'", default_tc));
}
return std::move(*tc);
} else {
return dds::toolchain::load_from_file(tc_path);
}
}
};

struct repo_where_flag : path_flag {
repo_where_flag(args::Group& grp)
: path_flag{grp,
"dir",
"Directory in which to initialize the repository",
{"repo-dir"},
dds::repository::default_local_path()} {}
};

/**
* Base class holds the actual argument parser
*/
struct cli_base {
args::ArgumentParser& parser;
args::HelpFlag _help{parser, "help", "Display this help message and exit", {'h', "help"}};

// Test argument:
args::Flag _verify_ident{parser,
"test",
"Print `yes` and exit 0. Useful for scripting.",
{"are-you-the-real-dds?"}};

args::Group cmd_group{parser, "Available Commands"};
};

/**
* Flags common to all subcommands
*/
struct common_flags {
args::Command& cmd;

args::HelpFlag _help{cmd, "help", "Print this help message and exit", {'h', "help"}};
};

/**
* Flags common to project-related commands
*/
struct common_project_flags {
args::Command& cmd;

path_flag root{cmd,
"project_dir",
"Path to the directory containing the project",
{"project-dir"},
{'p', "project-dir"},
dds::fs::current_path()};
};

/*
######## ######## ######## #######
## ## ## ## ## ## ##
## ## ## ## ## ## ##
######## ###### ######## ## ##
## ## ## ## ## ##
## ## ## ## ## ##
## ## ######## ## #######
*/

struct cli_repo {
cli_base& base;
args::Command cmd{base.cmd_group, "repo", "Manage the package repository"};
common_flags _common{cmd};

path_flag where{cmd, "dir", "Directory in which to initialize the repository", {'d', "dir"}};
repo_where_flag where{cmd};

args::Group repo_group{cmd, "Repo subcommands"};

@@ -56,9 +117,33 @@ struct cli_repo {
common_flags _common{cmd};

int run() {
return dds::repository::with_repository(dds::repository::default_local_path(),
dds::repo_flags::none,
[&](auto) { return 0; });
auto list_contents = [&](dds::repository repo) {
auto same_name
= [](auto&& a, auto&& b) { return a.manifest.name == b.manifest.name; };

auto all_sdists = repo.load_sdists();
auto grp_by_name = all_sdists //
| ranges::views::group_by(same_name) //
| ranges::views::transform(ranges::to_vector) //
| ranges::views::transform([](auto&& grp) {
assert(grp.size() > 0);
return std::pair(grp[0].manifest.name, grp);
});

for (const auto& [name, grp] : grp_by_name) {
spdlog::info("{}:", name);
for (const dds::sdist& sd : grp) {
spdlog::info(" - {} [{}]",
sd.manifest.version.to_string(),
sd.md5_string());
}
}

return 0;
};
return dds::repository::with_repository(parent.where.Get(),
dds::repo_flags::read,
list_contents);
}
} ls{*this};

@@ -90,6 +175,16 @@ struct cli_repo {
}
};

/*
###### ######## #### ###### ########
## ## ## ## ## ## ## ##
## ## ## ## ## ##
###### ## ## ## ###### ##
## ## ## ## ## ##
## ## ## ## ## ## ## ##
###### ######## #### ###### ##
*/

struct cli_sdist {
cli_base& base;
args::Command cmd{base.cmd_group, "sdist", "Create a source distribution of a project"};
@@ -104,11 +199,12 @@ struct cli_sdist {
{"out"},
dds::fs::current_path() / "project.dsd"};

args::Flag force{cmd, "force", "Forcibly replace an existing result", {"force"}};
args::Flag export_{cmd,
args::Flag force{cmd, "force", "Forcibly replace an existing result", {"force"}};
args::Flag export_{cmd,
"export",
"Export the result into the local repository",
{'E', "export"}};
repo_where_flag repo_where{cmd};

int run() {
dds::sdist_params params;
@@ -118,7 +214,7 @@ struct cli_sdist {
auto sdist = dds::create_sdist(params);
if (export_.Get()) {
dds::repository::with_repository( //
dds::repository::default_local_path(),
repo_where.Get(),
dds::repo_flags::create_if_absent | dds::repo_flags::write_lock,
[&](dds::repository repo) { //
repo.add_sdist(sdist);
@@ -128,6 +224,16 @@ struct cli_sdist {
}
};

/*
######## ## ## #### ## ########
## ## ## ## ## ## ## ##
## ## ## ## ## ## ## ##
######## ## ## ## ## ## ##
## ## ## ## ## ## ## ##
## ## ## ## ## ## ## ##
######## ####### #### ######## ########
*/

struct cli_build {
cli_base& base;
args::Command cmd{base.cmd_group, "build", "Build a project"};
@@ -136,15 +242,10 @@ struct cli_build {

common_project_flags project{cmd};

string_flag tc_filepath{cmd,
"toolchain_file",
"Path to the toolchain file to use",
{"toolchain", 'T'},
(dds::fs::current_path() / "toolchain.dds").string()};

args::Flag build_tests{cmd, "build_tests", "Build the tests", {"tests", 't'}};
args::Flag build_apps{cmd, "build_apps", "Build applications", {"apps", 'A'}};
args::Flag export_{cmd, "export", "Generate a library export", {"export", 'E'}};
args::Flag build_tests{cmd, "build_tests", "Build the tests", {"tests", 't'}};
args::Flag build_apps{cmd, "build_apps", "Build applications", {"apps", 'A'}};
args::Flag export_{cmd, "export", "Generate a library export", {"export", 'E'}};
toolchain_flag tc_filepath{cmd};

path_flag lm_index{cmd,
"lm_index",
@@ -174,26 +275,11 @@ struct cli_build {
{"out"},
dds::fs::current_path() / "_build"};

dds::toolchain _get_toolchain() {
const auto tc_path = tc_filepath.Get();
if (tc_path.find(":") == 0) {
auto default_tc = tc_path.substr(1);
auto tc = dds::toolchain::get_builtin(default_tc);
if (!tc.has_value()) {
throw std::runtime_error(
fmt::format("Invalid default toolchain name '{}'", default_tc));
}
return std::move(*tc);
} else {
return dds::toolchain::load_from_file(tc_path);
}
}

int run() {
dds::build_params params;
params.root = project.root.Get();
params.out_root = out.Get();
params.toolchain = _get_toolchain();
params.toolchain = tc_filepath.get_toolchain();
params.do_export = export_.Get();
params.build_tests = build_tests.Get();
params.build_apps = build_apps.Get();
@@ -216,8 +302,116 @@ struct cli_build {
}
};

/*
######## ######## ######## ######
## ## ## ## ## ## ##
## ## ## ## ## ##
## ## ###### ######## ######
## ## ## ## ##
## ## ## ## ## ##
######## ######## ## ######
*/

struct cli_deps {
cli_base& base;
args::Command cmd{base.cmd_group, "deps", "Obtain/inspect/build deps for the project"};

common_flags _flags{cmd};
common_project_flags project{cmd};

args::Group deps_group{cmd, "Subcommands"};

dds::package_manifest load_package_manifest() {
return dds::package_manifest::load_from_file(project.root.Get() / "package.dds");
}

struct {
cli_deps& parent;
args::Command cmd{parent.deps_group, "ls", "List project dependencies"};
common_flags _common{cmd};

int run() {
const auto man = parent.load_package_manifest();
for (const auto& dep : man.dependencies) {
std::cout << dep.name << " " << dep.version.to_string() << '\n';
}
return 0;
}
} ls{*this};

struct {
cli_deps& parent;
args::Command cmd{parent.deps_group, "build", "Build project dependencies"};
common_flags _common{cmd};

path_flag build_dir{cmd,
"build_dir",
"Directory where build results will be stored",
{"deps-build-dir"},
dds::fs::current_path() / "_build/deps"};
path_flag lmi_path{cmd, "lmi_path", "Destination for the INDEX.lmi file", {"lmi-path"}};
args::Flag no_lmi{cmd,
"no_lmi",
"If specified, will not generate an INDEX.lmi",
{"skip-lmi"}};

repo_where_flag repo_where{cmd};

toolchain_flag tc_filepath{cmd};

void _build_one_dep(const dds::sdist& dep) {
spdlog::info("Build dependency {} {}",
dep.manifest.name,
dep.manifest.version.to_string());
dds::build_params params;
params.root = dep.path;
params.toolchain = tc_filepath.get_toolchain();
params.out_root = build_dir.Get()
/ fmt::format("{}-{}", dep.manifest.name, dep.manifest.version.to_string());

dds::build(params, dep.manifest);
}

int run() {
auto man = parent.load_package_manifest();
auto deps = dds::repository::with_repository( //
repo_where.Get(),
dds::repo_flags::read,
[&](dds::repository repo) {
return find_dependencies(repo,
man.dependencies.begin(),
man.dependencies.end());
});
for (auto&& dep : deps) {
_build_one_dep(dep);
}
return 0;
}
} build{*this};

int run() {
if (ls.cmd) {
return ls.run();
} else if (build.cmd) {
return build.run();
}
std::terminate();
return 0;
}
};

} // namespace

/*
## ## ### #### ## ##
### ### ## ## ## ### ##
#### #### ## ## ## #### ##
## ### ## ## ## ## ## ## ##
## ## ######### ## ## ####
## ## ## ## ## ## ###
## ## ## ## #### ## ##
*/

int main(int argc, char** argv) {
spdlog::set_pattern("[%H:%M:%S] [%^%l%$] %v");
args::ArgumentParser parser("DDSLiM - The drop-dead-simple library manager");
@@ -226,6 +420,7 @@ int main(int argc, char** argv) {
cli_build build{cli};
cli_sdist sdist{cli};
cli_repo repo{cli};
cli_deps deps{cli};
try {
parser.ParseCLI(argc, argv);
} catch (const args::Help&) {
@@ -240,12 +435,17 @@ int main(int argc, char** argv) {
dds::install_signal_handlers();

try {
if (build.cmd) {
if (cli._verify_ident) {
std::cout << "yes\n";
return 0;
} else if (build.cmd) {
return build.run();
} else if (sdist.cmd) {
return sdist.run();
} else if (repo.cmd) {
return repo.run();
} else if (deps.cmd) {
return deps.run();
} else {
assert(false);
std::terminate();

+ 65
- 0
src/dds/deps.cpp 파일 보기

@@ -0,0 +1,65 @@
#include "./deps.hpp"

#include <dds/sdist.hpp>
#include <dds/repo/repo.hpp>
#include <dds/util/string.hpp>

#include <range/v3/algorithm/partition_point.hpp>
#include <spdlog/fmt/fmt.h>

#include <cctype>

using namespace dds;

dependency dependency::parse_depends_string(std::string_view str) {
const auto str_begin = str.data();
auto str_iter = str_begin;
const auto str_end = str_iter + str.size();

while (str_iter != str_end && !std::isspace(*str_iter)) {
++str_iter;
}

auto name = trim(std::string_view(str_begin, str_iter - str_begin));
auto version_str = trim(std::string_view(str_iter, str_end - str_iter));

semver::version version;
try {
version = semver::version::parse(version_str);
} catch (const semver::invalid_version&) {
throw std::runtime_error(
fmt::format("Invalid version string '{}' in dependency declaration '{}' (Should be a "
"semver string. See https://semver.org/ for info)",
version_str,
str));
}
return dependency{std::string(name), version};
}

std::vector<sdist> dds::find_dependencies(const repository& repo, const dependency& dep) {
std::vector<sdist> acc;
detail::do_find_deps(repo, dep, acc);
return acc;
}

void detail::do_find_deps(const repository& repo, const dependency& dep, std::vector<sdist>& sd) {
auto sdist_opt = repo.get_sdist(dep.name, dep.version.to_string());
if (!sdist_opt) {
throw std::runtime_error(
fmt::format("Unable to find dependency to satisfy requirement: {} {}",
dep.name,
dep.version.to_string()));
}
sdist& new_sd = *sdist_opt;
auto insert_point = ranges::partition_point(sd, [&](const sdist& cand) {
return cand.path < new_sd.path;
});
if (insert_point != sd.end() && insert_point->manifest.name == new_sd.manifest.name) {
if (insert_point->manifest.version != new_sd.manifest.version) {
assert(false && "Version conflict resolution not implemented yet");
std::terminate();
}
return;
}
sd.insert(insert_point, std::move(new_sd));
}

+ 43
- 0
src/dds/deps.hpp 파일 보기

@@ -0,0 +1,43 @@
#pragma once

#include <semver/version.hpp>

#include <string_view>

namespace dds {

class repository;
struct sdist;

enum class version_strength {
exact,
patch,
minor,
major,
};

struct dependency {
std::string name;
semver::version version;

static dependency parse_depends_string(std::string_view str);
};

namespace detail {

void do_find_deps(const repository&, const dependency& dep, std::vector<sdist>& acc);

} // namespace detail

std::vector<sdist> find_dependencies(const repository& repo, const dependency& dep);

template <typename Iter, typename Snt>
inline std::vector<sdist> find_dependencies(const repository& repo, Iter it, Snt stop) {
std::vector<sdist> acc;
while (it != stop) {
detail::do_find_deps(repo, *it++, acc);
}
return acc;
}

} // namespace dds

+ 25
- 4
src/dds/package_manifest.cpp 파일 보기

@@ -1,18 +1,39 @@
#include "./package_manifest.hpp"

#include <dds/util/string.hpp>
#include <libman/parse.hpp>

#include <range/v3/view/split.hpp>
#include <range/v3/view/split_when.hpp>
#include <range/v3/view/transform.hpp>
#include <spdlog/fmt/fmt.h>

using namespace dds;

package_manifest package_manifest::load_from_file(const fs::path& fpath) {
auto kvs = lm::parse_file(fpath);
package_manifest ret;
auto kvs = lm::parse_file(fpath);
package_manifest ret;
std::string version_str;
std::vector<std::string> depends_strs;
lm::read(fmt::format("Reading package manifest '{}'", fpath.string()),
kvs,
lm::read_opt("Name", ret.name),
lm::read_opt("Version", ret.version),
lm::read_required("Name", ret.name),
lm::read_required("Version", version_str),
lm::read_accumulate("Depends", depends_strs),
lm::reject_unknown());

if (ret.name.empty()) {
throw std::runtime_error(fmt::format("'Name' field in [{}] may not be an empty string", fpath.string()));
}
if (version_str.empty()) {
throw std::runtime_error(fmt::format("'Version' field in [{}] may not be an empty string", fpath.string()));
}

ret.version = semver::version::parse(version_str);

ret.dependencies = depends_strs //
| ranges::views::transform(dependency::parse_depends_string) //
| ranges::to_vector;

return ret;
}

+ 8
- 4
src/dds/package_manifest.hpp 파일 보기

@@ -1,16 +1,20 @@
#pragma once

#include <dds/deps.hpp>
#include <dds/util/fs.hpp>
#include <semver/version.hpp>

#include <optional>
#include <string>
#include <vector>

namespace dds {

struct package_manifest {
std::string name;
std::string version = "no-version";
std::string name;
semver::version version;
std::vector<dependency> dependencies;
static package_manifest load_from_file(path_ref);
};

} // namespace dds
} // namespace dds

+ 59
- 3
src/dds/repo/repo.cpp 파일 보기

@@ -2,10 +2,15 @@

#include <dds/sdist.hpp>
#include <dds/util/paths.hpp>
#include <dds/util/string.hpp>

#include <spdlog/spdlog.h>

#include <range/v3/action/join.hpp>
#include <range/v3/range/conversion.hpp>
#include <range/v3/view/filter.hpp>
#include <range/v3/view/join.hpp>
#include <range/v3/view/transform.hpp>

using namespace dds;

@@ -29,10 +34,11 @@ repository repository::open_for_directory(path_ref dirpath) {
}

void repository::add_sdist(const sdist& sd) {
auto sd_dest = _root / "dist" / sd.name() / sd.version() / sd.hash();
auto sd_dest
= _root / "dist" / sd.manifest.name / sd.manifest.version.to_string() / sd.md5_string();
if (fs::exists(sd_dest)) {
spdlog::info("Source distribution '{}' is already available in the local repo",
sd.path().string());
sd.path.string());
return;
}
auto tmp_copy = sd_dest;
@@ -41,7 +47,57 @@ void repository::add_sdist(const sdist& sd) {
fs::remove_all(tmp_copy);
}
fs::create_directories(tmp_copy.parent_path());
fs::copy(sd.path(), tmp_copy, fs::copy_options::recursive);
fs::copy(sd.path, tmp_copy, fs::copy_options::recursive);
fs::rename(tmp_copy, sd_dest);
spdlog::info("Source distribution '{}' successfully exported", sd.ident());
}

std::vector<sdist> repository::load_sdists() const noexcept {
using namespace ranges;
using namespace ranges::views;
auto drop_dot_dirs
= filter([](path_ref p) { return !starts_with(p.filename().string(), "."); });

auto iter_children = [&](path_ref p) { return fs::directory_iterator(p) | drop_dot_dirs; };

auto try_read_sdist = [](path_ref p) -> std::optional<sdist> {
try {
return sdist::from_directory(p);
} catch (const std::runtime_error& e) {
spdlog::error("Failed to load source distribution from directory '{}': {}",
p.string(),
e.what());
return std::nullopt;
}
};

return
// Get the top-level `name` dirs
fs::directory_iterator(_dist_dir()) //
// Get the next level `version` dirs
| transform(iter_children) //
| views::join //
// Get the next level `ident` dirs
| transform(iter_children) //
| views::join //
// // Convert each dir into an `sdist` object
| transform(try_read_sdist) //
// // Drop items that failed to load
| filter([](auto&& opt) { return opt.has_value(); }) //
| transform([](auto&& opt) { return *opt; }) //
| to_vector //
;
}

std::optional<sdist> repository::get_sdist(std::string_view name, std::string_view version) const {
auto expect_path = _dist_dir() / name / version;
if (!fs::is_directory(expect_path)) {
return std::nullopt;
}

auto dir_iter = fs::directory_iterator(expect_path);
if (dir_iter == fs::directory_iterator()) {
return std::nullopt;
}
return sdist::from_directory(*dir_iter);
}

+ 8
- 1
src/dds/repo/repo.hpp 파일 보기

@@ -6,6 +6,7 @@
#include <cassert>
#include <functional>
#include <shared_mutex>
#include <vector>

namespace dds {

@@ -13,6 +14,7 @@ class sdist;

enum repo_flags {
none = 0b00,
read = none,
create_if_absent = 0b01,
write_lock = 0b10,
};
@@ -23,12 +25,15 @@ inline repo_flags operator|(repo_flags a, repo_flags b) {

class repository {
fs::path _root;

repository(path_ref p)
: _root(p) {}

static void _log_blocking(path_ref dir) noexcept;
static void _init_repo_dir(path_ref dir) noexcept;

fs::path _dist_dir() const noexcept { return _root / "dist"; }

public:
template <typename Func>
static decltype(auto) with_repository(path_ref dirpath, repo_flags flags, Func&& fn) {
@@ -61,7 +66,9 @@ public:

static fs::path default_local_path() noexcept;

void add_sdist(const sdist&);
void add_sdist(const sdist&);
std::optional<sdist> get_sdist(std::string_view name, std::string_view version) const;
std::vector<sdist> load_sdists() const noexcept;
};

} // namespace dds

+ 11
- 11
src/dds/sdist.cpp 파일 보기

@@ -97,7 +97,8 @@ sdist dds::create_sdist(const sdist_params& params) {
}

sdist dds::create_sdist_in_dir(path_ref out, const sdist_params& params) {
auto project = project::from_directory(params.project_dir);
auto project = project::from_directory(params.project_dir);

browns::md5 md5;

if (project.main_library()) {
@@ -109,17 +110,17 @@ sdist dds::create_sdist_in_dir(path_ref out, const sdist_params& params) {
}

auto man_path = project.root() / "package.dds";
if (fs::is_regular_file(man_path)) {
sdist_export_file(out, params.project_dir, man_path, md5);
if (!fs::is_regular_file(man_path)) {
throw std::runtime_error(fmt::format(
"Creating a source distribution requires a package.dds file for the project"));
}
sdist_export_file(out, params.project_dir, man_path, md5);

md5.pad();
auto hash_str = browns::format_digest(md5.digest());
spdlog::info("Generated export as {}-{}", project.manifest().name, hash_str);

std::vector<lm::pair> pairs;
pairs.emplace_back("Name", project.manifest().name);
pairs.emplace_back("Version", project.manifest().version);
pairs.emplace_back("MD5-Hash", hash_str);
lm::write_pairs(out / "_sdist.dds", pairs);

@@ -127,15 +128,14 @@ sdist dds::create_sdist_in_dir(path_ref out, const sdist_params& params) {
}

sdist sdist::from_directory(path_ref where) {
auto pkg_man = package_manifest::load_from_file(where / "package.dds");
auto meta_pairs = lm::parse_file(where / "_sdist.dds");
std::string name;
std::string version;
std::string hash_str;
lm::read("Loading source distribution",
lm::read(fmt::format("Loading source distribution manifest from {}/_sdist.dds", where.string()),
meta_pairs,
lm::read_required("Name", name),
lm::read_required("Version", version),
lm::read_required("MD5-Hash", hash_str),
lm::reject_unknown());
return sdist{name, version, hash_str, where};
return sdist{std::move(pkg_man),
browns::parse_digest<browns::md5::digest_type>(hash_str),
where};
}

+ 16
- 16
src/dds/sdist.hpp 파일 보기

@@ -1,7 +1,11 @@
#pragma once

#include <dds/package_manifest.hpp>
#include <dds/util/fs.hpp>

#include <browns/md5.hpp>
#include <browns/output.hpp>

namespace dds {

struct sdist_params {
@@ -12,27 +16,23 @@ struct sdist_params {
bool include_tests = false;
};

class sdist {
std::string _name;
std::string _version;
std::string _hash;
fs::path _sdist_dir;
struct sdist {
package_manifest manifest;
browns::md5::digest_type md5;
fs::path path;

public:
sdist(std::string_view name, std::string_view version, std::string_view hash, path_ref path)
: _name(name)
, _version(version)
, _hash(hash)
, _sdist_dir(path) {}
sdist(package_manifest man, browns::md5::digest_type hash, path_ref path)
: manifest(std::move(man))
, md5(hash)
, path(path) {}

static sdist from_directory(path_ref p);

std::string_view name() const noexcept { return _name; }
std::string_view version() const noexcept { return _version; }
std::string_view hash() const noexcept { return _hash; }
path_ref path() const noexcept { return _sdist_dir; }
std::string md5_string() const noexcept { return browns::format_digest(md5); }

std::string ident() const noexcept { return _name + "." + _version + "." + _hash; }
std::string ident() const noexcept {
return manifest.name + "." + manifest.version.to_string() + "." + md5_string();
}
};

sdist create_sdist(const sdist_params&);

Write Preview
Loading…
취소
저장