visus
/
dds
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

Remove old build code

default_compile_flags
vector-of-bool 5 years ago
parent
b99d957bc3
commit
93ce3abbdf
8 changed files with 28 additions and 339 deletions
Unified View Show Diff Stats
  1. +12
    -26
      src/dds.main.cpp
  2. +0
    -275
      src/dds/build.cpp
  3. +0
    -10
      src/dds/build.hpp
  4. +13
    -17
      src/dds/build/builder.cpp
  5. +0
    -6
      src/dds/build/params.hpp
  6. +1
    -1
      tests/basics/test_simple.py
  7. +1
    -1
      tests/basics/test_test_only.py
  8. +1
    -3
      tests/dds.py

+ 12
- 26
src/dds.main.cpp View File

#include <dds/build.hpp>
#include <dds/build/builder.hpp> #include <dds/build/builder.hpp>
#include <dds/catalog/catalog.hpp> #include <dds/catalog/catalog.hpp>
#include <dds/catalog/get.hpp> #include <dds/catalog/get.hpp>
args::Flag no_warnings{cmd, "no-warings", "Disable build warnings", {"no-warnings"}}; args::Flag no_warnings{cmd, "no-warings", "Disable build warnings", {"no-warnings"}};
toolchain_flag tc_filepath{cmd}; toolchain_flag tc_filepath{cmd};


args::Flag export_{cmd, "export", "Generate a library export", {"export", 'E'}};

path_flag path_flag
lm_index{cmd, lm_index{cmd,
"lm_index", "lm_index",


int run() { int run() {
dds::build_params params; dds::build_params params;
params.root = project.root.Get();
params.out_root = out.Get();
params.toolchain = tc_filepath.get_toolchain();
params.do_export = export_.Get();
params.build_tests = !no_tests.Get();
params.build_apps = !no_apps.Get();
params.enable_warnings = !no_warnings.Get();
params.parallel_jobs = num_jobs.Get();
params.out_root = out.Get();
params.toolchain = tc_filepath.get_toolchain();
params.parallel_jobs = num_jobs.Get();
dds::package_manifest man; dds::package_manifest man;
const auto man_filepath = params.root / "package.dds";
const auto man_filepath = project.root.Get() / "package.dds";
if (exists(man_filepath)) { if (exists(man_filepath)) {
man = dds::package_manifest::load_from_file(man_filepath); man = dds::package_manifest::load_from_file(man_filepath);
} }
} else { } else {
// Download and build dependencies // Download and build dependencies
// Build the dependencies // Build the dependencies
auto cat = cat_path.open();
params.dep_sdists = dds::repository::with_repository( //
auto cat = cat_path.open();
dds::repository::with_repository( //
this->repo_path.Get(), this->repo_path.Get(),
dds::repo_flags::write_lock | dds::repo_flags::create_if_absent, dds::repo_flags::write_lock | dds::repo_flags::create_if_absent,
[&](dds::repository repo) { [&](dds::repository repo) {
auto tsd = dds::get_package_sdist(*opt_pkg); auto tsd = dds::get_package_sdist(*opt_pkg);
repo.add_sdist(tsd.sdist, dds::if_exists::throw_exc); repo.add_sdist(tsd.sdist, dds::if_exists::throw_exc);
} }
auto sdist_ptr = repo.find(pk);
assert(sdist_ptr);
dds::sdist_build_params deps_params;
deps_params.subdir
= dds::fs::path("_deps") / sdist_ptr->manifest.pkg_id.to_string();
bd.add(*sdist_ptr, deps_params);
} }
return deps //
| ranges::views::transform([&](auto& id) {
auto ptr = repo.find(id);
assert(ptr);
return *ptr;
})
| ranges::to_vector;
}); });
for (auto sd : params.dep_sdists) {
dds::sdist_build_params deps_params;
deps_params.subdir = dds::fs::path("_deps") / sd.manifest.pkg_id.to_string();
bd.add(std::move(sd), deps_params);
}
} }
bd.build(params); bd.build(params);
return 0; return 0;

+ 0
- 275
src/dds/build.cpp View File

#include "./build.hpp"

#include <dds/build/builder.hpp>
#include <dds/build/plan/compile_exec.hpp>
#include <dds/catch2_embedded.hpp>
#include <dds/compdb.hpp>
#include <dds/usage_reqs.hpp>
#include <dds/util/algo.hpp>
#include <dds/util/time.hpp>
#include <libman/index.hpp>
#include <libman/parse.hpp>

#include <range/v3/algorithm/transform.hpp>
#include <range/v3/range/conversion.hpp>
#include <range/v3/view/transform.hpp>
#include <spdlog/spdlog.h>

#include <array>
#include <map>
#include <set>
#include <stdexcept>

using namespace dds;

namespace {

struct archive_failure : std::runtime_error {
using runtime_error::runtime_error;
};

void copy_headers(const fs::path& source, const fs::path& dest) {
for (fs::path file : fs::recursive_directory_iterator(source)) {
if (infer_source_kind(file) != source_kind::header) {
continue;
}
auto relpath = fs::relative(file, source);
auto dest_fpath = dest / relpath;
spdlog::info("Export header: {}", relpath.string());
fs::create_directories(dest_fpath.parent_path());
fs::copy_file(file, dest_fpath);
}
}

fs::path export_project_library(const library_plan& lib, build_env_ref env, path_ref export_root) {
auto lib_out_root = export_root / lib.name();
auto header_root = lib.source_root() / "include";
if (!fs::is_directory(header_root)) {
header_root = lib.source_root() / "src";
}

auto lml_path = export_root / fmt::format("{}.lml", lib.name());
auto lml_parent_dir = lml_path.parent_path();

std::vector<lm::pair> pairs;
pairs.emplace_back("Type", "Library");
pairs.emplace_back("Name", lib.name());

if (fs::is_directory(header_root)) {
auto header_dest = lib_out_root / "include";
copy_headers(header_root, header_dest);
pairs.emplace_back("Include-Path", fs::relative(header_dest, lml_parent_dir).string());
}

if (lib.create_archive()) {
auto ar_path
= env.output_root / lib.create_archive()->calc_archive_file_path(env.toolchain);
auto ar_dest = lib_out_root / ar_path.filename();
fs::create_directories(ar_dest.parent_path());
fs::copy_file(ar_path, ar_dest);
pairs.emplace_back("Path", fs::relative(ar_dest, lml_parent_dir).string());
}

for (const auto& use : lib.uses()) {
pairs.emplace_back("Uses", fmt::format("{}/{}", use.namespace_, use.name));
}
for (const auto& links : lib.links()) {
pairs.emplace_back("Links", fmt::format("{}/{}", links.namespace_, links.name));
}

lm::write_pairs(lml_path, pairs);
return lml_path;
}

void export_project(const package_plan& pkg, build_env_ref env) {
if (pkg.name().empty()) {
throw compile_failure(
fmt::format("Cannot generate an export when the package has no name (Provide a "
"package.dds with a `Name` field)"));
}
const auto export_root = env.output_root / fmt::format("{}.lpk", pkg.name());
spdlog::info("Generating project export: {}", export_root.string());
fs::remove_all(export_root);
fs::create_directories(export_root);

std::vector<lm::pair> pairs;

pairs.emplace_back("Type", "Package");
pairs.emplace_back("Name", pkg.name());
pairs.emplace_back("Namespace", pkg.namespace_());

for (const auto& lib : pkg.libraries()) {
export_project_library(lib, env, export_root);
}

lm::write_pairs(export_root / "package.lmp", pairs);
}

usage_requirement_map
load_usage_requirements(path_ref project_root, path_ref build_root, path_ref user_lm_index) {
fs::path lm_index_path = user_lm_index;
for (auto cand : {project_root / "INDEX.lmi", build_root / "INDEX.lmi"}) {
if (fs::exists(lm_index_path) || !user_lm_index.empty()) {
break;
}
lm_index_path = cand;
}
if (!fs::exists(lm_index_path)) {
spdlog::warn("No INDEX.lmi found, so we won't be able to load/use any dependencies");
return {};
}
lm::index idx = lm::index::from_file(lm_index_path);
return usage_requirement_map::from_lm_index(idx);
}

void add_ureqs(usage_requirement_map& ureqs,
const sdist& sd,
const library& lib,
const library_plan& lib_plan,
build_env_ref env) {
lm::library& reqs = ureqs.add(sd.manifest.namespace_, lib.manifest().name);
reqs.include_paths.push_back(lib.public_include_dir());
reqs.uses = lib.manifest().uses;
reqs.links = lib.manifest().links;
if (lib_plan.create_archive()) {
reqs.linkable_path
= env.output_root / lib_plan.create_archive()->calc_archive_file_path(env.toolchain);
}
// TODO: preprocessor definitions
}

using sdist_index_type = std::map<std::string, std::reference_wrapper<const sdist>>;
using sdist_names = std::set<std::string>;

void add_sdist_to_build(build_plan& plan,
const sdist& sd,
const sdist_index_type& sd_idx,
build_env_ref env,
usage_requirement_map& ureqs,
sdist_names& already_added) {
if (already_added.find(sd.manifest.pkg_id.name) != already_added.end()) {
// This one has already been added
return;
}
spdlog::debug("Adding dependent build: {}", sd.manifest.pkg_id.name);
// Ensure that ever dependency is loaded up first)
for (const auto& dep : sd.manifest.dependencies) {
auto other = sd_idx.find(dep.name);
assert(other != sd_idx.end()
&& "Failed to load a transitive dependency shortly after initializing them. What?");
add_sdist_to_build(plan, other->second, sd_idx, env, ureqs, already_added);
}
// Record that we have been processed
already_added.insert(sd.manifest.pkg_id.name);
// Finally, actually add the package:
auto& pkg = plan.add_package(package_plan(sd.manifest.pkg_id.name, sd.manifest.namespace_));
auto libs = collect_libraries(sd.path);
for (const auto& lib : libs) {
shared_compile_file_rules comp_rules = lib.base_compile_rules();
library_build_params lib_params;
lib_params.out_subdir = fs::path("deps") / sd.manifest.pkg_id.name;
auto lib_plan = library_plan::create(lib, lib_params);
// Create usage requirements for this libary.
add_ureqs(ureqs, sd, lib, lib_plan, env);
// Add it to the plan:
pkg.add_library(std::move(lib_plan));
}
}

void add_deps_to_build(build_plan& plan,
usage_requirement_map& ureqs,
const build_params& params,
build_env_ref env) {
auto sd_idx = params.dep_sdists //
| ranges::views::transform([](const auto& sd) {
return std::pair(sd.manifest.pkg_id.name, std::cref(sd));
}) //
| ranges::to<sdist_index_type>();

sdist_names already_added;
for (const sdist& sd : params.dep_sdists) {
add_sdist_to_build(plan, sd, sd_idx, env, ureqs, already_added);
}
}

} // namespace

void dds::build(const build_params& params, const package_manifest& man) {
fs::create_directories(params.out_root);
auto db = database::open(params.out_root / ".dds.db");

// The build plan we will fill out:
build_plan plan;

// Create a source distribution for the project, even if it doesn't have a manifest of its own
sdist root_sdist{man, params.root};

// Collect libraries for the current project
auto libs = collect_libraries(params.root);
if (!libs.size()) {
spdlog::warn("Nothing found to build!");
return;
}

usage_requirement_map ureqs;

dds::build_env env{params.toolchain, params.out_root, db, ureqs};

if (params.existing_lm_index) {
ureqs = load_usage_requirements(params.root, params.out_root, *params.existing_lm_index);
} else {
add_deps_to_build(plan, ureqs, params, env);
}

// Initialize the build plan for this project.
auto& pkg = plan.add_package(package_plan(man.pkg_id.name, man.namespace_));

// assert(false && "Not ready yet!");

library_build_params lib_params;
lib_params.build_tests = params.build_tests;
lib_params.build_apps = params.build_apps;
lib_params.enable_warnings = params.enable_warnings;

for (const library& lib : libs) {
lib_params.out_subdir = fs::relative(lib.path(), params.root);
pkg.add_library(library_plan::create(lib, lib_params));
}

if (params.generate_compdb) {
generate_compdb(plan, env);
}

dds::stopwatch sw;
plan.compile_all(env, params.parallel_jobs);
spdlog::info("Compilation completed in {:n}ms", sw.elapsed_ms().count());

sw.reset();
plan.archive_all(env, params.parallel_jobs);
spdlog::info("Archiving completed in {:n}ms", sw.elapsed_ms().count());

if (params.build_apps || params.build_tests) {
sw.reset();
plan.link_all(env, params.parallel_jobs);
spdlog::info("Runtime binary linking completed in {:n}ms", sw.elapsed_ms().count());
}

if (params.build_tests) {
sw.reset();
auto test_failures = plan.run_all_tests(env, params.parallel_jobs);
spdlog::info("Test execution finished in {:n}ms", sw.elapsed_ms().count());

for (auto& failures : test_failures) {
spdlog::error("Test {} failed! Output:\n{}[dds - test output end]",
failures.executable_path.string(),
failures.output);
}
if (!test_failures.empty()) {
throw compile_failure("Test failures during the build!");
}
}

if (params.do_export) {
export_project(pkg, env);
}
}

+ 0
- 10
src/dds/build.hpp View File

#pragma once

#include <dds/build/params.hpp>
#include <dds/package/manifest.hpp>

namespace dds {

void build(const build_params&, const package_manifest& man);

} // namespace dds

+ 13
- 17
src/dds/build/builder.cpp View File

plan.archive_all(env, params.parallel_jobs); plan.archive_all(env, params.parallel_jobs);
spdlog::info("Archiving completed in {:n}ms", sw.elapsed_ms().count()); spdlog::info("Archiving completed in {:n}ms", sw.elapsed_ms().count());


if (params.build_tests || params.build_apps) {
sw.reset();
plan.link_all(env, params.parallel_jobs);
spdlog::info("Runtime binary linking completed in {:n}ms", sw.elapsed_ms().count());
}
sw.reset();
plan.link_all(env, params.parallel_jobs);
spdlog::info("Runtime binary linking completed in {:n}ms", sw.elapsed_ms().count());


if (params.build_tests) {
sw.reset();
auto test_failures = plan.run_all_tests(env, params.parallel_jobs);
spdlog::info("Test execution finished in {:n}ms", sw.elapsed_ms().count());
sw.reset();
auto test_failures = plan.run_all_tests(env, params.parallel_jobs);
spdlog::info("Test execution finished in {:n}ms", sw.elapsed_ms().count());


for (auto& failures : test_failures) {
spdlog::error("Test {} failed! Output:\n{}[dds - test output end]",
failures.executable_path.string(),
failures.output);
}
if (!test_failures.empty()) {
throw compile_failure("Test failures during the build!");
}
for (auto& failures : test_failures) {
spdlog::error("Test {} failed! Output:\n{}[dds - test output end]",
failures.executable_path.string(),
failures.output);
}
if (!test_failures.empty()) {
throw compile_failure("Test failures during the build!");
} }
} }

+ 0
- 6
src/dds/build/params.hpp View File

namespace dds { namespace dds {


struct build_params { struct build_params {
fs::path root;
fs::path out_root; fs::path out_root;
std::optional<fs::path> existing_lm_index; std::optional<fs::path> existing_lm_index;
dds::toolchain toolchain; dds::toolchain toolchain;
std::vector<sdist> dep_sdists;
bool do_export = false;
bool build_tests = false;
bool enable_warnings = false;
bool build_apps = false;
bool generate_compdb = true; bool generate_compdb = true;
int parallel_jobs = 0; int parallel_jobs = 0;
}; };

+ 1
- 1
tests/basics/test_simple.py View File

''', ''',
)) ))


dds.build(tests=True, apps=False, warnings=False, export=True)
dds.build(tests=True, apps=False, warnings=False)
assert (dds.build_dir / 'compile_commands.json').is_file() assert (dds.build_dir / 'compile_commands.json').is_file()
assert list(dds.build_dir.glob('libTestLibrary*')) != [] assert list(dds.build_dir.glob('libTestLibrary*')) != []

+ 1
- 1
tests/basics/test_test_only.py View File

b'int main() {}', b'int main() {}',
)) ))


dds.build(tests=True, apps=False, warnings=False, export=False)
dds.build(tests=True, apps=False, warnings=False)
assert (dds.build_dir / f'test/foo{dds.exe_suffix}').is_file() assert (dds.build_dir / f'test/foo{dds.exe_suffix}').is_file()

+ 1
- 3
tests/dds.py View File

toolchain: str = None, toolchain: str = None,
apps: bool = True, apps: bool = True,
warnings: bool = True, warnings: bool = True,
tests: bool = True,
export: bool = False) -> subprocess.CompletedProcess:
tests: bool = True) -> subprocess.CompletedProcess:
return self.run([ return self.run([
'build', 'build',
f'--out={self.build_dir}', f'--out={self.build_dir}',
['--no-tests'] if not tests else [], ['--no-tests'] if not tests else [],
['--no-apps'] if not apps else [], ['--no-apps'] if not apps else [],
['--no-warnings'] if not warnings else [], ['--no-warnings'] if not warnings else [],
['--export'] if export else [],
self.project_dir_arg, self.project_dir_arg,
]) ])



Write Preview
Loading…
Cancel
Save