sources // | sources // | ||||
| filter(should_compile_source) // | | filter(should_compile_source) // | ||||
| transform([&](auto&& src) { | | transform([&](auto&& src) { | ||||
return compile_file_plan{rules, src, lib.name()}; | |||||
return compile_file_plan{rules, "obj/" + lib.name(), src, lib.name()}; | |||||
}) // | }) // | ||||
| to_vector; | | to_vector; | ||||
} | } | ||||
ranges::views::all(compilations) // | ranges::views::all(compilations) // | ||||
| transform([&](const compile_file_plan& comp) -> std::pair<fs::path, fs::path> { | | transform([&](const compile_file_plan& comp) -> std::pair<fs::path, fs::path> { | ||||
return std::pair(comp.source.path, | return std::pair(comp.source.path, | ||||
params.out_root / comp.get_object_file_path(params.toolchain)); | |||||
comp.get_object_file_path(build_env{params.toolchain, params.out_root})); | |||||
}) // | }) // | ||||
| ranges::to<object_file_index>() // | | ranges::to<object_file_index>() // | ||||
; | ; | ||||
} // namespace | } // namespace | ||||
void dds::build(const build_params& params, const package_manifest&) { | |||||
void dds::build(const build_params& params, const package_manifest& man) { | |||||
auto libs = collect_libraries(params.root, man.name); | |||||
// auto sroot = dds::sroot{params.root}; | // auto sroot = dds::sroot{params.root}; | ||||
// auto comp_rules = sroot.base_compile_rules(); | // auto comp_rules = sroot.base_compile_rules(); | ||||
auto compiles = collect_compiles(params, project); | auto compiles = collect_compiles(params, project); | ||||
dds::execute_all(compiles, params.toolchain, params.parallel_jobs, params.out_root); | |||||
dds::build_env env{params.toolchain, params.out_root}; | |||||
dds::execute_all(compiles, params.parallel_jobs, env); | |||||
using namespace ranges::views; | using namespace ranges::views; | ||||
using namespace dds; | using namespace dds; | ||||
void compile_file_plan::compile(const toolchain& tc, path_ref out_prefix) const { | |||||
const auto obj_path = out_prefix / get_object_file_path(tc); | |||||
void compile_file_plan::compile(const build_env& env) const { | |||||
const auto obj_path = get_object_file_path(env); | |||||
fs::create_directories(obj_path.parent_path()); | fs::create_directories(obj_path.parent_path()); | ||||
spdlog::info("[{}] Compile: {}", | spdlog::info("[{}] Compile: {}", | ||||
extend(spec.include_dirs, rules.include_dirs()); | extend(spec.include_dirs, rules.include_dirs()); | ||||
extend(spec.definitions, rules.defs()); | extend(spec.definitions, rules.defs()); | ||||
auto cmd = tc.create_compile_command(spec); | |||||
auto cmd = env.toolchain.create_compile_command(spec); | |||||
auto compile_res = run_proc(cmd); | auto compile_res = run_proc(cmd); | ||||
auto end_time = std::chrono::steady_clock::now(); | auto end_time = std::chrono::steady_clock::now(); | ||||
} | } | ||||
} | } | ||||
fs::path compile_file_plan::get_object_file_path(const toolchain& tc) const noexcept { | |||||
fs::path compile_file_plan::get_object_file_path(const build_env& env) const noexcept { | |||||
auto relpath = fs::relative(source.path, source.basis_path); | auto relpath = fs::relative(source.path, source.basis_path); | ||||
relpath.replace_filename(relpath.filename().string() + tc.object_suffix()); | |||||
return relpath; | |||||
auto ret = env.output_root / subdir / relpath; | |||||
ret.replace_filename(relpath.filename().string() + env.toolchain.object_suffix()); | |||||
return ret; | |||||
} | } | ||||
void dds::execute_all(const std::vector<compile_file_plan>& compilations, | void dds::execute_all(const std::vector<compile_file_plan>& compilations, | ||||
const toolchain& tc, | |||||
int n_jobs, | int n_jobs, | ||||
path_ref out_prefix) { | |||||
const build_env& env) { | |||||
// We don't bother with a nice thread pool, as the overhead of compiling | // We don't bother with a nice thread pool, as the overhead of compiling | ||||
// source files dwarfs the cost of interlocking. | // source files dwarfs the cost of interlocking. | ||||
std::mutex mut; | std::mutex mut; | ||||
auto& compilation = *comp_iter++; | auto& compilation = *comp_iter++; | ||||
lk.unlock(); | lk.unlock(); | ||||
try { | try { | ||||
compilation.compile(tc, out_prefix); | |||||
compilation.compile(env); | |||||
cancellation_point(); | cancellation_point(); | ||||
} catch (...) { | } catch (...) { | ||||
lk.lock(); | lk.lock(); |
namespace dds { | namespace dds { | ||||
struct build_env { | |||||
dds::toolchain toolchain; | |||||
fs::path output_root; | |||||
}; | |||||
struct compile_failure : std::runtime_error { | struct compile_failure : std::runtime_error { | ||||
using runtime_error::runtime_error; | using runtime_error::runtime_error; | ||||
}; | }; | ||||
struct compile_file_plan { | struct compile_file_plan { | ||||
shared_compile_file_rules rules; | shared_compile_file_rules rules; | ||||
fs::path subdir; | |||||
dds::source_file source; | dds::source_file source; | ||||
std::string qualifier; | std::string qualifier; | ||||
fs::path get_object_file_path(const toolchain& tc) const noexcept; | |||||
void compile(const toolchain& tc, path_ref out_prefix) const; | |||||
fs::path get_object_file_path(const build_env& env) const noexcept; | |||||
void compile(const build_env&) const; | |||||
}; | }; | ||||
void execute_all(const std::vector<compile_file_plan>&, | |||||
const toolchain& tc, | |||||
int n_jobs, | |||||
path_ref out_prefix); | |||||
void execute_all(const std::vector<compile_file_plan>&, int n_jobs, const build_env& env); | |||||
} // namespace dds | } // namespace dds |
cf_plan.source = sfile; | cf_plan.source = sfile; | ||||
cf_plan.qualifier = lib.name(); | cf_plan.qualifier = lib.name(); | ||||
cf_plan.rules = params.compile_rules; | cf_plan.rules = params.compile_rules; | ||||
cf_plan.subdir = fs::path("obj") / lib.name(); | |||||
compile_files.push_back(std::move(cf_plan)); | compile_files.push_back(std::move(cf_plan)); | ||||
if (sfile.kind == source_kind::test) { | if (sfile.kind == source_kind::test) { | ||||
test_sources.push_back(sfile); | test_sources.push_back(sfile); | ||||
create_archive.emplace(std::move(ar_plan)); | create_archive.emplace(std::move(ar_plan)); | ||||
} | } | ||||
return library_plan{params.out_subdir, compile_files, create_archive, link_executables}; | |||||
return library_plan{lib.name(), | |||||
lib.path(), | |||||
params.out_subdir, | |||||
compile_files, | |||||
create_archive, | |||||
link_executables}; | |||||
} | } | ||||
namespace { | namespace { | ||||
} // namespace | } // namespace | ||||
fs::path create_archive_plan::archive_file_path(const toolchain& tc) const noexcept { | |||||
fs::path fname = fmt::format("{}{}{}", "lib", name, tc.archive_suffix()); | |||||
return fname; | |||||
fs::path create_archive_plan::archive_file_path(const build_env& env) const noexcept { | |||||
return env.output_root / fmt::format("{}{}{}", "lib", name, env.toolchain.archive_suffix()); | |||||
} | } | ||||
void create_archive_plan::archive(const toolchain& tc, | |||||
path_ref out_prefix, | |||||
void create_archive_plan::archive(const build_env& env, | |||||
const std::vector<fs::path>& objects) const { | const std::vector<fs::path>& objects) const { | ||||
archive_spec ar; | archive_spec ar; | ||||
ar.input_files = objects; | |||||
ar.out_path = out_prefix / archive_file_path(tc); | |||||
auto ar_cmd = tc.create_archive_command(ar); | |||||
auto out_relpath = fs::relative(ar.out_path, out_prefix).string(); | |||||
ar.input_files = objects; | |||||
ar.out_path = archive_file_path(env); | |||||
auto ar_cmd = env.toolchain.create_archive_command(ar); | |||||
auto out_relpath = fs::relative(ar.out_path, env.output_root).string(); | |||||
spdlog::info("[{}] Archive: {}", name, out_relpath); | spdlog::info("[{}] Archive: {}", name, out_relpath); | ||||
auto start_time = std::chrono::steady_clock::now(); | auto start_time = std::chrono::steady_clock::now(); | ||||
} | } | ||||
} | } | ||||
void build_plan::compile_all(const toolchain& tc, int njobs, path_ref out_prefix) const { | |||||
std::vector<std::pair<fs::path, std::reference_wrapper<const compile_file_plan>>> comps; | |||||
for (const auto& lib : create_libraries) { | |||||
const auto lib_out_prefix = out_prefix / lib.out_subdir; | |||||
for (auto&& cf_plan : lib.compile_files) { | |||||
comps.emplace_back(lib_out_prefix, cf_plan); | |||||
} | |||||
} | |||||
namespace { | |||||
auto okay = parallel_run(comps, njobs, [&](const auto& pair) { | |||||
const auto& [out_dir, cf_plan] = pair; | |||||
cf_plan.get().compile(tc, out_dir); | |||||
}); | |||||
auto all_libraries(const build_plan& plan) { | |||||
return // | |||||
plan.build_packages // | |||||
| ranges::views::transform(&package_plan::create_libraries) // | |||||
| ranges::views::join // | |||||
; | |||||
} | |||||
} // namespace | |||||
void build_plan::compile_all(const build_env& env, int njobs) const { | |||||
auto all_compiles = // | |||||
all_libraries(*this) // | |||||
| ranges::views::transform(&library_plan::compile_files) // | |||||
| ranges::views::join // | |||||
; | |||||
auto okay = parallel_run(all_compiles, njobs, [&](const auto& cf) { cf.compile(env); }); | |||||
if (!okay) { | if (!okay) { | ||||
throw std::runtime_error("Compilation failed."); | throw std::runtime_error("Compilation failed."); | ||||
} | } | ||||
} | } | ||||
void build_plan::archive_all(const toolchain& tc, int njobs, path_ref out_prefix) const { | |||||
parallel_run(create_libraries, njobs, [&](const library_plan& lib) { | |||||
void build_plan::archive_all(const build_env& env, int njobs) const { | |||||
parallel_run(all_libraries(*this), njobs, [&](const library_plan& lib) { | |||||
if (!lib.create_archive) { | if (!lib.create_archive) { | ||||
return; | return; | ||||
} | } | ||||
| ranges::views::filter( | | ranges::views::filter( | ||||
[](auto&& comp) { return comp.source.kind == source_kind::source; }) // | [](auto&& comp) { return comp.source.kind == source_kind::source; }) // | ||||
| ranges::views::transform([&](auto&& comp) { | | ranges::views::transform([&](auto&& comp) { | ||||
return out_prefix / lib.out_subdir / comp.get_object_file_path(tc); | |||||
return comp.get_object_file_path(env); | |||||
}) // | }) // | ||||
| ranges::to_vector // | | ranges::to_vector // | ||||
; | ; | ||||
lib.create_archive->archive(tc, out_prefix, objects); | |||||
lib.create_archive->archive(env, objects); | |||||
}); | }); | ||||
} | } |
std::string name; | std::string name; | ||||
fs::path out_dir; | fs::path out_dir; | ||||
fs::path archive_file_path(const toolchain& tc) const noexcept; | |||||
fs::path archive_file_path(const build_env& env) const noexcept; | |||||
void | |||||
archive(const toolchain& tc, path_ref out_prefix, const std::vector<fs::path>& objects) const; | |||||
void archive(const build_env& env, const std::vector<fs::path>& objects) const; | |||||
}; | }; | ||||
struct create_exe_plan { | struct create_exe_plan { | ||||
}; | }; | ||||
struct library_plan { | struct library_plan { | ||||
std::string name; | |||||
fs::path source_root; | |||||
fs::path out_subdir; | fs::path out_subdir; | ||||
std::vector<compile_file_plan> compile_files; | std::vector<compile_file_plan> compile_files; | ||||
std::optional<create_archive_plan> create_archive; | std::optional<create_archive_plan> create_archive; | ||||
static library_plan create(const library& lib, const library_build_params& params); | static library_plan create(const library& lib, const library_build_params& params); | ||||
}; | }; | ||||
struct build_plan { | |||||
struct package_plan { | |||||
std::string name; | |||||
std::vector<std::string> pkg_requires; | |||||
std::vector<library_plan> create_libraries; | std::vector<library_plan> create_libraries; | ||||
// static build_plan generate(const build_params& params); | |||||
void add_library(const library& lib, const library_build_params& params) { | void add_library(const library& lib, const library_build_params& params) { | ||||
create_libraries.push_back(library_plan::create(lib, params)); | create_libraries.push_back(library_plan::create(lib, params)); | ||||
} | } | ||||
}; | |||||
struct build_plan { | |||||
std::vector<package_plan> build_packages; | |||||
void compile_all(const toolchain& tc, int njobs, path_ref out_prefix) const; | |||||
void archive_all(const toolchain& tc, int njobs, path_ref out_prefix) const; | |||||
void compile_all(const build_env& env, int njobs) const; | |||||
void archive_all(const build_env& env, int njobs) const; | |||||
}; | }; | ||||
} // namespace dds | } // namespace dds |
"Directory where build results will be stored", | "Directory where build results will be stored", | ||||
{"deps-build-dir"}, | {"deps-build-dir"}, | ||||
dds::fs::current_path() / "_build/deps"}; | dds::fs::current_path() / "_build/deps"}; | ||||
path_flag lmi_path{cmd, "lmi_path", "Destination for the INDEX.lmi file", {"lmi-path"}}; | |||||
path_flag lmi_path{cmd, | |||||
"lmi_path", | |||||
"Destination for the INDEX.lmi file", | |||||
{"lmi-path"}, | |||||
dds::fs::current_path() / "_build/INDEX.lmi"}; | |||||
args::Flag no_lmi{cmd, | args::Flag no_lmi{cmd, | ||||
"no_lmi", | "no_lmi", | ||||
"If specified, will not generate an INDEX.lmi", | "If specified, will not generate an INDEX.lmi", | ||||
auto plan = dds::create_deps_build_plan(deps); | auto plan = dds::create_deps_build_plan(deps); | ||||
auto tc = tc_filepath.get_toolchain(); | auto tc = tc_filepath.get_toolchain(); | ||||
auto bdir = build_dir.Get(); | auto bdir = build_dir.Get(); | ||||
plan.compile_all(tc, 6, bdir); | |||||
plan.archive_all(tc, 6, bdir); | |||||
dds::build_env env{std::move(tc), bdir}; | |||||
plan.compile_all(env, 6); | |||||
plan.archive_all(env, 6); | |||||
if (!no_lmi.Get()) { | |||||
write_libman_index(lmi_path.Get(), plan, env); | |||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
} build{*this}; | } build{*this}; |
#include <dds/repo/repo.hpp> | #include <dds/repo/repo.hpp> | ||||
#include <dds/sdist.hpp> | #include <dds/sdist.hpp> | ||||
#include <dds/util/string.hpp> | #include <dds/util/string.hpp> | ||||
#include <libman/index.hpp> | |||||
#include <libman/parse.hpp> | |||||
#include <range/v3/range/conversion.hpp> | #include <range/v3/range/conversion.hpp> | ||||
#include <range/v3/view/transform.hpp> | #include <range/v3/view/transform.hpp> | ||||
namespace { | namespace { | ||||
void add_dep_includes(shared_compile_file_rules& rules, | |||||
const package_manifest& man, | |||||
const sdist_index_type& sd_idx) { | |||||
void linkup_dependencies(shared_compile_file_rules& rules, | |||||
const package_manifest& man, | |||||
const sdist_index_type& sd_idx) { | |||||
for (const dependency& dep : man.dependencies) { | for (const dependency& dep : man.dependencies) { | ||||
auto found = sd_idx.find(dep.name); | auto found = sd_idx.find(dep.name); | ||||
if (found == sd_idx.end()) { | if (found == sd_idx.end()) { | ||||
dep.name, | dep.name, | ||||
man.name)); | man.name)); | ||||
} | } | ||||
add_dep_includes(rules, found->second.get().manifest, sd_idx); | |||||
linkup_dependencies(rules, found->second.get().manifest, sd_idx); | |||||
auto lib_src = found->second.get().path / "src"; | auto lib_src = found->second.get().path / "src"; | ||||
auto lib_include = found->second.get().path / "include"; | auto lib_include = found->second.get().path / "include"; | ||||
if (fs::exists(lib_include)) { | if (fs::exists(lib_include)) { | ||||
} | } | ||||
void add_sdist_to_dep_plan(build_plan& plan, const sdist& sd, const sdist_index_type& sd_idx) { | void add_sdist_to_dep_plan(build_plan& plan, const sdist& sd, const sdist_index_type& sd_idx) { | ||||
auto& pkg = plan.build_packages.emplace_back(); | |||||
pkg.name = sd.manifest.name; | |||||
auto libs = collect_libraries(sd.path, sd.manifest.name); | auto libs = collect_libraries(sd.path, sd.manifest.name); | ||||
for (const auto& lib : libs) { | for (const auto& lib : libs) { | ||||
shared_compile_file_rules comp_rules = lib.base_compile_rules(); | shared_compile_file_rules comp_rules = lib.base_compile_rules(); | ||||
add_dep_includes(comp_rules, sd.manifest, sd_idx); | |||||
linkup_dependencies(comp_rules, sd.manifest, sd_idx); | |||||
library_build_params params; | library_build_params params; | ||||
params.compile_rules = comp_rules; | params.compile_rules = comp_rules; | ||||
plan.add_library(lib, params); | |||||
pkg.add_library(lib, params); | |||||
} | } | ||||
} | } | ||||
} | } | ||||
return plan; | return plan; | ||||
} | } | ||||
namespace { | |||||
fs::path generate_lml(const library_plan& lib, path_ref libdir, const build_env& env) { | |||||
auto fname = lib.name + ".lml"; | |||||
auto lml_path = libdir / fname; | |||||
std::vector<lm::pair> kvs; | |||||
kvs.emplace_back("Type", "Library"); | |||||
kvs.emplace_back("Name", lib.name); | |||||
if (lib.create_archive) { | |||||
kvs.emplace_back("Path", | |||||
fs::relative(lib.create_archive->archive_file_path(env), | |||||
lml_path.parent_path()).string()); | |||||
} | |||||
auto pub_inc_dir = lib.source_root / "include"; | |||||
auto src_dir = lib.source_root / "src"; | |||||
if (fs::exists(src_dir)) { | |||||
pub_inc_dir = src_dir; | |||||
} | |||||
kvs.emplace_back("Include-Path", pub_inc_dir.string()); | |||||
// TODO: Uses, Preprocessor-Define, and Special-Uses | |||||
fs::create_directories(lml_path.parent_path()); | |||||
lm::write_pairs(lml_path, kvs); | |||||
return lml_path; | |||||
} | |||||
fs::path generate_lmp(const package_plan& pkg, path_ref basedir, const build_env& env) { | |||||
auto fname = pkg.name + ".lmp"; | |||||
auto lmp_path = basedir / fname; | |||||
std::vector<lm::pair> kvs; | |||||
kvs.emplace_back("Type", "Package"); | |||||
kvs.emplace_back("Name", pkg.name); | |||||
kvs.emplace_back("Namespace", pkg.name); | |||||
for (auto&& lib : pkg.create_libraries) { | |||||
auto lml = generate_lml(lib, basedir / pkg.name, env); | |||||
kvs.emplace_back("Library", fs::relative(lml, lmp_path.parent_path()).string()); | |||||
} | |||||
// TODO: `Requires` for transitive package imports | |||||
fs::create_directories(lmp_path.parent_path()); | |||||
lm::write_pairs(lmp_path, kvs); | |||||
return lmp_path; | |||||
} | |||||
} // namespace | |||||
void dds::write_libman_index(path_ref out_filepath, const build_plan& plan, const build_env& env) { | |||||
fs::create_directories(out_filepath.parent_path()); | |||||
auto lm_items_dir = out_filepath.parent_path() / "_libman"; | |||||
std::vector<lm::pair> kvs; | |||||
kvs.emplace_back("Type", "Index"); | |||||
for (const package_plan& pkg : plan.build_packages) { | |||||
auto pkg_lmp = generate_lmp(pkg, lm_items_dir, env); | |||||
kvs.emplace_back("Package", fmt::format( | |||||
"{}; {}", | |||||
pkg.name, | |||||
fs::relative(pkg_lmp, out_filepath.parent_path()).string() | |||||
)); | |||||
} | |||||
lm::write_pairs(out_filepath, kvs); | |||||
} |
build_plan create_deps_build_plan(const std::vector<sdist>& deps); | build_plan create_deps_build_plan(const std::vector<sdist>& deps); | ||||
void write_libman_index(path_ref where, const build_plan& plan, const build_env& env); | |||||
} // namespace dds | } // namespace dds |