Browse Source

'pkg import' can import directories that are source distributions

default_compile_flags
vector-of-bool 3 years ago
parent
commit
2170331461
6 changed files with 62 additions and 17 deletions
  1. +1
    -1
      src/dds/cli/cmd/pkg_create.cpp
  2. +34
    -10
      src/dds/cli/cmd/pkg_import.cpp
  3. +16
    -3
      src/dds/pkg/cache.cpp
  4. +1
    -1
      src/dds/pkg/cache.hpp
  5. +1
    -1
      src/dds/pkg/get/get.cpp
  6. +9
    -1
      tests/test_pkg.py

+ 1
- 1
src/dds/cli/cmd/pkg_create.cpp View File

[&](std::error_code ec, e_human_message msg, boost::leaf::e_file_name file) { [&](std::error_code ec, e_human_message msg, boost::leaf::e_file_name file) {
dds_log(error, "Error: .bold.red[{}]"_styled, msg.value); dds_log(error, "Error: .bold.red[{}]"_styled, msg.value);
dds_log(error, dds_log(error,
"Failed to access file [.bold.red[{}]]: .br.yellow[{}]",
"Failed to access file [.bold.red[{}]]: .br.yellow[{}]"_styled,
file.value, file.value,
ec.message()); ec.message());
write_error_marker("failed-package-json5-scan"); write_error_marker("failed-package-json5-scan");

+ 34
- 10
src/dds/cli/cmd/pkg_import.cpp View File

#include <dds/util/result.hpp> #include <dds/util/result.hpp>


#include <boost/leaf/handle_exception.hpp> #include <boost/leaf/handle_exception.hpp>
#include <fansi/styled.hpp>
#include <json5/parse_data.hpp> #include <json5/parse_data.hpp>
#include <neo/assert.hpp> #include <neo/assert.hpp>
#include <neo/url/parse.hpp> #include <neo/url/parse.hpp>
#include <iostream> #include <iostream>
#include <string_view> #include <string_view>


using namespace fansi::literals;

namespace dds::cli::cmd { namespace dds::cli::cmd {

struct e_importing {
std::string value;
};

static int _pkg_import(const options& opts) { static int _pkg_import(const options& opts) {
return pkg_cache::with_cache( // return pkg_cache::with_cache( //
opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()), opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()),
pkg_cache_flags::write_lock | pkg_cache_flags::create_if_absent, pkg_cache_flags::write_lock | pkg_cache_flags::create_if_absent,
[&](auto repo) { [&](auto repo) {
for (std::string_view tgz_where : opts.pkg.import.items) {
neo_assertion_breadcrumbs("Importing sdist", tgz_where);
auto tmp_sd
= (tgz_where.starts_with("http://") || tgz_where.starts_with("https://"))
? download_expand_sdist_targz(tgz_where)
: expand_sdist_targz(tgz_where);
neo_assertion_breadcrumbs("Importing from temporary directory",
tmp_sd.tmpdir.path());
repo.add_sdist(tmp_sd.sdist, dds::if_exists(opts.if_exists));
// Lambda to import an sdist object
auto import_sdist
= [&](const sdist& sd) { repo.import_sdist(sd, dds::if_exists(opts.if_exists)); };

for (std::string_view sdist_where : opts.pkg.import.items) {
DDS_E_SCOPE(e_importing{std::string(sdist_where)});
neo_assertion_breadcrumbs("Importing sdist", sdist_where);
if (sdist_where.starts_with("http://") || sdist_where.starts_with("https://")) {
auto tmp_sd = download_expand_sdist_targz(sdist_where);
import_sdist(tmp_sd.sdist);
} else if (fs::is_directory(sdist_where)) {
auto sd = sdist::from_directory(sdist_where);
import_sdist(sd);
} else {
auto tmp_sd = expand_sdist_targz(sdist_where);
import_sdist(tmp_sd.sdist);
}
} }
if (opts.pkg.import.from_stdin) { if (opts.pkg.import.from_stdin) {
auto tmp_sd = dds::expand_sdist_from_istream(std::cin, "<stdin>"); auto tmp_sd = dds::expand_sdist_from_istream(std::cin, "<stdin>");
repo.add_sdist(tmp_sd.sdist, dds::if_exists(opts.if_exists));
repo.import_sdist(tmp_sd.sdist, dds::if_exists(opts.if_exists));
} }
return 0; return 0;
}); });
[](dds::e_sqlite3_error_exc e) { [](dds::e_sqlite3_error_exc e) {
dds_log(error, "Unexpected database error: {}", e.message); dds_log(error, "Unexpected database error: {}", e.message);
return 1; return 1;
},
[](e_system_error_exc err, e_importing what) {
dds_log(
error,
"Error while importing source distribution from [.bold.red[{}]]: .br.yellow[{}]"_styled,
what.value,
err.message);
return 1;
}); });
} }
} // namespace dds::cli::cmd } // namespace dds::cli::cmd

+ 16
- 3
src/dds/pkg/cache.cpp View File

return {writeable, dirpath, std::move(entries)}; return {writeable, dirpath, std::move(entries)};
} }


void pkg_cache::add_sdist(const sdist& sd, if_exists ife_action) {
void pkg_cache::import_sdist(const sdist& sd, if_exists ife_action) {
neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.id.to_string()); neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.id.to_string());
if (!_write_enabled) { if (!_write_enabled) {
dds_log(critical, dds_log(critical,
dds_log(info, msg + " - Replacing"); dds_log(info, msg + " - Replacing");
} }
} }

// Create a temporary location where we are creating it
auto tmp_copy = sd_dest; auto tmp_copy = sd_dest;
tmp_copy.replace_filename(".tmp-import"); tmp_copy.replace_filename(".tmp-import");
if (fs::exists(tmp_copy)) { if (fs::exists(tmp_copy)) {
fs::remove_all(tmp_copy); fs::remove_all(tmp_copy);
} }
fs::create_directories(tmp_copy.parent_path()); fs::create_directories(tmp_copy.parent_path());
fs::copy(sd.path, tmp_copy, fs::copy_options::recursive);

// Re-create an sdist from the given sdist. This will prune non-sdist files, rather than just
// fs::copy_all from the source, which may contain extras.
sdist_params params{
.project_dir = sd.path,
.dest_path = tmp_copy,
.include_apps = true,
.include_tests = true,
};
create_sdist_in_dir(tmp_copy, params);

// Swap out the temporary to the final location
if (fs::exists(sd_dest)) { if (fs::exists(sd_dest)) {
fs::remove_all(sd_dest); fs::remove_all(sd_dest);
} }
fs::rename(tmp_copy, sd_dest); fs::rename(tmp_copy, sd_dest);
_sdists.insert(sdist::from_directory(sd_dest)); _sdists.insert(sdist::from_directory(sd_dest));
dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.id.to_string());
dds_log(info, "Source distribution for '{}' successfully imported", sd.manifest.id.to_string());
} }


const sdist* pkg_cache::find(const pkg_id& pkg) const noexcept { const sdist* pkg_cache::find(const pkg_id& pkg) const noexcept {

+ 1
- 1
src/dds/pkg/cache.hpp View File



static fs::path default_local_path() noexcept; static fs::path default_local_path() noexcept;


void add_sdist(const sdist&, if_exists = if_exists::throw_exc);
void import_sdist(const sdist&, if_exists = if_exists::throw_exc);


const sdist* find(const pkg_id& pk) const noexcept; const sdist* find(const pkg_id& pk) const noexcept;



+ 1
- 1
src/dds/pkg/get/get.cpp View File

dds_log(info, "Download package: {}", inf.ident.to_string()); dds_log(info, "Download package: {}", inf.ident.to_string());
auto tsd = get_package_sdist(inf); auto tsd = get_package_sdist(inf);
std::scoped_lock lk{repo_mut}; std::scoped_lock lk{repo_mut};
repo.add_sdist(tsd.sdist, if_exists::throw_exc);
repo.import_sdist(tsd.sdist, if_exists::throw_exc);
}); });


if (!okay) { if (!okay) {

+ 9
- 1
tests/test_pkg.py View File



def test_import_sdist_stdin(_test_pkg: Tuple[Path, Project]) -> None: def test_import_sdist_stdin(_test_pkg: Tuple[Path, Project]) -> None:
sdist, project = _test_pkg sdist, project = _test_pkg
repo_content_path = project.dds.repo_dir / 'foo@1.2.3'
pipe = subprocess.Popen( pipe = subprocess.Popen(
list(proc.flatten_cmd([ list(proc.flatten_cmd([
project.dds.path, project.dds.path,


rc = pipe.wait() rc = pipe.wait()
assert rc == 0, 'Subprocess failed' assert rc == 0, 'Subprocess failed'
_check_import(project.dds.repo_dir / 'foo@1.2.3')


def test_import_sdist_dir(test_project: Project) -> None:
test_project.dds.run(['pkg', 'import', test_project.dds.repo_dir_arg, test_project.root])
_check_import(test_project.dds.repo_dir / 'foo@1.2.3')


def _check_import(repo_content_path: Path) -> None:
assert repo_content_path.is_dir(), \ assert repo_content_path.is_dir(), \
'The package did not appear in the local cache' 'The package did not appear in the local cache'
assert repo_content_path.joinpath('library.jsonc').is_file(), \ assert repo_content_path.joinpath('library.jsonc').is_file(), \

Loading…
Cancel
Save