'pkg import' can import directories that are source distributions

src/dds/cli/cmd/pkg_create.cpp

@@ -44,7 +44,7 @@ int pkg_create(const options& opts) {
         [&](std::error_code ec, e_human_message msg, boost::leaf::e_file_name file) {
             dds_log(error, "Error: .bold.red[{}]"_styled, msg.value);
             dds_log(error,
-                    "Failed to access file [.bold.red[{}]]: .br.yellow[{}]",
+                    "Failed to access file [.bold.red[{}]]: .br.yellow[{}]"_styled,
                     file.value,
                     ec.message());
             write_error_marker("failed-package-json5-scan");

src/dds/cli/cmd/pkg_import.cpp

@@ -5,6 +5,7 @@
 #include <dds/util/result.hpp>
 
 #include <boost/leaf/handle_exception.hpp>
+#include <fansi/styled.hpp>
 #include <json5/parse_data.hpp>
 #include <neo/assert.hpp>
 #include <neo/url/parse.hpp>
@@ -12,25 +13,40 @@
 #include <iostream>
 #include <string_view>
 
+using namespace fansi::literals;
+
 namespace dds::cli::cmd {
+
+struct e_importing {
+    std::string value;
+};
+
 static int _pkg_import(const options& opts) {
     return pkg_cache::with_cache(  //
         opts.pkg_cache_dir.value_or(pkg_cache::default_local_path()),
         pkg_cache_flags::write_lock | pkg_cache_flags::create_if_absent,
         [&](auto repo) {
-            for (std::string_view tgz_where : opts.pkg.import.items) {
-                neo_assertion_breadcrumbs("Importing sdist", tgz_where);
-                auto tmp_sd
-                    = (tgz_where.starts_with("http://") || tgz_where.starts_with("https://"))
-                    ? download_expand_sdist_targz(tgz_where)
-                    : expand_sdist_targz(tgz_where);
-                neo_assertion_breadcrumbs("Importing from temporary directory",
-                                          tmp_sd.tmpdir.path());
-                repo.add_sdist(tmp_sd.sdist, dds::if_exists(opts.if_exists));
+            // Lambda to import an sdist object
+            auto import_sdist
+                = [&](const sdist& sd) { repo.import_sdist(sd, dds::if_exists(opts.if_exists)); };
+
+            for (std::string_view sdist_where : opts.pkg.import.items) {
+                DDS_E_SCOPE(e_importing{std::string(sdist_where)});
+                neo_assertion_breadcrumbs("Importing sdist", sdist_where);
+                if (sdist_where.starts_with("http://") || sdist_where.starts_with("https://")) {
+                    auto tmp_sd = download_expand_sdist_targz(sdist_where);
+                    import_sdist(tmp_sd.sdist);
+                } else if (fs::is_directory(sdist_where)) {
+                    auto sd = sdist::from_directory(sdist_where);
+                    import_sdist(sd);
+                } else {
+                    auto tmp_sd = expand_sdist_targz(sdist_where);
+                    import_sdist(tmp_sd.sdist);
+                }
             }
             if (opts.pkg.import.from_stdin) {
                 auto tmp_sd = dds::expand_sdist_from_istream(std::cin, "<stdin>");
-                repo.add_sdist(tmp_sd.sdist, dds::if_exists(opts.if_exists));
+                repo.import_sdist(tmp_sd.sdist, dds::if_exists(opts.if_exists));
             }
             return 0;
         });
@@ -52,6 +68,14 @@ int pkg_import(const options& opts) {
         [](dds::e_sqlite3_error_exc e) {
             dds_log(error, "Unexpected database error: {}", e.message);
             return 1;
+        },
+        [](e_system_error_exc err, e_importing what) {
+            dds_log(
+                error,
+                "Error while importing source distribution from [.bold.red[{}]]: .br.yellow[{}]"_styled,
+                what.value,
+                err.message);
+            return 1;
         });
 }
 }  // namespace dds::cli::cmd

src/dds/pkg/cache.cpp

@@ -59,7 +59,7 @@ pkg_cache pkg_cache::_open_for_directory(bool writeable, path_ref dirpath) {
     return {writeable, dirpath, std::move(entries)};
 }
 
-void pkg_cache::add_sdist(const sdist& sd, if_exists ife_action) {
+void pkg_cache::import_sdist(const sdist& sd, if_exists ife_action) {
     neo_assertion_breadcrumbs("Importing sdist archive", sd.manifest.id.to_string());
     if (!_write_enabled) {
         dds_log(critical,
@@ -83,19 +83,32 @@ void pkg_cache::add_sdist(const sdist& sd, if_exists ife_action) {
             dds_log(info, msg + " - Replacing");
         }
     }
+
+    // Create a temporary location where we are creating it
     auto tmp_copy = sd_dest;
     tmp_copy.replace_filename(".tmp-import");
     if (fs::exists(tmp_copy)) {
         fs::remove_all(tmp_copy);
     }
     fs::create_directories(tmp_copy.parent_path());
-    fs::copy(sd.path, tmp_copy, fs::copy_options::recursive);
+
+    // Re-create an sdist from the given sdist. This will prune non-sdist files, rather than just
+    // fs::copy_all from the source, which may contain extras.
+    sdist_params params{
+        .project_dir   = sd.path,
+        .dest_path     = tmp_copy,
+        .include_apps  = true,
+        .include_tests = true,
+    };
+    create_sdist_in_dir(tmp_copy, params);
+
+    // Swap out the temporary to the final location
     if (fs::exists(sd_dest)) {
         fs::remove_all(sd_dest);
     }
     fs::rename(tmp_copy, sd_dest);
     _sdists.insert(sdist::from_directory(sd_dest));
-    dds_log(info, "Source distribution '{}' successfully exported", sd.manifest.id.to_string());
+    dds_log(info, "Source distribution for '{}' successfully imported", sd.manifest.id.to_string());
 }
 
 const sdist* pkg_cache::find(const pkg_id& pkg) const noexcept {

src/dds/pkg/cache.hpp

@@ -81,7 +81,7 @@ public:
 
     static fs::path default_local_path() noexcept;
 
-    void add_sdist(const sdist&, if_exists = if_exists::throw_exc);
+    void import_sdist(const sdist&, if_exists = if_exists::throw_exc);
 
     const sdist* find(const pkg_id& pk) const noexcept;
 

src/dds/pkg/get/get.cpp

@@ -62,7 +62,7 @@ void dds::get_all(const std::vector<pkg_id>& pkgs, pkg_cache& repo, const pkg_db
         dds_log(info, "Download package: {}", inf.ident.to_string());
         auto             tsd = get_package_sdist(inf);
         std::scoped_lock lk{repo_mut};
-        repo.add_sdist(tsd.sdist, if_exists::throw_exc);
+        repo.import_sdist(tsd.sdist, if_exists::throw_exc);
     });
 
     if (!okay) {

tests/test_pkg.py

@@ -49,7 +49,6 @@ def test_import_sdist_archive(_test_pkg: Tuple[Path, Project]) -> None:
 
 def test_import_sdist_stdin(_test_pkg: Tuple[Path, Project]) -> None:
     sdist, project = _test_pkg
-    repo_content_path = project.dds.repo_dir / 'foo@1.2.3'
     pipe = subprocess.Popen(
         list(proc.flatten_cmd([
             project.dds.path,
@@ -70,6 +69,15 @@ def test_import_sdist_stdin(_test_pkg: Tuple[Path, Project]) -> None:
 
     rc = pipe.wait()
     assert rc == 0, 'Subprocess failed'
+    _check_import(project.dds.repo_dir / 'foo@1.2.3')
+
+
+def test_import_sdist_dir(test_project: Project) -> None:
+    test_project.dds.run(['pkg', 'import', test_project.dds.repo_dir_arg, test_project.root])
+    _check_import(test_project.dds.repo_dir / 'foo@1.2.3')
+
+
+def _check_import(repo_content_path: Path) -> None:
     assert repo_content_path.is_dir(), \
         'The package did not appear in the local cache'
     assert repo_content_path.joinpath('library.jsonc').is_file(), \