Add repo-1.dds.pizza as a pkg remote on initial db migration:

- Add a getenv() abstraction that is more versatile than the std:: one

src/dds.main.cpp

@@ -1,5 +1,6 @@
 #include <dds/cli/dispatch_main.hpp>
 #include <dds/cli/options.hpp>
+#include <dds/util/env.hpp>
 #include <dds/util/log.hpp>
 #include <dds/util/output.hpp>
 #include <dds/util/signal.hpp>
@@ -17,12 +18,12 @@
 #include <locale>
 
 static void load_locale() {
-    auto lang = std::getenv("LANG");
+    auto lang = dds::getenv("LANG");
     if (!lang) {
         return;
     }
     try {
-        std::locale::global(std::locale(lang));
+        std::locale::global(std::locale(*lang));
     } catch (const std::runtime_error& e) {
         // No locale with the given name
         return;

src/dds/pkg/db.cpp

@@ -4,6 +4,7 @@
 #include <dds/error/errors.hpp>
 #include <dds/error/nonesuch.hpp>
 #include <dds/solve/solve.hpp>
+#include <dds/util/env.hpp>
 #include <dds/util/log.hpp>
 #include <dds/util/paths.hpp>
 
@@ -24,6 +25,12 @@ using namespace dds;
 namespace nsql = neo::sqlite3;
 using namespace neo::sqlite3::literals;
 
+namespace dds {
+
+void add_init_repo(nsql::database_ref db) noexcept;
+
+}  // namespace dds
+
 namespace {
 
 void migrate_repodb_1(nsql::database& db) {
@@ -225,6 +232,13 @@ void ensure_migrated(nsql::database& db) {
     }
     meta["version"] = current_database_version;
     exec(db.prepare("UPDATE dds_cat_meta SET meta=?"), meta.dump());
+    tr.commit();
+
+    if (version < 3 && !getenv_bool("DDS_NO_ADD_INITIAL_REPO")) {
+        // Version 3 introduced remote repositories. If we're updating to 3, add that repo now
+        dds_log(info, "Downloading initial repository");
+        dds::add_init_repo(db);
+    }
 }
 
 }  // namespace

src/dds/pkg/remote.cpp

@@ -9,7 +9,9 @@
 #include <dds/util/log.hpp>
 #include <dds/util/result.hpp>
 
+#include <boost/leaf/handle_exception.hpp>
 #include <fansi/styled.hpp>
+#include <fmt/ostream.h>
 #include <neo/event.hpp>
 #include <neo/io/stream/buffers.hpp>
 #include <neo/io/stream/file.hpp>
@@ -224,18 +226,18 @@ void dds::update_all_remotes(nsql::database_ref db) {
 }
 
 void dds::remove_remote(pkg_db& pkdb, std::string_view name) {
-    auto&                           db = pkdb.database();
-    neo::sqlite3::transaction_guard tr{db};
+    auto&                   db = pkdb.database();
+    nsql::transaction_guard tr{db};
     auto get_rowid_st          = db.prepare("SELECT remote_id FROM dds_pkg_remotes WHERE name = ?");
     get_rowid_st.bindings()[1] = name;
-    auto row                   = neo::sqlite3::unpack_single_opt<std::int64_t>(get_rowid_st);
+    auto row                   = nsql::unpack_single_opt<std::int64_t>(get_rowid_st);
     if (!row) {
         BOOST_LEAF_THROW_EXCEPTION(  //
             make_user_error<errc::no_catalog_remote_info>("There is no remote with name '{}'",
                                                           name),
             [&] {
                 auto all_st = db.prepare("SELECT name FROM dds_pkg_remotes");
-                auto tups   = neo::sqlite3::iter_tuples<std::string>(all_st);
+                auto tups   = nsql::iter_tuples<std::string>(all_st);
                 auto names  = tups | ranges::views::transform([](auto&& tup) {
                                  auto&& [n] = tup;
                                  return n;
@@ -245,5 +247,63 @@ void dds::remove_remote(pkg_db& pkdb, std::string_view name) {
             });
     }
     auto [rowid] = *row;
-    neo::sqlite3::exec(db.prepare("DELETE FROM dds_pkg_remotes WHERE remote_id = ?"), rowid);
+    nsql::exec(db.prepare("DELETE FROM dds_pkg_remotes WHERE remote_id = ?"), rowid);
+}
+
+void dds::add_init_repo(nsql::database_ref db) noexcept {
+    std::string_view init_repo = "https://repo-1.dds.pizza";
+    // _Do not_ let errors stop us from continuing
+    bool okay = boost::leaf::try_catch(
+        [&]() -> bool {
+            try {
+                auto remote = pkg_remote::connect(init_repo);
+                remote.store(db);
+                update_all_remotes(db);
+                return true;
+            } catch (...) {
+                capture_exception();
+            }
+        },
+        [](http_status_error err, http_response_info resp, neo::url url) {
+            dds_log(error,
+                    "An HTTP error occurred while adding the initial repository [{}]: HTTP Status "
+                    "{} {}",
+                    err.what(),
+                    url.to_string(),
+                    resp.status,
+                    resp.status_message);
+            return false;
+        },
+        [](e_sqlite3_error_exc e, neo::url url) {
+            dds_log(error,
+                    "Error accessing remote database while adding initial repository: {}: {}",
+                    url.to_string(),
+                    e.message);
+            return false;
+        },
+        [](e_sqlite3_error_exc e) {
+            dds_log(error, "Unexpected database error: {}", e.message);
+            return false;
+        },
+        [](e_system_error_exc e, network_origin conn) {
+            dds_log(error,
+                    "Error communicating with [.br.red[{}://{}:{}]`]: {}"_styled,
+                    conn.protocol,
+                    conn.hostname,
+                    conn.port,
+                    e.message);
+            return false;
+        },
+        [](boost::leaf::diagnostic_info const& diag) -> int {
+            dds_log(critical, "Unhandled error while adding initial package repository: ", diag);
+            throw;
+        });
+    if (!okay) {
+        dds_log(warn, "We failed to add the initial package repository [{}]", init_repo);
+        dds_log(warn, "No remote packages will be available until the above issue is resolved.");
+        dds_log(
+            warn,
+            "The remote package repository can be added again with [.br.yellow[dds pkg repo add \"{}\"]]"_styled,
+            init_repo);
+    }
 }

src/dds/pkg/remote.hpp

@@ -34,4 +34,6 @@ public:
 void update_all_remotes(neo::sqlite3::database_ref);
 void remove_remote(pkg_db& db, std::string_view name);
 
+void add_init_repo(neo::sqlite3::database_ref db) noexcept;
+
 }  // namespace dds

src/dds/util/env.cpp

@@ -0,0 +1,18 @@
+#include "./env.hpp"
+
+#include <neo/utility.hpp>
+
+#include <cstdlib>
+
+std::optional<std::string> dds::getenv(const std::string& varname) noexcept {
+    auto cptr = std::getenv(varname.data());
+    if (cptr) {
+        return std::string(cptr);
+    }
+    return {};
+}
+
+bool dds::getenv_bool(const std::string& varname) noexcept {
+    auto s = getenv(varname);
+    return s == neo::oper::any_of("1", "true", "on", "TRUE", "ON", "YES", "yes");
+}

src/dds/util/env.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <neo/concepts.hpp>
+
+#include <optional>
+#include <string>
+
+namespace dds {
+
+std::optional<std::string> getenv(const std::string& env) noexcept;
+
+bool getenv_bool(const std::string& env) noexcept;
+
+template <neo::invocable Func>
+std::string getenv(const std::string& name, Func&& fn) noexcept(noexcept(fn())) {
+    auto val = getenv(name);
+    if (!val) {
+        return std::string(fn());
+    }
+    return *val;
+}
+
+}  // namespace dds

src/dds/util/paths.linux_fbsd.cpp

@@ -2,6 +2,7 @@
 
 #include "./paths.hpp"
 
+#include <dds/util/env.hpp>
 #include <dds/util/log.hpp>
 
 #include <cstdlib>
@@ -10,45 +11,34 @@ using namespace dds;
 
 fs::path dds::user_home_dir() {
     static auto ret = []() -> fs::path {
-        auto home_env = std::getenv("HOME");
-        if (!home_env) {
+        return fs::absolute(dds::getenv("HOME", [] {
             dds_log(error, "No HOME environment variable set!");
             return "/";
-        }
-        return fs::absolute(fs::path(home_env));
+        }));
     }();
     return ret;
 }
 
 fs::path dds::user_data_dir() {
     static auto ret = []() -> fs::path {
-        auto xdg_data_home = std::getenv("XDG_DATA_HOME");
-        if (xdg_data_home) {
-            return fs::absolute(fs::path(xdg_data_home));
-        }
-        return user_home_dir() / ".local/share";
+        return fs::absolute(
+            dds::getenv("XDG_DATA_HOME", [] { return user_home_dir() / ".local/share"; }));
     }();
     return ret;
 }
 
 fs::path dds::user_cache_dir() {
     static auto ret = []() -> fs::path {
-        auto xdg_cache_home = std::getenv("XDG_CACHE_HOME");
-        if (xdg_cache_home) {
-            return fs::absolute(fs::path(xdg_cache_home));
-        }
-        return user_home_dir() / ".cache";
+        return fs::absolute(
+            dds::getenv("XDG_CACHE_HOME", [] { return user_home_dir() / ".cache"; }));
     }();
     return ret;
 }
 
 fs::path dds::user_config_dir() {
     static auto ret = []() -> fs::path {
-        auto xdg_config_home = std::getenv("XDG_CONFIG_HOME");
-        if (xdg_config_home) {
-            return fs::absolute(fs::path(xdg_config_home));
-        }
-        return user_home_dir() / ".config";
+        return fs::absolute(
+            dds::getenv("XDG_CONFIG_HOME", [] { return user_home_dir() / ".config"; }));
     }();
     return ret;
 }

src/dds/util/paths.macos.cpp

@@ -2,6 +2,7 @@
 
 #include "./paths.hpp"
 
+#include <dds/util/env.hpp>
 #include <dds/util/log.hpp>
 
 #include <cstdlib>
@@ -10,12 +11,10 @@ using namespace dds;
 
 fs::path dds::user_home_dir() {
     static auto ret = []() -> fs::path {
-        auto home_env = std::getenv("HOME");
-        if (!home_env) {
-            dds_log(warn, "No HOME environment variable set!");
+        return fs::absolute(dds::getenv("HOME", [] {
+            dds_log(error, "No HOME environment variable set!");
             return "/";
-        }
-        return fs::absolute(fs::path(home_env));
+        }));
     }();
     return ret;
 }
@@ -24,4 +23,4 @@ fs::path dds::user_data_dir() { return user_home_dir() / "Library/Application Su
 fs::path dds::user_cache_dir() { return user_home_dir() / "Library/Caches"; }
 fs::path dds::user_config_dir() { return user_home_dir() / "Preferences"; }
 
-#endif
+#endif

src/dds/util/result.cpp

@@ -1,5 +1,6 @@
 #include "./result.hpp"
 
+#include <dds/util/env.hpp>
 #include <dds/util/log.hpp>
 
 #include <fmt/ostream.h>
@@ -23,9 +24,9 @@ void dds::capture_exception() {
 
 void dds::write_error_marker(std::string_view error) noexcept {
     dds_log(trace, "[error marker {}]", error);
-    auto efile_path = std::getenv("DDS_WRITE_ERROR_MARKER");
+    auto efile_path = dds::getenv("DDS_WRITE_ERROR_MARKER");
     if (efile_path) {
-        std::ofstream outfile{efile_path, std::ios::binary};
+        std::ofstream outfile{*efile_path, std::ios::binary};
         fmt::print(outfile, "{}", error);
     }
 }

tools/dds_ci/dds.py

@@ -1,5 +1,6 @@
 import multiprocessing
 import shutil
+import os
 from pathlib import Path
 import copy
 from typing import Optional, TypeVar, Iterable
@@ -61,7 +62,9 @@ class DDSWrapper:
 
     def run(self, args: proc.CommandLine, *, cwd: Optional[Pathish] = None) -> None:
         """Execute the 'dds' executable with the given arguments"""
-        proc.check_run([self.path, args], cwd=cwd or self.default_cwd)
+        env = os.environ.copy()
+        env['DDS_NO_ADD_INITIAL_REPO'] = '1'
+        proc.check_run([self.path, args], cwd=cwd or self.default_cwd, env=env)
 
     def catalog_json_import(self, path: Path) -> None:
         """Run 'catalog import' to import the given JSON. Only applicable to older 'dds'"""

tools/dds_ci/proc.py

@@ -1,5 +1,5 @@
 from pathlib import PurePath
-from typing import Iterable, Union, Optional, Iterator, NoReturn, Sequence
+from typing import Iterable, Union, Optional, Iterator, NoReturn, Sequence, Mapping
 from typing_extensions import Protocol
 import subprocess
 
@@ -40,9 +40,12 @@ def flatten_cmd(cmd: CommandLine) -> Iterable[str]:
         assert False, f'Invalid command line element: {repr(cmd)}'
 
 
-def run(*cmd: CommandLine, cwd: Optional[Pathish] = None, check: bool = False) -> ProcessResult:
+def run(*cmd: CommandLine,
+        cwd: Optional[Pathish] = None,
+        check: bool = False,
+        env: Optional[Mapping[str, str]] = None) -> ProcessResult:
     command = list(flatten_cmd(cmd))
-    res = subprocess.run(command, cwd=cwd, check=False)
+    res = subprocess.run(command, cwd=cwd, check=False, env=env)
     if res.returncode and check:
         raise_error(res)
     return res
@@ -52,5 +55,7 @@ def raise_error(proc: ProcessResult) -> NoReturn:
     raise subprocess.CalledProcessError(proc.returncode, proc.args, output=proc.stdout, stderr=proc.stderr)
 
 
-def check_run(*cmd: CommandLine, cwd: Optional[Pathish] = None) -> ProcessResult:
-    return run(cmd, cwd=cwd, check=True)
+def check_run(*cmd: CommandLine,
+              cwd: Optional[Pathish] = None,
+              env: Optional[Mapping[str, str]] = None) -> ProcessResult:
+    return run(cmd, cwd=cwd, check=True, env=env)