checkout locations: support extra environment variables to inherit

12 files changed, 105 insertions, 45 deletions

diff --git a/share/man/just-mr.1.md b/share/man/just-mr.1.md
index f2d4cc73..25658a96 100644
--- a/share/man/just-mr.1.md
+++ b/share/man/just-mr.1.md
@@ -90,6 +90,13 @@ This file contains a JSON object with several known keys:
    specifying known hostnames. When fetching from a non-local mirror, URLs
    with hostnames in the given list are preferred (in the order given)
    over URLs with other hostnames.
+ - the key *`"extra inherit env"`*, if given, is a list of strings
+   specifying additional variable names to be inherited from the
+   environment (besides the ones specified in *`"inherit env"`*
+   of the respective repository definition). This can be useful,
+   if the local git mirrors use a different protocol (like `ssh`
+   instead of `https`) and hence require different variables to
+   pass the credentials.
 
 This options overwrites any values set in the **`just-mrrc`**(5) file.  
 Default: file path *`".just-local.json"`* in user's home directory.
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index 415fca55..8ac53212 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -523,6 +523,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
         &import_to_git_map,
         common_args.git_path->string(),
         *common_args.local_launcher,
+        common_args.alternative_mirrors,
         serve ? &*serve : nullptr,
         &native_storage_config,
         compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index d65cbb50..ef91acc3 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -315,6 +315,9 @@ auto main(int argc, char* argv[]) -> int {
                 arguments.common.alternative_mirrors->preferred_hostnames =
                     checkout_locations_json.value("preferred hostnames",
                                                   nlohmann::json::array());
+                arguments.common.alternative_mirrors->extra_inherit_env =
+                    checkout_locations_json.value("extra inherit env",
+                                                  nlohmann::json::array());
             } catch (std::exception const& e) {
                 Logger::Log(
                     LogLevel::Error,
diff --git a/src/other_tools/just_mr/mirrors.hpp b/src/other_tools/just_mr/mirrors.hpp
index 47fc98be..7480a839 100644
--- a/src/other_tools/just_mr/mirrors.hpp
+++ b/src/other_tools/just_mr/mirrors.hpp
@@ -34,6 +34,7 @@
 struct Mirrors {
     nlohmann::json local_mirrors;        // maps URLs to list of local mirrors
     nlohmann::json preferred_hostnames;  // list of mirror hostnames
+    nlohmann::json extra_inherit_env;
 };
 
 using MirrorsPtr = std::shared_ptr<Mirrors>;
@@ -105,6 +106,36 @@ namespace MirrorsUtils {
     };
 }
 
+/// \brief Get the list of extra variables to inherit.
+[[nodiscard]] static inline auto GetInheritEnv(
+    MirrorsPtr const& additional_mirrors,
+    std::vector<std::string> const& base) noexcept -> std::vector<std::string> {
+    try {
+        std::vector<std::string> res(base);
+        res.reserve(additional_mirrors->extra_inherit_env.size() + base.size());
+        for (auto const& [_, val] :
+             additional_mirrors->extra_inherit_env.items()) {
+            if (val.is_string()) {
+                res.emplace_back(val.get<std::string>());
+            }
+            else {
+                Logger::Log(
+                    LogLevel::Warning,
+                    "Retrieving extra variables to inherit: found non-string "
+                    "list entry {}",
+                    val.dump());
+            }
+        }
+        return res;
+    } catch (std::exception const& ex) {
+        Logger::Log(LogLevel::Warning,
+                    "Retrieving extra environment variables to inherit failed "
+                    "with:\n{}",
+                    ex.what());
+        return {};
+    };
+}
+
 /// \brief Sort mirrors by the order of given hostnames.
 [[nodiscard]] static inline auto SortByHostname(
     std::vector<std::string> const& mirrors,
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 59007c9e..67a7a1b1 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -321,6 +321,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
         &import_to_git_map,
         common_args.git_path->string(),
         *common_args.local_launcher,
+        common_args.alternative_mirrors,
         serve ? &*serve : nullptr,
         &native_storage_config,
         compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
diff --git a/src/other_tools/just_mr/update.cpp b/src/other_tools/just_mr/update.cpp
index 93a84bcd..e2ed9197 100644
--- a/src/other_tools/just_mr/update.cpp
+++ b/src/other_tools/just_mr/update.cpp
@@ -236,6 +236,7 @@ auto MultiRepoUpdate(std::shared_ptr<Configuration> const& config,
     auto git_update_map = CreateGitUpdateMap(git_repo->GetGitCAS(),
                                              common_args.git_path->string(),
                                              *common_args.local_launcher,
+                                             common_args.alternative_mirrors,
                                              &native_storage_config,
                                              &stats,
                                              &progress,
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS
index 202f029b..587b8903 100644
--- a/src/other_tools/ops_maps/TARGETS
+++ b/src/other_tools/ops_maps/TARGETS
@@ -45,6 +45,7 @@
     , ["src/buildtool/file_system", "git_cas"]
     , ["src/buildtool/multithreading", "async_map_consumer"]
     , ["src/buildtool/storage", "config"]
+    , ["src/other_tools/just_mr", "mirrors"]
     , ["src/other_tools/just_mr/progress_reporting", "progress"]
     , ["src/other_tools/just_mr/progress_reporting", "statistics"]
     , ["src/utils/cpp", "hash_combine"]
@@ -127,6 +128,7 @@
     , ["src/buildtool/multithreading", "async_map_consumer"]
     , ["src/buildtool/serve_api/remote", "serve_api"]
     , ["src/buildtool/storage", "config"]
+    , ["src/other_tools/just_mr", "mirrors"]
     , ["src/other_tools/just_mr/progress_reporting", "progress"]
     , ["src/other_tools/ops_maps", "critical_git_op_map"]
     , ["src/other_tools/ops_maps", "import_to_git_map"]
diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.cpp b/src/other_tools/ops_maps/git_tree_fetch_map.cpp
index d4e7c0f6..d5de57d9 100644
--- a/src/other_tools/ops_maps/git_tree_fetch_map.cpp
+++ b/src/other_tools/ops_maps/git_tree_fetch_map.cpp
@@ -292,6 +292,7 @@ auto CreateGitTreeFetchMap(
     gsl::not_null<ImportToGitMap*> const& import_to_git_map,
     std::string const& git_bin,
     std::vector<std::string> const& launcher,
+    MirrorsPtr const& mirrors,
     ServeApi const* serve,
     gsl::not_null<StorageConfig const*> const& native_storage_config,
     StorageConfig const* compat_storage_config,
@@ -304,6 +305,7 @@ auto CreateGitTreeFetchMap(
                           import_to_git_map,
                           git_bin,
                           launcher,
+                          mirrors,
                           serve,
                           native_storage_config,
                           compat_storage_config,
@@ -334,6 +336,7 @@ auto CreateGitTreeFetchMap(
              import_to_git_map,
              git_bin,
              launcher,
+             mirrors,
              serve,
              native_storage_config,
              compat_storage_config,
@@ -524,8 +527,10 @@ auto CreateGitTreeFetchMap(
                 std::copy(key.command.begin(),
                           key.command.end(),
                           std::back_inserter(cmdline));
+                auto inherit_env =
+                    MirrorsUtils::GetInheritEnv(mirrors, key.inherit_env);
                 std::map<std::string, std::string> env{key.env_vars};
-                for (auto const& k : key.inherit_env) {
+                for (auto const& k : inherit_env) {
                     const char* v = std::getenv(k.c_str());
                     if (v != nullptr) {
                         env[k] = std::string(v);
@@ -571,6 +576,7 @@ auto CreateGitTreeFetchMap(
                      key,
                      git_bin,
                      launcher,
+                     mirrors,
                      native_storage_config,
                      compat_storage_config,
                      local_api,
diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.hpp b/src/other_tools/ops_maps/git_tree_fetch_map.hpp
index 7f698247..1e3f5156 100644
--- a/src/other_tools/ops_maps/git_tree_fetch_map.hpp
+++ b/src/other_tools/ops_maps/git_tree_fetch_map.hpp
@@ -27,6 +27,7 @@
 #include "src/buildtool/multithreading/async_map_consumer.hpp"
 #include "src/buildtool/serve_api/remote/serve_api.hpp"
 #include "src/buildtool/storage/config.hpp"
+#include "src/other_tools/just_mr/mirrors.hpp"
 #include "src/other_tools/just_mr/progress_reporting/progress.hpp"
 #include "src/other_tools/ops_maps/critical_git_op_map.hpp"
 #include "src/other_tools/ops_maps/import_to_git_map.hpp"
@@ -64,6 +65,7 @@ using GitTreeFetchMap = AsyncMapConsumer<GitTreeInfo, bool>;
     gsl::not_null<ImportToGitMap*> const& import_to_git_map,
     std::string const& git_bin,
     std::vector<std::string> const& launcher,
+    MirrorsPtr const& mirrors,
     ServeApi const* serve,
     gsl::not_null<StorageConfig const*> const& native_storage_config,
     StorageConfig const* compat_storage_config,
diff --git a/src/other_tools/ops_maps/git_update_map.cpp b/src/other_tools/ops_maps/git_update_map.cpp
index ddbec3bc..f22629f2 100644
--- a/src/other_tools/ops_maps/git_update_map.cpp
+++ b/src/other_tools/ops_maps/git_update_map.cpp
@@ -25,53 +25,54 @@ auto CreateGitUpdateMap(
     GitCASPtr const& git_cas,
     std::string const& git_bin,
     std::vector<std::string> const& launcher,
+    MirrorsPtr const& mirrors,
     gsl::not_null<StorageConfig const*> const& storage_config,
     gsl::not_null<JustMRStatistics*> const& stats,
     gsl::not_null<JustMRProgress*> const& progress,
     std::size_t jobs) -> GitUpdateMap {
-    auto update_commits = [git_cas,
-                           git_bin,
-                           launcher,
-                           storage_config,
-                           stats,
-                           progress](auto /* unused */,
-                                     auto setter,
-                                     auto logger,
-                                     auto /* unused */,
-                                     auto const& key) {
-        // perform git update commit
-        auto git_repo = GitRepoRemote::Open(git_cas);  // wrap the tmp odb
-        if (not git_repo) {
-            (*logger)(
-                fmt::format("Failed to open tmp Git repository for remote {}",
-                            key.repo),
-                /*fatal=*/true);
-            return;
-        }
-        // setup wrapped logger
-        auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
-            [logger](auto const& msg, bool fatal) {
-                (*logger)(
-                    fmt::format("While updating commit from remote:\n{}", msg),
-                    fatal);
-            });
-        // update commit
-        auto id = fmt::format("{}:{}", key.repo, key.branch);
-        progress->TaskTracker().Start(id);
-        auto new_commit = git_repo->UpdateCommitViaTmpRepo(*storage_config,
-                                                           key.repo,
-                                                           key.branch,
-                                                           key.inherit_env,
-                                                           git_bin,
-                                                           launcher,
-                                                           wrapped_logger);
-        progress->TaskTracker().Stop(id);
-        if (not new_commit) {
-            return;
-        }
-        stats->IncrementExecutedCounter();
-        (*setter)(new_commit->c_str());
-    };
+    auto update_commits =
+        [git_cas, git_bin, launcher, mirrors, storage_config, stats, progress](
+            auto /* unused */,
+            auto setter,
+            auto logger,
+            auto /* unused */,
+            auto const& key) {
+            // perform git update commit
+            auto git_repo = GitRepoRemote::Open(git_cas);  // wrap the tmp odb
+            if (not git_repo) {
+                (*logger)(fmt::format(
+                              "Failed to open tmp Git repository for remote {}",
+                              key.repo),
+                          /*fatal=*/true);
+                return;
+            }
+            // setup wrapped logger
+            auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+                [logger](auto const& msg, bool fatal) {
+                    (*logger)(
+                        fmt::format("While updating commit from remote:\n{}",
+                                    msg),
+                        fatal);
+                });
+            auto inherit_env =
+                MirrorsUtils::GetInheritEnv(mirrors, key.inherit_env);
+            // update commit
+            auto id = fmt::format("{}:{}", key.repo, key.branch);
+            progress->TaskTracker().Start(id);
+            auto new_commit = git_repo->UpdateCommitViaTmpRepo(*storage_config,
+                                                               key.repo,
+                                                               key.branch,
+                                                               inherit_env,
+                                                               git_bin,
+                                                               launcher,
+                                                               wrapped_logger);
+            progress->TaskTracker().Stop(id);
+            if (not new_commit) {
+                return;
+            }
+            stats->IncrementExecutedCounter();
+            (*setter)(new_commit->c_str());
+        };
     return AsyncMapConsumer<RepoDescriptionForUpdating, std::string>(
         update_commits, jobs);
 }
diff --git a/src/other_tools/ops_maps/git_update_map.hpp b/src/other_tools/ops_maps/git_update_map.hpp
index 1bf9256c..39ab61c8 100644
--- a/src/other_tools/ops_maps/git_update_map.hpp
+++ b/src/other_tools/ops_maps/git_update_map.hpp
@@ -24,6 +24,7 @@
 #include "src/buildtool/file_system/git_cas.hpp"
 #include "src/buildtool/multithreading/async_map_consumer.hpp"
 #include "src/buildtool/storage/config.hpp"
+#include "src/other_tools/just_mr/mirrors.hpp"
 #include "src/other_tools/just_mr/progress_reporting/progress.hpp"
 #include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
 #include "src/utils/cpp/hash_combine.hpp"
@@ -59,6 +60,7 @@ struct hash<RepoDescriptionForUpdating> {
     GitCASPtr const& git_cas,
     std::string const& git_bin,
     std::vector<std::string> const& launcher,
+    MirrorsPtr const& mirrors,
     gsl::not_null<StorageConfig const*> const&
         storage_config,  // native storage config
     gsl::not_null<JustMRStatistics*> const& stats,
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index 9acc5e1b..2b89c819 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -413,6 +413,9 @@ void NetworkFetchAndSetPresentRoot(
     all_mirrors.insert(
         all_mirrors.begin(), local_mirrors.begin(), local_mirrors.end());
 
+    auto inherit_env =
+        MirrorsUtils::GetInheritEnv(additional_mirrors, repo_info.inherit_env);
+
     for (auto mirror : all_mirrors) {
         auto mirror_path = GitURLIsPath(mirror);
         if (mirror_path) {
@@ -426,7 +429,7 @@ void NetworkFetchAndSetPresentRoot(
         if (git_repo->FetchViaTmpRepo(native_storage_config,
                                       mirror,
                                       repo_info.branch,
-                                      repo_info.inherit_env,
+                                      inherit_env,
                                       git_bin,
                                       launcher,
                                       wrapped_logger)) {