GitRepo: Create commit from a directory explicitly...

...by writing its tree directly in the object database instead of
working with the index. This allows the creation of trees that
contain also entries with 'magic' names, such as the .git folder
or .gitignore files.

Callers must ensure the given directory only contains the needed
entries. In particular, just-mr maps and serve service are updated
to separate the import-to-Git repository path from the temporary
path containing the content to be committed, to avoid polluting the
content path with entries generated on repository initialization.

2 files changed, 73 insertions, 44 deletions

diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.cpp b/src/other_tools/ops_maps/git_tree_fetch_map.cpp
index 7a4d9e0a..95582556 100644
--- a/src/other_tools/ops_maps/git_tree_fetch_map.cpp
+++ b/src/other_tools/ops_maps/git_tree_fetch_map.cpp
@@ -413,8 +413,9 @@ auto CreateGitTreeFetchMap(
                     return;
                 }
                 // create temporary location for command execution root
-                auto tmp_dir = storage_config->CreateTypedTmpDir("git-tree");
-                if (not tmp_dir) {
+                auto content_dir =
+                    storage_config->CreateTypedTmpDir("git-tree");
+                if (not content_dir) {
                     (*logger)(
                         "Failed to create execution root tmp directory for "
                         "tree id map!",
@@ -444,28 +445,39 @@ auto CreateGitTreeFetchMap(
                     }
                 }
                 auto const exit_code = system.Execute(
-                    cmdline, env, tmp_dir->GetPath(), out_dir->GetPath());
+                    cmdline, env, content_dir->GetPath(), out_dir->GetPath());
                 if (not exit_code) {
                     (*logger)(fmt::format("Failed to execute command:\n{}",
                                           nlohmann::json(cmdline).dump()),
                               /*fatal=*/true);
                     return;
                 }
+                // create temporary location for the import repository
+                auto repo_dir =
+                    storage_config->CreateTypedTmpDir("import-repo");
+                if (not repo_dir) {
+                    (*logger)(
+                        "Failed to create tmp directory for import repository",
+                        /*fatal=*/true);
+                    return;
+                }
                 // do an import to git with tree check
-                GitOpKey op_key = {.params =
-                                       {
-                                           tmp_dir->GetPath(),  // target_path
-                                           "",                  // git_hash
-                                           fmt::format("Content of tree {}",
-                                                       key.hash),  // message
-                                           tmp_dir->GetPath()  // source_path
-                                       },
-                                   .op_type = GitOpType::INITIAL_COMMIT};
+                GitOpKey op_key = {
+                    .params =
+                        {
+                            repo_dir->GetPath(),  // target_path
+                            "",                   // git_hash
+                            fmt::format("Content of tree {}",
+                                        key.hash),  // message
+                            content_dir->GetPath()  // source_path
+                        },
+                    .op_type = GitOpType::INITIAL_COMMIT};
                 critical_git_op_map->ConsumeAfterKeysReady(
                     ts,
                     {std::move(op_key)},
-                    [tmp_dir,  // keep tmp_dir alive
-                     out_dir,  // keep stdout/stderr of command alive
+                    [repo_dir,     // keep repo_dir alive
+                     content_dir,  // keep content_dir alive
+                     out_dir,      // keep stdout/stderr of command alive
                      critical_git_op_map,
                      just_git_cas = op_result.git_cas,
                      cmdline,
@@ -492,7 +504,7 @@ auto CreateGitTreeFetchMap(
                         if (not git_repo) {
                             (*logger)(
                                 fmt::format("Could not open repository {}",
-                                            tmp_dir->GetPath().string()),
+                                            repo_dir->GetPath().string()),
                                 /*fatal=*/true);
                             return;
                         }
@@ -541,14 +553,14 @@ auto CreateGitTreeFetchMap(
                                 /*fatal=*/true);
                             return;
                         }
-                        auto target_path = tmp_dir->GetPath();
+                        auto target_path = repo_dir->GetPath();
                         // fetch all into Git cache
                         auto just_git_repo = GitRepoRemote::Open(just_git_cas);
                         if (not just_git_repo) {
-                            (*logger)(fmt::format("Could not open Git "
-                                                  "repository {}",
-                                                  target_path.string()),
-                                      /*fatal=*/true);
+                            (*logger)(
+                                fmt::format("Could not open Git repository {}",
+                                            storage_config->GitRoot().string()),
+                                /*fatal=*/true);
                             return;
                         }
                         // define temp repo path
@@ -634,21 +646,28 @@ auto CreateGitTreeFetchMap(
                                 // success
                                 (*setter)(false /*no cache hit*/);
                             },
-                            [logger, commit = *op_result.result](
+                            [logger,
+                             commit = *op_result.result,
+                             target_path = storage_config->GitRoot()](
                                 auto const& msg, bool fatal) {
                                 (*logger)(
                                     fmt::format("While running critical Git op "
-                                                "KEEP_TAG for commit {}:\n{}",
+                                                "KEEP_TAG for commit {} in "
+                                                "repository {}:\n{}",
                                                 commit,
+                                                target_path.string(),
                                                 msg),
                                     fatal);
                             });
                     },
-                    [logger](auto const& msg, bool fatal) {
-                        (*logger)(fmt::format("While running critical Git op "
-                                              "INITIAL_COMMIT:\n{}",
-                                              msg),
-                                  fatal);
+                    [logger, target_path = repo_dir->GetPath()](auto const& msg,
+                                                                bool fatal) {
+                        (*logger)(
+                            fmt::format("While running critical Git op "
+                                        "INITIAL_COMMIT for target {}:\n{}",
+                                        target_path.string(),
+                                        msg),
+                            fatal);
                     });
             },
             [logger, target_path = storage_config->GitRoot()](auto const& msg,
diff --git a/src/other_tools/ops_maps/import_to_git_map.cpp b/src/other_tools/ops_maps/import_to_git_map.cpp
index e883e0dd..02da0475 100644
--- a/src/other_tools/ops_maps/import_to_git_map.cpp
+++ b/src/other_tools/ops_maps/import_to_git_map.cpp
@@ -25,7 +25,6 @@ namespace {
 void KeepCommitAndSetTree(
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
     std::string const& commit,
-    std::filesystem::path const& target_path,
     GitCASPtr const& just_git_cas,
     StorageConfig const& storage_config,
     gsl::not_null<TaskSystem*> const& ts,
@@ -42,7 +41,7 @@ void KeepCommitAndSetTree(
     critical_git_op_map->ConsumeAfterKeysReady(
         ts,
         {std::move(op_key)},
-        [commit, target_path, just_git_cas, setter, logger](
+        [commit, just_git_cas, storage_config, setter, logger](
             auto const& values) {
             GitOpValue op_result = *values[0];
             // check flag
@@ -54,7 +53,7 @@ void KeepCommitAndSetTree(
             auto just_git_repo = GitRepoRemote::Open(just_git_cas);
             if (not just_git_repo) {
                 (*logger)(fmt::format("Could not open Git repository {}",
-                                      target_path.string()),
+                                      storage_config.GitRoot().string()),
                           /*fatal=*/true);
                 return;
             }
@@ -75,7 +74,8 @@ void KeepCommitAndSetTree(
             (*setter)(std::pair<std::string, GitCASPtr>(*std::move(res),
                                                         just_git_cas));
         },
-        [logger, commit, target_path](auto const& msg, bool fatal) {
+        [logger, commit, target_path = storage_config.GitRoot()](
+            auto const& msg, bool fatal) {
             (*logger)(fmt::format("While running critical Git op KEEP_TAG for "
                                   "commit {} in target {}:\n{}",
                                   commit,
@@ -101,11 +101,21 @@ auto CreateImportToGitMap(
                                           auto logger,
                                           auto /*unused*/,
                                           auto const& key) {
-        // Perform initial commit at import location: init + add . + commit
+        // The repository path that imports the content must be separate from
+        // the content path, to avoid polluting the entries
+        auto repo_dir = storage_config->CreateTypedTmpDir("import-repo");
+        if (not repo_dir) {
+            (*logger)(fmt::format("Failed to create import repository tmp "
+                                  "directory for target {}",
+                                  key.target_path.string()),
+                      true);
+            return;
+        }
+        // Commit content from target_path via the tmp repository
         GitOpKey op_key = {.params =
                                {
-                                   key.target_path,  // target_path
-                                   "",               // git_hash
+                                   repo_dir->GetPath(),  // target_path
+                                   "",                   // git_hash
                                    fmt::format("Content of {} {}",
                                                key.repo_type,
                                                key.content),  // message
@@ -115,8 +125,8 @@ auto CreateImportToGitMap(
         critical_git_op_map->ConsumeAfterKeysReady(
             ts,
             {std::move(op_key)},
-            [critical_git_op_map,
-             target_path = key.target_path,
+            [repo_dir,  // keep repo_dir alive
+             critical_git_op_map,
              git_bin,
              launcher,
              storage_config,
@@ -145,9 +155,9 @@ auto CreateImportToGitMap(
                 critical_git_op_map->ConsumeAfterKeysReady(
                     ts,
                     {std::move(op_key)},
-                    [critical_git_op_map,
+                    [repo_dir,  // keep repo_dir alive
+                     critical_git_op_map,
                      commit = *op_result.result,
-                     target_path,
                      git_bin,
                      launcher,
                      storage_config,
@@ -172,6 +182,7 @@ auto CreateImportToGitMap(
                                 /*fatal=*/true);
                             return;
                         }
+                        auto const& target_path = repo_dir->GetPath();
                         auto wrapped_logger =
                             std::make_shared<AsyncMapConsumerLogger>(
                                 [logger, target_path](auto const& msg,
@@ -198,16 +209,15 @@ auto CreateImportToGitMap(
                                 [logger, target_path](auto const& msg,
                                                       bool fatal) {
                                     (*logger)(
-                                        fmt::format(
-                                            "While doing keep commit "
-                                            "and setting Git tree {}:\n{}",
-                                            target_path.string(),
-                                            msg),
+                                        fmt::format("While doing keep commit "
+                                                    "and setting Git tree for "
+                                                    "target {}:\n{}",
+                                                    target_path.string(),
+                                                    msg),
                                         fatal);
                                 });
                         KeepCommitAndSetTree(critical_git_op_map,
                                              commit,
-                                             target_path,
                                              op_result.git_cas, /*just_git_cas*/
                                              *storage_config,
                                              ts,