just-mr maps: Add logic for resolving symlinks in file repositories

5 files changed, 144 insertions, 23 deletions

diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index 9bd70659..aac66487 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -20,6 +20,7 @@
     , ["src/other_tools/just_mr/progress_reporting", "statistics"]
     , ["src/other_tools/just_mr/progress_reporting", "progress_reporter"]
     , ["src/buildtool/storage", "storage"]
+    , ["src/other_tools/symlinks_map", "resolve_symlinks_map"]
     ]
   , "stage": ["src", "other_tools", "just_mr"]
   , "private-ldflags":
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index c15970d2..3c8e5d57 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -33,6 +33,7 @@
 #include "src/other_tools/ops_maps/git_update_map.hpp"
 #include "src/other_tools/ops_maps/repo_fetch_map.hpp"
 #include "src/other_tools/repo_map/repos_to_setup_map.hpp"
+#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp"
 
 namespace {
 
@@ -1144,6 +1145,7 @@ void DefaultReachableRepositories(
                              arguments.common.git_path->string(),
                              *arguments.common.local_launcher,
                              arguments.common.jobs);
+    auto resolve_symlinks_map = CreateResolveSymlinksMap();
 
     auto commit_git_map =
         CreateCommitGitMap(&critical_git_op_map,
@@ -1158,6 +1160,7 @@ void DefaultReachableRepositories(
     auto fpath_git_map = CreateFilePathGitMap(arguments.just_cmd.subcmd_name,
                                               &critical_git_op_map,
                                               &import_to_git_map,
+                                              &resolve_symlinks_map,
                                               arguments.common.jobs);
     auto distdir_git_map = CreateDistdirGitMap(&content_cas_map,
                                                &import_to_git_map,
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index 469caf49..8ece5565 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -54,6 +54,7 @@
     , ["src/utils/cpp", "path_hash"]
     , ["src/utils/cpp", "hash_combine"]
     , ["src/other_tools/just_mr", "utils"]
+    , ["src/other_tools/symlinks_map", "resolve_symlinks_map"]
     ]
   , "stage": ["src", "other_tools", "root_maps"]
   , "private-deps":
@@ -61,6 +62,8 @@
     , ["src/buildtool/storage", "config"]
     , ["src/utils/cpp", "tmp_dir"]
     , ["src/buildtool/file_system", "file_root"]
+    , ["src/buildtool/file_system", "git_repo"]
+    , ["src/other_tools/git_operations", "git_repo_remote"]
     ]
   }
 , "content_git_map":
diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp
index e26b3a9d..f0a12fd4 100644
--- a/src/other_tools/root_maps/fpath_git_map.cpp
+++ b/src/other_tools/root_maps/fpath_git_map.cpp
@@ -16,20 +16,121 @@
 
 #include "src/buildtool/execution_api/local/config.hpp"
 #include "src/buildtool/file_system/file_root.hpp"
+#include "src/buildtool/file_system/git_repo.hpp"
 #include "src/buildtool/storage/config.hpp"
+#include "src/other_tools/git_operations/git_repo_remote.hpp"
 #include "src/utils/cpp/tmp_dir.hpp"
 
+namespace {
+
+void ResolveFilePathTree(
+
+    std::string const& repo_root,
+    std::string const& target_path,
+    std::string const& tree_hash,
+    std::optional<PragmaSpecial> const& pragma_special,
+    gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
+    gsl::not_null<TaskSystem*> const& ts,
+    FilePathGitMap::SetterPtr const& ws_setter,
+    FilePathGitMap::LoggerPtr const& logger) {
+    if (pragma_special) {
+        // get the resolved tree
+        auto tree_id_file =
+            JustMR::Utils::GetResolvedTreeIDFile(tree_hash, *pragma_special);
+        if (FileSystemManager::Exists(tree_id_file)) {
+            // read resolved tree id
+            auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file);
+            if (not resolved_tree_id) {
+                (*logger)(fmt::format("Failed to read resolved "
+                                      "tree id from file {}",
+                                      tree_id_file.string()),
+                          /*fatal=*/true);
+                return;
+            }
+            // set the workspace root
+            (*ws_setter)(nlohmann::json::array(
+                {FileRoot::kGitTreeMarker, *resolved_tree_id, repo_root}));
+        }
+        else {
+            // resolve tree
+            resolve_symlinks_map->ConsumeAfterKeysReady(
+                ts,
+                {GitObjectToResolve(tree_hash,
+                                    ".",
+                                    *pragma_special,
+                                    /*known_info=*/std::nullopt)},
+                [resolve_symlinks_map,
+                 tree_hash,
+                 repo_root,
+                 tree_id_file,
+                 ws_setter,
+                 logger](auto const& hashes) {
+                    if (not hashes[0]) {
+                        // check for cycles
+                        auto error = DetectAndReportCycle(*resolve_symlinks_map,
+                                                          tree_hash);
+                        if (error) {
+                            (*logger)(fmt::format("Failed to resolve symlinks "
+                                                  "in tree {}:\n{}",
+                                                  tree_hash,
+                                                  *error),
+                                      /*fatal=*/true);
+                            return;
+                        }
+                        (*logger)(fmt::format("Unknown error in resolving "
+                                              "symlinks in tree {}",
+                                              tree_hash),
+                                  /*fatal=*/true);
+                        return;
+                    }
+                    auto const& resolved_tree = *hashes[0];
+                    // cache the resolved tree in the CAS map
+                    if (not JustMR::Utils::WriteTreeIDFile(tree_id_file,
+                                                           resolved_tree.id)) {
+                        (*logger)(fmt::format("Failed to write resolved tree "
+                                              "id to file {}",
+                                              tree_id_file.string()),
+                                  /*fatal=*/true);
+                        return;
+                    }
+                    // set the workspace root
+                    (*ws_setter)(
+                        nlohmann::json::array({FileRoot::kGitTreeMarker,
+                                               resolved_tree.id,
+                                               repo_root}));
+                },
+                [logger, target_path](auto const& msg, bool fatal) {
+                    (*logger)(fmt::format(
+                                  "While resolving symlinks for target {}:\n{}",
+                                  target_path,
+                                  msg),
+                              fatal);
+                });
+        }
+    }
+    else {
+        // set the workspace root as-is
+        (*ws_setter)(nlohmann::json::array(
+            {FileRoot::kGitTreeMarker, tree_hash, repo_root}));
+    }
+}
+
+}  // namespace
+
 auto CreateFilePathGitMap(
     std::optional<std::string> const& current_subcmd,
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
     gsl::not_null<ImportToGitMap*> const& import_to_git_map,
+    gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
     std::size_t jobs) -> FilePathGitMap {
-    auto dir_to_git = [current_subcmd, critical_git_op_map, import_to_git_map](
-                          auto ts,
-                          auto setter,
-                          auto logger,
-                          auto /*unused*/,
-                          auto const& key) {
+    auto dir_to_git = [current_subcmd,
+                       critical_git_op_map,
+                       import_to_git_map,
+                       resolve_symlinks_map](auto ts,
+                                             auto setter,
+                                             auto logger,
+                                             auto /*unused*/,
+                                             auto const& key) {
         // setup wrapped logger
         auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
             [logger](auto const& msg, bool fatal) {
@@ -75,6 +176,8 @@ auto CreateFilePathGitMap(
                  pragma_special = key.pragma_special,
                  git_cas = std::move(git_cas),
                  repo_root = std::move(*repo_root),
+                 resolve_symlinks_map,
+                 ts,
                  setter,
                  logger](auto const& values) {
                     GitOpValue op_result = *values[0];
@@ -102,19 +205,21 @@ auto CreateFilePathGitMap(
                                                 msg),
                                     fatal);
                             });
-                    // get tree id and return workspace root
+                    // get tree id
                     auto tree_hash = git_repo->GetSubtreeFromPath(
                         fpath, *op_result.result, wrapped_logger);
                     if (not tree_hash) {
                         return;
                     }
-                    // set the workspace root
-                    (*setter)(nlohmann::json::array(
-                        {pragma_special == PragmaSpecial::Ignore
-                             ? FileRoot::kGitTreeIgnoreSpecialMarker
-                             : FileRoot::kGitTreeMarker,
-                         *tree_hash,
-                         repo_root}));
+                    // resolve tree and set workspace root
+                    ResolveFilePathTree(repo_root.string(),
+                                        fpath.string(),
+                                        *tree_hash,
+                                        pragma_special,
+                                        resolve_symlinks_map,
+                                        ts,
+                                        setter,
+                                        logger);
                 },
                 [logger, target_path = *repo_root](auto const& msg,
                                                    bool fatal) {
@@ -159,8 +264,13 @@ auto CreateFilePathGitMap(
                 {std::move(c_info)},
                 // tmp_dir passed, to ensure folder is not removed until import
                 // to git is done
-                [tmp_dir, pragma_special = key.pragma_special, setter, logger](
-                    auto const& values) {
+                [tmp_dir,
+                 fpath = key.fpath,
+                 pragma_special = key.pragma_special,
+                 resolve_symlinks_map,
+                 ts,
+                 setter,
+                 logger](auto const& values) {
                     // check for errors
                     if (not values[0]->second) {
                         (*logger)("Importing to git failed",
@@ -169,13 +279,15 @@ auto CreateFilePathGitMap(
                     }
                     // we only need the tree
                     std::string tree = values[0]->first;
-                    // set the workspace root
-                    (*setter)(nlohmann::json::array(
-                        {pragma_special == PragmaSpecial::Ignore
-                             ? FileRoot::kGitTreeIgnoreSpecialMarker
-                             : FileRoot::kGitTreeMarker,
-                         tree,
-                         StorageConfig::GitRoot()}));
+                    // resolve tree and set workspace root
+                    ResolveFilePathTree(StorageConfig::GitRoot().string(),
+                                        fpath.string(),
+                                        tree,
+                                        pragma_special,
+                                        resolve_symlinks_map,
+                                        ts,
+                                        setter,
+                                        logger);
                 },
                 [logger, target_path = key.fpath](auto const& msg, bool fatal) {
                     (*logger)(
diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp
index 25721c19..39cae653 100644
--- a/src/other_tools/root_maps/fpath_git_map.hpp
+++ b/src/other_tools/root_maps/fpath_git_map.hpp
@@ -18,6 +18,7 @@
 #include "nlohmann/json.hpp"
 #include "src/other_tools/just_mr/utils.hpp"
 #include "src/other_tools/ops_maps/import_to_git_map.hpp"
+#include "src/other_tools/symlinks_map/resolve_symlinks_map.hpp"
 #include "src/utils/cpp/hash_combine.hpp"
 #include "src/utils/cpp/path_hash.hpp"
 
@@ -39,6 +40,7 @@ using FilePathGitMap = AsyncMapConsumer<FpathInfo, nlohmann::json>;
     std::optional<std::string> const& current_subcmd,
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
     gsl::not_null<ImportToGitMap*> const& import_to_git_map,
+    gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
     std::size_t jobs) -> FilePathGitMap;
 
 namespace std {