Just-MR: Add distdir to WS root git map

3 files changed, 243 insertions, 0 deletions

diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
new file mode 100644
index 00000000..de73ee00
--- /dev/null
+++ b/src/other_tools/root_maps/TARGETS
@@ -0,0 +1,20 @@
+{ "distdir_git_map":
+  { "type": ["@", "rules", "CC", "library"]
+  , "name": ["distdir_git_map"]
+  , "hdrs": ["distdir_git_map.hpp"]
+  , "srcs": ["distdir_git_map.cpp"]
+  , "deps":
+    [ ["@", "json", "", "json"]
+    , ["src/other_tools/ops_maps", "import_to_git_map"]
+    ]
+  , "stage": ["src", "other_tools", "root_maps"]
+  , "private-deps":
+    [ ["src/other_tools/just_mr", "utils"]
+    , ["src/other_tools/ops_maps", "critical_git_op_map"]
+    , ["src/buildtool/execution_api/common", "common"]
+    , ["src/buildtool/execution_api/local", "config"]
+    , ["src/buildtool/execution_api/local", "local"]
+    , ["src/utils/cpp", "tmp_dir"]
+    ]
+  }
+}
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
new file mode 100644
index 00000000..e1d15c2d
--- /dev/null
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -0,0 +1,173 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/other_tools/root_maps/distdir_git_map.hpp"
+
+#include <algorithm>
+
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/execution_api/local/file_storage.hpp"
+#include "src/buildtool/execution_api/local/local_cas.hpp"
+#include "src/other_tools/just_mr/utils.hpp"
+#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
+
+namespace {
+
+/// \brief Create links from CAS content to distdir tmp directory
+[[nodiscard]] auto LinkToCAS(
+    std::shared_ptr<std::unordered_map<std::string, std::string>> const&
+        content_list,
+    std::filesystem::path const& tmp_dir) noexcept -> bool {
+    auto const& casf = LocalCAS<ObjectType::File>::Instance();
+    return std::all_of(content_list->begin(),
+                       content_list->end(),
+                       [&casf, tmp_dir](auto const& kv) {
+                           auto content_path = casf.BlobPath(
+                               ArtifactDigest(kv.second, 0, false));
+                           if (content_path) {
+                               return FileSystemManager::CreateFileHardlink(
+                                   *content_path,  // from: cas_path/content_id
+                                   tmp_dir / kv.first);  // to: tmp_dir/name
+                           }
+                           return false;
+                       });
+}
+
+}  // namespace
+
+auto CreateDistdirGitMap(
+    gsl::not_null<ImportToGitMap*> const& import_to_git_map,
+    gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+    std::size_t jobs) -> DistdirGitMap {
+    auto distdir_to_git = [import_to_git_map, critical_git_op_map](
+                              auto ts,
+                              auto setter,
+                              auto logger,
+                              auto /* unused */,
+                              auto const& key) {
+        auto distdir_tree_id_file =
+            JustMR::Utils::GetDistdirTreeIDFile(key.content_id);
+        if (FileSystemManager::Exists(distdir_tree_id_file)) {
+            // read distdir_tree_id from file tree_id_file
+            auto distdir_tree_id =
+                FileSystemManager::ReadFile(distdir_tree_id_file);
+            if (not distdir_tree_id) {
+                (*logger)(fmt::format("Failed to read tree id from file {}",
+                                      distdir_tree_id_file.string()),
+                          /*fatal=*/true);
+                return;
+            }
+            // ensure Git cache
+            // define Git operation to be done
+            GitOpKey op_key = {
+                {
+                    JustMR::Utils::GetGitCacheRoot(),  // target_path
+                    "",                                // git_hash
+                    "",                                // branch
+                    std::nullopt,                      // message
+                    true                               // init_bare
+                },
+                GitOpType::ENSURE_INIT};
+            critical_git_op_map->ConsumeAfterKeysReady(
+                ts,
+                {std::move(op_key)},
+                [distdir_tree_id = *distdir_tree_id, setter, logger](
+                    auto const& values) {
+                    GitOpValue op_result = *values[0];
+                    // check flag
+                    if (not op_result.result) {
+                        (*logger)("Git init failed",
+                                  /*fatal=*/true);
+                        return;
+                    }
+                    // subdir is ".", so no need to deal with the Git cache
+                    // set the workspace root
+                    (*setter)(nlohmann::json::array(
+                        {"git tree",
+                         distdir_tree_id,
+                         JustMR::Utils::GetGitCacheRoot().string()}));
+                },
+                [logger, target_path = JustMR::Utils::GetGitCacheRoot()](
+                    auto const& msg, bool fatal) {
+                    (*logger)(fmt::format("While running critical Git "
+                                          "op ENSURE_INIT for "
+                                          "target {}:\n{}",
+                                          target_path.string(),
+                                          msg),
+                              fatal);
+                });
+        }
+        else {
+            // create the links to CAS
+            auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("distdir");
+            if (not tmp_dir) {
+                (*logger)(fmt::format(
+                              "Failed to create tmp path for distdir target {}",
+                              key.content_id),
+                          /*fatal=*/true);
+                return;
+            }
+            // link content from CAS into tmp dir
+            if (not LinkToCAS(key.content_list, tmp_dir->GetPath())) {
+                (*logger)(fmt::format("Failed to create links to CAS content!",
+                                      key.content_id),
+                          /*fatal=*/true);
+                return;
+            }
+            // do import to git
+            CommitInfo c_info{tmp_dir->GetPath(), "distdir", key.content_id};
+            import_to_git_map->ConsumeAfterKeysReady(
+                ts,
+                {std::move(c_info)},
+                [tmp_dir,  // keep tmp_dir alive
+                 distdir_tree_id_file,
+                 setter,
+                 logger](auto const& values) {
+                    // check for errors
+                    if (not values[0]->second) {
+                        (*logger)("Importing to git failed",
+                                  /*fatal=*/true);
+                        return;
+                    }
+                    // only the tree is of interest
+                    std::string distdir_tree_id = values[0]->first;
+                    // write to tree id file
+                    if (not JustMR::Utils::WriteTreeIDFile(distdir_tree_id_file,
+                                                           distdir_tree_id)) {
+                        (*logger)(
+                            fmt::format("Failed to write tree id to file {}",
+                                        distdir_tree_id_file.string()),
+                            /*fatal=*/true);
+                        return;
+                    }
+                    // set the workspace root
+                    (*setter)(nlohmann::json::array(
+                        {"git tree",
+                         distdir_tree_id,
+                         JustMR::Utils::GetGitCacheRoot().string()}));
+                },
+                [logger, target_path = tmp_dir->GetPath()](auto const& msg,
+                                                           bool fatal) {
+                    (*logger)(
+                        fmt::format("While importing target {} to git:\n{}",
+                                    target_path.string(),
+                                    msg),
+                        fatal);
+                });
+        }
+    };
+    return AsyncMapConsumer<DistdirInfo, nlohmann::json>(distdir_to_git, jobs);
+}
\ No newline at end of file
diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp
new file mode 100644
index 00000000..91c26ad5
--- /dev/null
+++ b/src/other_tools/root_maps/distdir_git_map.hpp
@@ -0,0 +1,50 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP
+#define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP
+
+#include "nlohmann/json.hpp"
+#include "src/other_tools/ops_maps/import_to_git_map.hpp"
+
+struct DistdirInfo {
+    std::string content_id; /* key */
+    std::shared_ptr<std::unordered_map<std::string, std::string>> content_list;
+
+    [[nodiscard]] auto operator==(const DistdirInfo& other) const noexcept
+        -> bool {
+        return content_id == other.content_id;
+    }
+};
+
+/// \brief Maps a list of repositories belonging to a distdir to its
+/// corresponding workspace root.
+using DistdirGitMap = AsyncMapConsumer<DistdirInfo, nlohmann::json>;
+
+[[nodiscard]] auto CreateDistdirGitMap(
+    gsl::not_null<ImportToGitMap*> const& import_to_git_map,
+    gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+    std::size_t jobs) -> DistdirGitMap;
+
+namespace std {
+template <>
+struct hash<DistdirInfo> {
+    [[nodiscard]] auto operator()(const DistdirInfo& dd) const noexcept
+        -> std::size_t {
+        return std::hash<std::string>{}(dd.content_id);
+    }
+};
+}  // namespace std
+
+#endif  // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP
\ No newline at end of file