summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/other_tools/ops_maps/TARGETS18
-rw-r--r--src/other_tools/ops_maps/import_to_git_map.cpp222
-rw-r--r--src/other_tools/ops_maps/import_to_git_map.hpp74
3 files changed, 314 insertions, 0 deletions
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS
index 8bde49eb..8c045e14 100644
--- a/src/other_tools/ops_maps/TARGETS
+++ b/src/other_tools/ops_maps/TARGETS
@@ -10,4 +10,22 @@
]
, "stage": ["src", "other_tools", "ops_maps"]
}
+, "import_to_git_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["import_to_git_map"]
+ , "hdrs": ["import_to_git_map.hpp"]
+ , "srcs": ["import_to_git_map.cpp"]
+ , "deps":
+ [ "critical_git_op_map"
+ , ["src/buildtool/file_system", "git_repo"]
+ , ["@", "fmt", "", "fmt"]
+ , ["src/utils/cpp", "path"]
+ ]
+ , "stage": ["src", "other_tools", "ops_maps"]
+ , "private-deps":
+ [ ["src/other_tools/just_mr", "utils"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/execution_api/local", "config"]
+ ]
+ }
}
diff --git a/src/other_tools/ops_maps/import_to_git_map.cpp b/src/other_tools/ops_maps/import_to_git_map.cpp
new file mode 100644
index 00000000..45e7265c
--- /dev/null
+++ b/src/other_tools/ops_maps/import_to_git_map.cpp
@@ -0,0 +1,222 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/other_tools/ops_maps/import_to_git_map.hpp"
+
+#include "fmt/core.h"
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/other_tools/just_mr/utils.hpp"
+
+auto CreateImportToGitMap(
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ std::size_t jobs) -> ImportToGitMap {
+ auto import_to_git = [critical_git_op_map](auto ts,
+ auto setter,
+ auto logger,
+ auto /*unused*/,
+ auto const& key) {
+ // Perform initial commit at location: init + add . + commit
+ GitOpKey op_key = {
+ {
+ key.target_path, // target_path
+ "", // git_hash
+ "", // branch
+ fmt::format(
+ "Content of {} {}", key.repo_type, key.content), // message
+ },
+ GitOpType::INITIAL_COMMIT};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [critical_git_op_map,
+ target_path = key.target_path,
+ ts,
+ setter,
+ logger](auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Initial commit failed",
+ /*fatal=*/true);
+ return;
+ }
+ // ensure Git cache
+ // define Git operation to be done
+ GitOpKey op_key = {
+ {
+ JustMR::Utils::GetGitCacheRoot(), // target_path
+ "", // git_hash
+ "", // branch
+ std::nullopt, // message
+ true // init_bare
+ },
+ GitOpType::ENSURE_INIT};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [critical_git_op_map,
+ commit = *op_result.result,
+ target_path,
+ git_cas = op_result.git_cas,
+ ts,
+ setter,
+ logger](auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Git init failed",
+ /*fatal=*/true);
+ return;
+ }
+ // fetch all into Git cache
+ auto just_git_repo = GitRepo::Open(op_result.git_cas);
+ if (not just_git_repo) {
+ (*logger)(fmt::format("Could not open Git "
+ "repository {}",
+ target_path.string()),
+ /*fatal=*/true);
+ return;
+ }
+ // define temp repo path
+ auto tmp_repo_path = CreateUniquePath(target_path);
+ if (not tmp_repo_path) {
+ (*logger)(
+ fmt::format("Could not create unique path "
+ "for target {}",
+ target_path.string()),
+ /*fatal=*/true);
+ return;
+ }
+ auto wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [&logger, target_path](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While fetch via tmp repo "
+ "for target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ auto success =
+ just_git_repo->FetchViaTmpRepo(*tmp_repo_path,
+ target_path.string(),
+ "",
+ wrapped_logger);
+ if (not success) {
+ return;
+ }
+ // setup a wrapped_logger
+ wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [&logger, target_path](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While doing keep commit "
+ "and setting Git tree:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ KeepCommitAndSetTree(critical_git_op_map,
+ commit,
+ target_path,
+ git_cas,
+ ts,
+ setter,
+ wrapped_logger);
+ },
+ [logger, target_path = JustMR::Utils::GetGitCacheRoot()](
+ auto const& msg, bool fatal) {
+ (*logger)(fmt::format("While running critical Git "
+ "op ENSURE_INIT bare for "
+ "target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ },
+ [logger, target_path = key.target_path](auto const& msg,
+ bool fatal) {
+ (*logger)(fmt::format("While running critical Git op "
+ "INITIAL_COMMIT for target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ };
+ return AsyncMapConsumer<CommitInfo, std::pair<std::string, GitCASPtr>>(
+ import_to_git, jobs);
+}
+
+void KeepCommitAndSetTree(
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ std::string const& commit,
+ std::filesystem::path const& target_path,
+ GitCASPtr const& git_cas,
+ gsl::not_null<TaskSystem*> const& ts,
+ ImportToGitMap::SetterPtr const& setter,
+ ImportToGitMap::LoggerPtr const& logger) {
+ // Keep tag for commit
+ GitOpKey op_key = {{
+ JustMR::Utils::GetGitCacheRoot(), // target_path
+ commit, // git_hash
+ "", // branch
+ "Keep referenced tree alive" // message
+ },
+ GitOpType::KEEP_TAG};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [commit, target_path, git_cas, setter, logger](auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Keep tag failed",
+ /*fatal=*/true);
+ return;
+ }
+ auto git_repo = GitRepo::Open(git_cas);
+ if (not git_repo) {
+ (*logger)(fmt::format("Could not open Git repository {}",
+ target_path.string()),
+ /*fatal=*/true);
+ return;
+ }
+ // get tree id and return it
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [&logger, commit](auto const& msg, bool fatal) {
+ (*logger)(
+ fmt::format("While getting subtree from commit {}:\n{}",
+ commit,
+ msg),
+ fatal);
+ });
+ auto tree_hash =
+ git_repo->GetSubtreeFromCommit(commit, ".", wrapped_logger);
+ if (not tree_hash) {
+ return;
+ }
+ (*setter)(std::pair<std::string, GitCASPtr>(*tree_hash, git_cas));
+ },
+ [logger, commit, target_path](auto const& msg, bool fatal) {
+ (*logger)(fmt::format("While running critical Git op KEEP_TAG for "
+ "commit {} in target {}:\n{}",
+ commit,
+ target_path.string(),
+ msg),
+ fatal);
+ });
+}
diff --git a/src/other_tools/ops_maps/import_to_git_map.hpp b/src/other_tools/ops_maps/import_to_git_map.hpp
new file mode 100644
index 00000000..32b7b638
--- /dev/null
+++ b/src/other_tools/ops_maps/import_to_git_map.hpp
@@ -0,0 +1,74 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_IMPORT_TO_GIT_MAP_HPP
+#define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_IMPORT_TO_GIT_MAP_HPP
+
+#include <filesystem>
+#include <string>
+
+#include "src/buildtool/file_system/git_repo.hpp"
+#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
+#include "src/utils/cpp/path.hpp"
+
+struct CommitInfo {
+ std::filesystem::path target_path{}; /*key*/
+ std::string repo_type{};
+ std::string content{}; // hash or path
+
+ CommitInfo(std::filesystem::path const& target_path_,
+ std::string repo_type_,
+ std::string content_)
+ : target_path{std::filesystem::absolute(ToNormalPath(target_path_))},
+ repo_type{std::move(repo_type_)},
+ content{std::move(content_)} {};
+
+ [[nodiscard]] auto operator==(CommitInfo const& other) const noexcept
+ -> bool {
+ return target_path == other.target_path;
+ }
+};
+
+namespace std {
+template <>
+struct hash<CommitInfo> {
+ [[nodiscard]] auto operator()(CommitInfo const& ct) const noexcept
+ -> std::size_t {
+ // hash_value is used due to a bug in stdlibc++
+ // (see critical_git_op_map.hpp)
+ return std::filesystem::hash_value(ct.target_path);
+ }
+};
+} // namespace std
+
+/// \brief Maps a directory on the file system to a pair of the tree hash of the
+/// content of the directory and the Git ODB it is now a part of.
+using ImportToGitMap =
+ AsyncMapConsumer<CommitInfo, std::pair<std::string, GitCASPtr>>;
+
+[[nodiscard]] auto CreateImportToGitMap(
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ std::size_t jobs) -> ImportToGitMap;
+
+// Helper function
+void KeepCommitAndSetTree(
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ std::string const& commit,
+ std::filesystem::path const& target,
+ GitCASPtr const& git_cas,
+ gsl::not_null<TaskSystem*> const& ts,
+ ImportToGitMap::SetterPtr const& setter,
+ ImportToGitMap::LoggerPtr const& logger);
+
+#endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_IMPORT_TO_GIT_MAP_HPP \ No newline at end of file
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-06 03:14:35 +0000