summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/other_tools/root_maps/TARGETS15
-rw-r--r--src/other_tools/root_maps/commit_git_map.cpp301
-rw-r--r--src/other_tools/root_maps/commit_git_map.hpp64
3 files changed, 380 insertions, 0 deletions
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index de73ee00..733e3d1b 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -17,4 +17,19 @@
, ["src/utils/cpp", "tmp_dir"]
]
}
+, "commit_git_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["commit_git_map"]
+ , "hdrs": ["commit_git_map.hpp"]
+ , "srcs": ["commit_git_map.cpp"]
+ , "deps":
+ [ ["src/other_tools/just_mr", "utils"]
+ , ["src/other_tools/ops_maps", "critical_git_op_map"]
+ , ["src/utils/cpp", "hash_combine"]
+ , ["@", "json", "", "json"]
+ ]
+ , "stage": ["src", "other_tools", "root_maps"]
+ , "private-deps":
+ [["src/buildtool/file_system", "git_repo"], ["src/utils/cpp", "tmp_dir"]]
+ }
}
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
new file mode 100644
index 00000000..deefaa32
--- /dev/null
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -0,0 +1,301 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/other_tools/root_maps/commit_git_map.hpp"
+
+#include <algorithm>
+
+#include "src/buildtool/file_system/git_repo.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
+
+namespace {
+
+[[nodiscard]] auto GitURLIsPath(std::string const& url) noexcept -> bool {
+ auto prefixes = std::vector<std::string>{"ssh://", "http://", "https://"};
+ // return true if no URL prefix exists
+ return std::none_of(
+ prefixes.begin(), prefixes.end(), [url](auto const& prefix) {
+ return (url.rfind(prefix, 0) == 0);
+ });
+}
+
+} // namespace
+
+/// \brief Create a CommitGitMap object
+auto CreateCommitGitMap(
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ JustMR::PathsPtr const& just_mr_paths,
+ std::size_t jobs) -> CommitGitMap {
+ auto commit_to_git = [critical_git_op_map, just_mr_paths](auto ts,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
+ // get root for repo (making sure that if repo is a path, it is
+ // absolute)
+ std::string fetch_repo = key.repo_url;
+ if (GitURLIsPath(fetch_repo)) {
+ fetch_repo = std::filesystem::absolute(fetch_repo).string();
+ }
+ std::filesystem::path repo_root =
+ JustMR::Utils::GetGitRoot(just_mr_paths, fetch_repo);
+ // ensure git repo
+ // define Git operation to be done
+ GitOpKey op_key = {
+ {
+ repo_root, // target_path
+ "", // git_hash
+ "", // branch
+ std::nullopt, // message
+ not just_mr_paths->git_checkout_locations.contains(
+ fetch_repo) // init_bare
+ },
+ GitOpType::ENSURE_INIT};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [key,
+ fetch_repo,
+ repo_root,
+ critical_git_op_map,
+ ts,
+ setter,
+ logger](auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Git init failed",
+ /*fatal=*/true);
+ return;
+ }
+ // setup a wrapped_logger
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [&logger, target_path = repo_root](auto const& msg,
+ bool fatal) {
+ (*logger)(fmt::format("While ensuring commit for "
+ "repository {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ EnsureCommit(key,
+ fetch_repo,
+ repo_root,
+ op_result.git_cas,
+ critical_git_op_map,
+ ts,
+ setter,
+ wrapped_logger);
+ },
+ [logger, target_path = repo_root](auto const& msg, bool fatal) {
+ (*logger)(fmt::format("While running critical Git op "
+ "ENSURE_INIT for target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ };
+ return AsyncMapConsumer<GitRepoInfo, nlohmann::json>(commit_to_git, jobs);
+}
+
+void EnsureCommit(GitRepoInfo const& repo_info,
+ std::string const& fetch_repo,
+ std::filesystem::path const& repo_root,
+ GitCASPtr const& git_cas,
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<TaskSystem*> const& ts,
+ CommitGitMap::SetterPtr const& ws_setter,
+ CommitGitMap::LoggerPtr const& logger) {
+ // ensure commit exists, and fetch if needed
+ auto git_repo = GitRepo::Open(git_cas); // link fake repo to odb
+ if (not git_repo) {
+ (*logger)(
+ fmt::format("Could not open repository {}", repo_root.string()),
+ /*fatal=*/true);
+ return;
+ }
+ // setup wrapped logger
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [&logger](auto const& msg, bool fatal) {
+ (*logger)(fmt::format("While checking commit exists:\n{}", msg),
+ fatal);
+ });
+ auto is_commit_present =
+ git_repo->CheckCommitExists(repo_info.hash, wrapped_logger);
+ if (is_commit_present == std::nullopt) {
+ return;
+ }
+ if (not *is_commit_present) {
+ // if commit not there, fetch it
+ // get refspec for branch
+ GitOpKey op_key = {{
+ repo_root, // target_path
+ "", // git_hash
+ repo_info.branch, // branch
+ },
+ GitOpType::GET_BRANCH_REFNAME};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [critical_git_op_map,
+ git_cas,
+ repo_info,
+ fetch_repo,
+ repo_root,
+ ts,
+ ws_setter,
+ logger](auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Get branch refname failed",
+ /*fatal=*/true);
+ return;
+ }
+ // ensure commit exists, and fetch if needed
+ auto git_repo =
+ GitRepo::Open(git_cas); // link fake repo to odb
+ if (not git_repo) {
+ (*logger)(fmt::format("Could not open repository {}",
+ repo_root.string()),
+ /*fatal=*/true);
+ return;
+ }
+ // do fetch
+ auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("fetch");
+ if (not tmp_dir) {
+ (*logger)("Failed to create fetch tmp directory!",
+ /*fatal=*/true);
+ return;
+ }
+ // setup wrapped logger
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [&logger](auto const& msg, bool fatal) {
+ (*logger)(fmt::format(
+ "While fetching via tmp repo:\n{}", msg),
+ fatal);
+ });
+ if (not git_repo->FetchViaTmpRepo(tmp_dir->GetPath(),
+ fetch_repo,
+ *op_result.result,
+ wrapped_logger)) {
+ return;
+ }
+ // setup wrapped logger
+ wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [&logger](auto const& msg, bool fatal) {
+ (*logger)(fmt::format(
+ "While checking commit exists:\n{}", msg),
+ fatal);
+ });
+ // check if commit exists now, after fetch
+ auto is_commit_present =
+ git_repo->CheckCommitExists(repo_info.hash, wrapped_logger);
+ if (not is_commit_present) {
+ return;
+ }
+ if (not *is_commit_present) {
+ // commit could not be fetched, so fail
+ (*logger)(fmt::format("Could not update commit from branch "
+ "{} for remote {}",
+ repo_info.branch,
+ fetch_repo),
+ /*fatal=*/true);
+ return;
+ }
+ // keep tag
+ GitOpKey op_key = {{
+ repo_root, // target_path
+ repo_info.hash, // git_hash
+ "", // branch
+ "Keep referenced tree alive" // message
+ },
+ GitOpType::KEEP_TAG};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [git_cas, repo_info, repo_root, ws_setter, logger](
+ auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Keep tag failed",
+ /*fatal=*/true);
+ return;
+ }
+ // ensure commit exists, and fetch if needed
+ auto git_repo =
+ GitRepo::Open(git_cas); // link fake repo to odb
+ if (not git_repo) {
+ (*logger)(
+ fmt::format("Could not open repository {}",
+ repo_root.string()),
+ /*fatal=*/true);
+ return;
+ }
+ // setup wrapped logger
+ auto wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [&logger](auto const& msg, bool fatal) {
+ (*logger)(
+ fmt::format("While getting subtree "
+ "from commit:\n{}",
+ msg),
+ fatal);
+ });
+ // get tree id and return workspace root
+ auto subtree = git_repo->GetSubtreeFromCommit(
+ repo_info.hash, repo_info.subdir, wrapped_logger);
+ if (not subtree) {
+ return;
+ }
+ // set the workspace root
+ (*ws_setter)(nlohmann::json::array(
+ {"git tree", *subtree, repo_root}));
+ },
+ [logger, target_path = repo_root](auto const& msg,
+ bool fatal) {
+ (*logger)(fmt::format("While running critical Git op "
+ "KEEP_TAG for target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ },
+ [logger, target_path = repo_root](auto const& msg, bool fatal) {
+ (*logger)(fmt::format("While running critical Git op "
+ "GET_BRANCH_REFNAME for target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ }
+ else {
+ // setup wrapped logger
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [&logger](auto const& msg, bool fatal) {
+ (*logger)(
+ fmt::format("While getting subtree from commit:\n{}", msg),
+ fatal);
+ });
+ // get tree id and return workspace root
+ auto subtree = git_repo->GetSubtreeFromCommit(
+ repo_info.hash, repo_info.subdir, wrapped_logger);
+ if (not subtree) {
+ return;
+ }
+ // set the workspace root
+ (*ws_setter)(nlohmann::json::array({"git tree", *subtree, repo_root}));
+ }
+}
diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp
new file mode 100644
index 00000000..e032cf65
--- /dev/null
+++ b/src/other_tools/root_maps/commit_git_map.hpp
@@ -0,0 +1,64 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP
+#define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP
+
+#include <string>
+
+#include "nlohmann/json.hpp"
+#include "src/other_tools/just_mr/utils.hpp"
+#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+
+struct GitRepoInfo {
+ // hash can be a commit or tree
+ std::string hash{}; /* key */
+ std::string repo_url{};
+ std::string branch{};
+ std::string subdir{};
+
+ [[nodiscard]] auto operator==(const GitRepoInfo& other) const -> bool {
+ return hash == other.hash;
+ }
+};
+
+namespace std {
+template <>
+struct hash<GitRepoInfo> {
+ [[nodiscard]] auto operator()(const GitRepoInfo& ct) const noexcept
+ -> std::size_t {
+ return std::hash<std::string>{}(ct.hash);
+ }
+};
+} // namespace std
+
+/// \brief Maps a Git repository commit hash to its tree workspace root.
+using CommitGitMap = AsyncMapConsumer<GitRepoInfo, nlohmann::json>;
+
+[[nodiscard]] auto CreateCommitGitMap(
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ JustMR::PathsPtr const& just_mr_paths,
+ std::size_t jobs) -> CommitGitMap;
+
+void EnsureCommit(GitRepoInfo const& repo_info,
+ std::string const& fetch_repo,
+ std::filesystem::path const& repo_root,
+ GitCASPtr const& git_cas,
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<TaskSystem*> const& ts,
+ CommitGitMap::SetterPtr const& ws_setter,
+ CommitGitMap::LoggerPtr const& logger);
+
+#endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP \ No newline at end of file
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-05 13:26:25 +0000