summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/other_tools/just_mr/TARGETS2
-rw-r--r--src/other_tools/just_mr/setup.cpp18
-rw-r--r--src/other_tools/ops_maps/TARGETS27
-rw-r--r--src/other_tools/ops_maps/git_tree_fetch_map.cpp406
-rw-r--r--src/other_tools/ops_maps/git_tree_fetch_map.hpp64
-rw-r--r--src/other_tools/repo_map/TARGETS2
-rw-r--r--src/other_tools/repo_map/repos_to_setup_map.cpp10
-rw-r--r--src/other_tools/root_maps/TARGETS16
-rw-r--r--src/other_tools/root_maps/tree_id_git_map.cpp462
-rw-r--r--src/other_tools/root_maps/tree_id_git_map.hpp26
10 files changed, 555 insertions, 478 deletions
diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index 391a8240..8973026a 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -153,7 +153,9 @@
, ["src/other_tools/just_mr/progress_reporting", "progress"]
, ["src/other_tools/just_mr/progress_reporting", "progress_reporter"]
, "utils"
+ , ["src/other_tools/ops_maps", "content_cas_map"]
, ["src/other_tools/ops_maps", "critical_git_op_map"]
+ , ["src/other_tools/ops_maps", "git_tree_fetch_map"]
, ["src/other_tools/repo_map", "repos_to_setup_map"]
, ["src/other_tools/root_maps", "commit_git_map"]
, ["src/other_tools/root_maps", "content_git_map"]
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 6cd9462b..b82993ab 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -29,7 +29,9 @@
#include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp"
#include "src/other_tools/just_mr/setup_utils.hpp"
#include "src/other_tools/just_mr/utils.hpp"
+#include "src/other_tools/ops_maps/content_cas_map.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
+#include "src/other_tools/ops_maps/git_tree_fetch_map.hpp"
#include "src/other_tools/repo_map/repos_to_setup_map.hpp"
#include "src/other_tools/root_maps/commit_git_map.hpp"
#include "src/other_tools/root_maps/content_git_map.hpp"
@@ -115,6 +117,14 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
common_args.git_path->string(),
*common_args.local_launcher,
common_args.jobs);
+ auto git_tree_fetch_map =
+ CreateGitTreeFetchMap(&critical_git_op_map,
+ &import_to_git_map,
+ common_args.git_path->string(),
+ *common_args.local_launcher,
+ local_api ? &(*local_api) : nullptr,
+ remote_api ? &(*remote_api) : nullptr,
+ common_args.jobs);
auto resolve_symlinks_map = CreateResolveSymlinksMap();
auto commit_git_map =
@@ -152,13 +162,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
&critical_git_op_map,
common_args.jobs);
auto tree_id_git_map =
- CreateTreeIdGitMap(&critical_git_op_map,
- &import_to_git_map,
- common_args.git_path->string(),
- *common_args.local_launcher,
- local_api ? &(*local_api) : nullptr,
- remote_api ? &(*remote_api) : nullptr,
- common_args.jobs);
+ CreateTreeIdGitMap(&git_tree_fetch_map, common_args.jobs);
auto repos_to_setup_map = CreateReposToSetupMap(config,
main,
interactive,
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS
index c0f05d63..f77ac3e9 100644
--- a/src/other_tools/ops_maps/TARGETS
+++ b/src/other_tools/ops_maps/TARGETS
@@ -92,4 +92,31 @@
, ["src/other_tools/just_mr/progress_reporting", "progress"]
]
}
+, "git_tree_fetch_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["git_tree_fetch_map"]
+ , "hdrs": ["git_tree_fetch_map.hpp"]
+ , "srcs": ["git_tree_fetch_map.cpp"]
+ , "deps":
+ [ ["@", "json", "", "json"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/other_tools/ops_maps", "critical_git_op_map"]
+ , ["src/other_tools/ops_maps", "import_to_git_map"]
+ ]
+ , "stage": ["src", "other_tools", "ops_maps"]
+ , "private-deps":
+ [ ["src/other_tools/ops_maps", "critical_git_op_map"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/storage", "config"]
+ , ["src/buildtool/storage", "fs_utils"]
+ , ["src/buildtool/system", "system_command"]
+ , ["src/other_tools/git_operations", "git_repo_remote"]
+ , ["src/other_tools/just_mr/progress_reporting", "progress"]
+ , ["src/other_tools/just_mr/progress_reporting", "statistics"]
+ , ["src/other_tools/ops_maps", "content_cas_map"]
+ , ["src/other_tools/ops_maps", "import_to_git_map"]
+ , ["@", "fmt", "", "fmt"]
+ ]
+ }
}
diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.cpp b/src/other_tools/ops_maps/git_tree_fetch_map.cpp
new file mode 100644
index 00000000..e928fb2f
--- /dev/null
+++ b/src/other_tools/ops_maps/git_tree_fetch_map.cpp
@@ -0,0 +1,406 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/other_tools/ops_maps/git_tree_fetch_map.hpp"
+
+#include <cstdlib>
+
+#include "fmt/core.h"
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/storage/config.hpp"
+#include "src/buildtool/storage/fs_utils.hpp"
+#include "src/buildtool/system/system_command.hpp"
+#include "src/other_tools/git_operations/git_repo_remote.hpp"
+#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
+#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
+
+auto CreateGitTreeFetchMap(
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
+ std::string const& git_bin,
+ std::vector<std::string> const& launcher,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
+ std::size_t jobs) -> GitTreeFetchMap {
+ auto tree_to_cache = [critical_git_op_map,
+ import_to_git_map,
+ git_bin,
+ launcher,
+ local_api,
+ remote_api](auto ts,
+ auto setter,
+ auto logger,
+ auto /*unused*/,
+ auto const& key) {
+ // check whether tree exists already in Git cache;
+ // ensure Git cache exists
+ GitOpKey op_key = {.params =
+ {
+ StorageConfig::GitRoot(), // target_path
+ "", // git_hash
+ "", // branch
+ std::nullopt, // message
+ true // init_bare
+ },
+ .op_type = GitOpType::ENSURE_INIT};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [critical_git_op_map,
+ import_to_git_map,
+ git_bin,
+ launcher,
+ local_api,
+ remote_api,
+ key,
+ ts,
+ setter,
+ logger](auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Git cache init failed",
+ /*fatal=*/true);
+ return;
+ }
+ // Open fake tmp repo to check if tree is known to Git cache
+ auto git_repo = GitRepoRemote::Open(
+ op_result.git_cas); // link fake repo to odb
+ if (not git_repo) {
+ (*logger)(fmt::format("Could not open repository {}",
+ StorageConfig::GitRoot().string()),
+ /*fatal=*/true);
+ return;
+ }
+ // setup wrapped logger
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [logger](auto const& msg, bool fatal) {
+ (*logger)(fmt::format("While checking tree exists in "
+ "Git cache:\n{}",
+ msg),
+ fatal);
+ });
+ // check if the desired tree ID is in Git cache
+ auto tree_found =
+ git_repo->CheckTreeExists(key.hash, wrapped_logger);
+ if (not tree_found) {
+ // errors encountered
+ return;
+ }
+ if (*tree_found) {
+ (*setter)(true /*cache hit*/);
+ return;
+ }
+ JustMRProgress::Instance().TaskTracker().Start(key.origin);
+ // check if tree is in remote CAS, if a remote is given
+ auto digest = ArtifactDigest{key.hash, 0, /*is_tree=*/true};
+ if (remote_api != nullptr and local_api != nullptr and
+ remote_api->IsAvailable(digest) and
+ remote_api->RetrieveToCas(
+ {Artifact::ObjectInfo{.digest = digest,
+ .type = ObjectType::Tree}},
+ local_api)) {
+ JustMRProgress::Instance().TaskTracker().Stop(key.origin);
+ // Move tree from CAS to local Git storage
+ auto tmp_dir = StorageUtils::CreateTypedTmpDir(
+ "fetch-remote-git-tree");
+ if (not tmp_dir) {
+ (*logger)(fmt::format("Failed to create tmp "
+ "directory for copying "
+ "git-tree {} from remote CAS",
+ key.hash),
+ true);
+ return;
+ }
+ if (not local_api->RetrieveToPaths(
+ {Artifact::ObjectInfo{.digest = digest,
+ .type = ObjectType::Tree}},
+ {tmp_dir->GetPath()})) {
+ (*logger)(
+ fmt::format("Failed to copy git-tree {} to {}",
+ key.hash,
+ tmp_dir->GetPath().string()),
+ true);
+ return;
+ }
+ CommitInfo c_info{tmp_dir->GetPath(), "tree", key.hash};
+ import_to_git_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(c_info)},
+ [tmp_dir, // keep tmp_dir alive
+ key,
+ setter,
+ logger](auto const& values) {
+ if (not values[0]->second) {
+ (*logger)("Importing to git failed",
+ /*fatal=*/true);
+ return;
+ }
+ // success
+ (*setter)(false /*no cache hit*/);
+ },
+ [logger, tmp_dir, tree_id = key.hash](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While moving git-tree {} from "
+ "{} to local git:\n{}",
+ tree_id,
+ tmp_dir->GetPath().string(),
+ msg),
+ fatal);
+ });
+
+ return;
+ }
+ // create temporary location for command execution root
+ auto tmp_dir = StorageUtils::CreateTypedTmpDir("git-tree");
+ if (not tmp_dir) {
+ (*logger)("Failed to create tmp directory for tree id map!",
+ /*fatal=*/true);
+ return;
+ }
+ // create temporary location for storing command result files
+ auto out_dir = StorageUtils::CreateTypedTmpDir("git-tree");
+ if (not out_dir) {
+ (*logger)("Failed to create tmp directory for tree id map!",
+ /*fatal=*/true);
+ return;
+ }
+ // execute command in temporary location
+ SystemCommand system{key.hash};
+ auto cmdline = launcher;
+ std::copy(key.command.begin(),
+ key.command.end(),
+ std::back_inserter(cmdline));
+ std::map<std::string, std::string> env{key.env_vars};
+ for (auto const& k : key.inherit_env) {
+ const char* v = std::getenv(k.c_str());
+ if (v != nullptr) {
+ env[k] = std::string(v);
+ }
+ }
+ auto const command_output = system.Execute(
+ cmdline, env, tmp_dir->GetPath(), out_dir->GetPath());
+ if (not command_output) {
+ (*logger)(fmt::format("Failed to execute command:\n{}",
+ nlohmann::json(cmdline).dump()),
+ /*fatal=*/true);
+ return;
+ }
+ // do an import to git with tree check
+ GitOpKey op_key = {.params =
+ {
+ tmp_dir->GetPath(), // target_path
+ "", // git_hash
+ "", // branch
+ fmt::format("Content of tree {}",
+ key.hash), // message
+ },
+ .op_type = GitOpType::INITIAL_COMMIT};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [tmp_dir, // keep tmp_dir alive
+ out_dir, // keep stdout/stderr of command alive
+ critical_git_op_map,
+ just_git_cas = op_result.git_cas,
+ cmdline,
+ command_output,
+ key,
+ git_bin,
+ launcher,
+ ts,
+ setter,
+ logger](auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Commit failed",
+ /*fatal=*/true);
+ return;
+ }
+ // Open fake tmp repository to check for tree
+ auto git_repo = GitRepoRemote::Open(
+ op_result.git_cas); // link fake repo to odb
+ if (not git_repo) {
+ (*logger)(
+ fmt::format("Could not open repository {}",
+ tmp_dir->GetPath().string()),
+ /*fatal=*/true);
+ return;
+ }
+ // setup wrapped logger
+ auto wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [logger](auto const& msg, bool fatal) {
+ (*logger)(fmt::format("While checking tree "
+ "exists:\n{}",
+ msg),
+ fatal);
+ });
+ // check that the desired tree ID is part of the repo
+ auto tree_check =
+ git_repo->CheckTreeExists(key.hash, wrapped_logger);
+ if (not tree_check) {
+ // errors encountered
+ return;
+ }
+ if (not *tree_check) {
+ std::string out_str{};
+ std::string err_str{};
+ auto cmd_out = FileSystemManager::ReadFile(
+ command_output->stdout_file);
+ auto cmd_err = FileSystemManager::ReadFile(
+ command_output->stderr_file);
+ if (cmd_out) {
+ out_str = *cmd_out;
+ }
+ if (cmd_err) {
+ err_str = *cmd_err;
+ }
+ std::string output{};
+ if (!out_str.empty() || !err_str.empty()) {
+ output =
+ fmt::format(".\nOutput of command:\n{}{}",
+ out_str,
+ err_str);
+ }
+ (*logger)(
+ fmt::format("Executing {} did not create "
+ "specified tree {}{}",
+ nlohmann::json(cmdline).dump(),
+ key.hash,
+ output),
+ /*fatal=*/true);
+ return;
+ }
+ auto target_path = tmp_dir->GetPath();
+ // fetch all into Git cache
+ auto just_git_repo = GitRepoRemote::Open(just_git_cas);
+ if (not just_git_repo) {
+ (*logger)(fmt::format("Could not open Git "
+ "repository {}",
+ target_path.string()),
+ /*fatal=*/true);
+ return;
+ }
+ // define temp repo path
+ auto tmp_dir =
+ StorageUtils::CreateTypedTmpDir("git-tree");
+ ;
+ if (not tmp_dir) {
+ (*logger)(fmt::format("Could not create unique "
+ "path for target {}",
+ target_path.string()),
+ /*fatal=*/true);
+ return;
+ }
+ wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [logger, target_path](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While fetch via tmp repo "
+ "for target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ if (not just_git_repo->FetchViaTmpRepo(
+ tmp_dir->GetPath(),
+ target_path.string(),
+ std::nullopt,
+ git_bin,
+ launcher,
+ wrapped_logger)) {
+ return;
+ }
+ // setup a wrapped_logger
+ wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [logger, target_path](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While doing keep commit "
+ "and setting Git tree for "
+ "target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ // Keep tag for commit
+ GitOpKey op_key = {
+ .params =
+ {
+ StorageConfig::GitRoot(), // target_path
+ *op_result.result, // git_hash
+ "", // branch
+ "Keep referenced tree alive" // message
+ },
+ .op_type = GitOpType::KEEP_TAG};
+ critical_git_op_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(op_key)},
+ [tmp_dir, // keep tmp_dir alive
+ key,
+ setter,
+ logger](auto const& values) {
+ GitOpValue op_result = *values[0];
+ // check flag
+ if (not op_result.result) {
+ (*logger)("Keep tag failed",
+ /*fatal=*/true);
+ return;
+ }
+ JustMRProgress::Instance().TaskTracker().Stop(
+ key.origin);
+ // success
+ (*setter)(false /*no cache hit*/);
+ },
+ [logger,
+ commit = *op_result.result,
+ target_path = tmp_dir->GetPath()](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While running critical Git op "
+ "KEEP_TAG for commit {} in "
+ "target {}:\n{}",
+ commit,
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ },
+ [logger, target_path = tmp_dir->GetPath()](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While running critical Git op "
+ "INITIAL_COMMIT for target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ },
+ [logger, target_path = StorageConfig::GitRoot()](auto const& msg,
+ bool fatal) {
+ (*logger)(fmt::format("While running critical Git op "
+ "ENSURE_INIT bare for target {}:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
+ };
+ return AsyncMapConsumer<GitTreeInfo, bool>(tree_to_cache, jobs);
+}
diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.hpp b/src/other_tools/ops_maps/git_tree_fetch_map.hpp
new file mode 100644
index 00000000..f84056a8
--- /dev/null
+++ b/src/other_tools/ops_maps/git_tree_fetch_map.hpp
@@ -0,0 +1,64 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_TREE_FETCH_MAP_HPP
+#define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_TREE_FETCH_MAP_HPP
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
+#include "src/other_tools/ops_maps/import_to_git_map.hpp"
+
+// Stores all the information needed to make a Git tree available
+struct GitTreeInfo {
+ std::string hash{}; /* key */
+ std::map<std::string, std::string> env_vars{};
+ std::vector<std::string> inherit_env{};
+ std::vector<std::string> command{};
+ // name of repository for which work is done; used in progress reporting
+ std::string origin{};
+
+ [[nodiscard]] auto operator==(const GitTreeInfo& other) const -> bool {
+ return hash == other.hash;
+ }
+};
+
+namespace std {
+template <>
+struct hash<GitTreeInfo> {
+ [[nodiscard]] auto operator()(const GitTreeInfo& gti) const noexcept
+ -> std::size_t {
+ return std::hash<std::string>{}(gti.hash);
+ }
+};
+} // namespace std
+
+/// \brief Maps a known tree provided through a generic command to a flag
+/// signaling if there was a cache hit (i.e., tree was already present).
+using GitTreeFetchMap = AsyncMapConsumer<GitTreeInfo, bool>;
+
+[[nodiscard]] auto CreateGitTreeFetchMap(
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
+ std::string const& git_bin,
+ std::vector<std::string> const& launcher,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
+ std::size_t jobs) -> GitTreeFetchMap;
+
+#endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_GIT_TREE_FETCH_MAP_HPP
diff --git a/src/other_tools/repo_map/TARGETS b/src/other_tools/repo_map/TARGETS
index c397d82c..166e0696 100644
--- a/src/other_tools/repo_map/TARGETS
+++ b/src/other_tools/repo_map/TARGETS
@@ -17,6 +17,8 @@
, ["src/other_tools/just_mr/progress_reporting", "statistics"]
, ["src/buildtool/file_system", "file_root"]
, ["src/buildtool/file_system/symlinks_map", "pragma_special"]
+ , ["src/other_tools/ops_maps", "content_cas_map"]
+ , ["src/other_tools/ops_maps", "git_tree_fetch_map"]
]
}
}
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index c0a8cac1..3f9390b2 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -18,6 +18,8 @@
#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp"
#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
+#include "src/other_tools/ops_maps/content_cas_map.hpp"
+#include "src/other_tools/ops_maps/git_tree_fetch_map.hpp"
namespace {
@@ -740,10 +742,10 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc,
pragma_absent->get()->Bool();
// populate struct
TreeIdInfo tree_id_info = {
- .hash = repo_desc_hash->get()->String(),
- .env_vars = std::move(env),
- .inherit_env = std::move(inherit_env),
- .command = std::move(cmd),
+ .tree_info = GitTreeInfo{.hash = repo_desc_hash->get()->String(),
+ .env_vars = std::move(env),
+ .inherit_env = std::move(inherit_env),
+ .command = std::move(cmd)},
.ignore_special = pragma_special_value == PragmaSpecial::Ignore,
.absent = not fetch_absent and pragma_absent_value};
// get the WS root as git tree
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index 4669d30c..6a5955e2 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -115,24 +115,12 @@
, "srcs": ["tree_id_git_map.cpp"]
, "deps":
[ ["@", "json", "", "json"]
- , ["src/buildtool/execution_api/common", "common"]
- , ["src/other_tools/ops_maps", "critical_git_op_map"]
- , ["src/other_tools/ops_maps", "import_to_git_map"]
+ , ["src/other_tools/ops_maps", "git_tree_fetch_map"]
, ["src/utils/cpp", "hash_combine"]
]
, "stage": ["src", "other_tools", "root_maps"]
, "private-deps":
- [ ["src/other_tools/ops_maps", "critical_git_op_map"]
- , ["src/buildtool/execution_api/common", "common"]
- , ["src/buildtool/file_system", "file_system_manager"]
- , ["src/buildtool/storage", "config"]
- , ["src/buildtool/storage", "fs_utils"]
- , ["src/buildtool/system", "system_command"]
- , ["src/other_tools/git_operations", "git_repo_remote"]
- , ["src/other_tools/just_mr/progress_reporting", "progress"]
- , ["src/other_tools/just_mr/progress_reporting", "statistics"]
- , ["src/other_tools/ops_maps", "content_cas_map"]
- , ["src/other_tools/ops_maps", "import_to_git_map"]
+ [ ["src/buildtool/storage", "config"]
, ["src/buildtool/file_system", "file_root"]
, ["@", "fmt", "", "fmt"]
]
diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp
index a61ded3a..faedccf4 100644
--- a/src/other_tools/root_maps/tree_id_git_map.cpp
+++ b/src/other_tools/root_maps/tree_id_git_map.cpp
@@ -14,457 +14,51 @@
#include "src/other_tools/root_maps/tree_id_git_map.hpp"
-#include <cstdlib>
-
#include "fmt/core.h"
-#include "src/buildtool/execution_api/common/execution_common.hpp"
#include "src/buildtool/file_system/file_root.hpp"
-#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/storage/config.hpp"
-#include "src/buildtool/storage/fs_utils.hpp"
-#include "src/buildtool/system/system_command.hpp"
-#include "src/other_tools/git_operations/git_repo_remote.hpp"
-#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
-#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
-
-namespace {
-
-void KeepCommitAndSetRoot(
- TreeIdInfo const& tree_id_info,
- std::string const& commit,
- TmpDirPtr const& tmp_dir, // need to keep tmp_dir alive
- gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
- gsl::not_null<TaskSystem*> const& ts,
- TreeIdGitMap::SetterPtr const& ws_setter,
- TreeIdGitMap::LoggerPtr const& logger) {
- // Keep tag for commit
- GitOpKey op_key = {.params =
- {
- StorageConfig::GitRoot(), // target_path
- commit, // git_hash
- "", // branch
- "Keep referenced tree alive" // message
- },
- .op_type = GitOpType::KEEP_TAG};
- critical_git_op_map->ConsumeAfterKeysReady(
- ts,
- {std::move(op_key)},
- [tmp_dir, // keep tmp_dir alive
- tree_id_info,
- ws_setter,
- logger](auto const& values) {
- GitOpValue op_result = *values[0];
- // check flag
- if (not op_result.result) {
- (*logger)("Keep tag failed",
- /*fatal=*/true);
- return;
- }
- JustMRProgress::Instance().TaskTracker().Stop(tree_id_info.origin);
- // set the workspace root
- auto root = nlohmann::json::array(
- {tree_id_info.ignore_special
- ? FileRoot::kGitTreeIgnoreSpecialMarker
- : FileRoot::kGitTreeMarker,
- tree_id_info.hash});
- if (not tree_id_info.absent) {
- root.emplace_back(StorageConfig::GitRoot().string());
- }
- (*ws_setter)(std::pair(std::move(root), false));
- },
- [logger, commit, target_path = tmp_dir->GetPath()](auto const& msg,
- bool fatal) {
- (*logger)(fmt::format("While running critical Git op KEEP_TAG for "
- "commit {} in target {}:\n{}",
- commit,
- target_path.string(),
- msg),
- fatal);
- });
-}
-
-} // namespace
auto CreateTreeIdGitMap(
- gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
- gsl::not_null<ImportToGitMap*> const& import_to_git_map,
- std::string const& git_bin,
- std::vector<std::string> const& launcher,
- IExecutionApi* local_api,
- IExecutionApi* remote_api,
+ gsl::not_null<GitTreeFetchMap*> const& git_tree_fetch_map,
std::size_t jobs) -> TreeIdGitMap {
- auto tree_to_git = [critical_git_op_map,
- import_to_git_map,
- git_bin,
- launcher,
- local_api,
- remote_api](auto ts,
- auto setter,
- auto logger,
- auto /*unused*/,
- auto const& key) {
+ auto tree_to_git = [git_tree_fetch_map](auto ts,
+ auto setter,
+ auto logger,
+ auto /*unused*/,
+ auto const& key) {
// if root is absent, no work needs to be done
if (key.absent) {
auto root = nlohmann::json::array(
{key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
- key.hash});
+ key.tree_info.hash});
(*setter)(std::pair(std::move(root), false));
return;
}
- // check whether tree exists already in CAS
- // ensure Git cache
- // define Git operation to be done
- GitOpKey op_key = {.params =
- {
- StorageConfig::GitRoot(), // target_path
- "", // git_hash
- "", // branch
- std::nullopt, // message
- true // init_bare
- },
- .op_type = GitOpType::ENSURE_INIT};
- critical_git_op_map->ConsumeAfterKeysReady(
+ // make sure the required tree is in Git cache
+ git_tree_fetch_map->ConsumeAfterKeysReady(
ts,
- {std::move(op_key)},
- [critical_git_op_map,
- import_to_git_map,
- git_bin,
- launcher,
- local_api,
- remote_api,
- key,
- ts,
- setter,
- logger](auto const& values) {
- GitOpValue op_result = *values[0];
- // check flag
- if (not op_result.result) {
- (*logger)("Git cache init failed",
- /*fatal=*/true);
- return;
- }
- // Open fake tmp repo to check if tree is known to Git cache
- auto git_repo = GitRepoRemote::Open(
- op_result.git_cas); // link fake repo to odb
- if (not git_repo) {
- (*logger)(fmt::format("Could not open repository {}",
- StorageConfig::GitRoot().string()),
- /*fatal=*/true);
- return;
- }
- // setup wrapped logger
- auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
- [logger](auto const& msg, bool fatal) {
- (*logger)(fmt::format("While checking tree exists in "
- "Git cache:\n{}",
- msg),
- fatal);
- });
- // check if the desired tree ID is in Git cache
- auto tree_found =
- git_repo->CheckTreeExists(key.hash, wrapped_logger);
- if (not tree_found) {
- // errors encountered
- return;
- }
- if (not *tree_found) {
- JustMRProgress::Instance().TaskTracker().Start(key.origin);
- // check if tree is in remote CAS, if a remote is given
- auto digest = ArtifactDigest{key.hash, 0, /*is_tree=*/true};
- if (remote_api != nullptr and local_api != nullptr and
- remote_api->IsAvailable(digest) and
- remote_api->RetrieveToCas(
- {Artifact::ObjectInfo{.digest = digest,
- .type = ObjectType::Tree}},
- local_api)) {
- JustMRProgress::Instance().TaskTracker().Stop(
- key.origin);
- // Move tree from CAS to local Git storage
- auto tmp_dir = StorageUtils::CreateTypedTmpDir(
- "fetch-remote-git-tree");
- if (not tmp_dir) {
- (*logger)(fmt::format("Failed to create tmp "
- "directory for copying "
- "git-tree {} from remote CAS",
- key.hash),
- true);
- return;
- }
- if (not local_api->RetrieveToPaths(
- {Artifact::ObjectInfo{
- .digest = digest,
- .type = ObjectType::Tree}},
- {tmp_dir->GetPath()})) {
- (*logger)(
- fmt::format("Failed to copy git-tree {} to {}",
- key.hash,
- tmp_dir->GetPath().string()),
- true);
- return;
- }
- CommitInfo c_info{tmp_dir->GetPath(), "tree", key.hash};
- import_to_git_map->ConsumeAfterKeysReady(
- ts,
- {std::move(c_info)},
- [tmp_dir, // keep tmp_dir alive
- key,
- setter,
- logger](auto const& values) {
- if (not values[0]->second) {
- (*logger)("Importing to git failed",
- /*fatal=*/true);
- return;
- }
- // set the workspace root
- auto root = nlohmann::json::array(
- {key.ignore_special
- ? FileRoot::kGitTreeIgnoreSpecialMarker
- : FileRoot::kGitTreeMarker,
- key.hash});
- if (not key.absent) {
- root.emplace_back(
- StorageConfig::GitRoot().string());
- }
- (*setter)(std::pair(std::move(root), false));
- },
- [logger, tmp_dir, tree_id = key.hash](
- auto const& msg, bool fatal) {
- (*logger)(
- fmt::format("While moving git-tree {} from "
- "{} to local git:\n{}",
- tree_id,
- tmp_dir->GetPath().string(),
- msg),
- fatal);
- });
-
- return;
- }
- // create temporary location for command execution root
- auto tmp_dir = StorageUtils::CreateTypedTmpDir("git-tree");
- if (not tmp_dir) {
- (*logger)(
- "Failed to create tmp directory for tree id map!",
- /*fatal=*/true);
- return;
- }
- // create temporary location for storing command result
- // files
- auto out_dir = StorageUtils::CreateTypedTmpDir("git-tree");
- if (not out_dir) {
- (*logger)(
- "Failed to create tmp directory for tree id map!",
- /*fatal=*/true);
- return;
- }
- // execute command in temporary location
- SystemCommand system{key.hash};
- auto cmdline = launcher;
- std::copy(key.command.begin(),
- key.command.end(),
- std::back_inserter(cmdline));
- std::map<std::string, std::string> env{key.env_vars};
- for (auto const& k : key.inherit_env) {
- const char* v = std::getenv(k.c_str());
- if (v != nullptr) {
- env[k] = std::string(v);
- }
- }
- auto const command_output = system.Execute(
- cmdline, env, tmp_dir->GetPath(), out_dir->GetPath());
- if (not command_output) {
- (*logger)(fmt::format("Failed to execute command:\n{}",
- nlohmann::json(cmdline).dump()),
- /*fatal=*/true);
- return;
- }
-
- // do an import to git with tree check
- GitOpKey op_key = {
- .params =
- {
- tmp_dir->GetPath(), // target_path
- "", // git_hash
- "", // branch
- fmt::format("Content of tree {}",
- key.hash), // message
- },
- .op_type = GitOpType::INITIAL_COMMIT};
- critical_git_op_map->ConsumeAfterKeysReady(
- ts,
- {std::move(op_key)},
- [tmp_dir, // keep tmp_dir alive
- out_dir, // keep stdout/stderr of command alive
- critical_git_op_map,
- just_git_cas = op_result.git_cas,
- cmdline,
- command_output,
- key,
- git_bin,
- launcher,
- ts,
- setter,
- logger](auto const& values) {
- GitOpValue op_result = *values[0];
- // check flag
- if (not op_result.result) {
- (*logger)("Commit failed",
- /*fatal=*/true);
- return;
- }
- // Open fake tmp repository to check for tree
- auto git_repo = GitRepoRemote::Open(
- op_result.git_cas); // link fake repo to odb
- if (not git_repo) {
- (*logger)(
- fmt::format("Could not open repository {}",
- tmp_dir->GetPath().string()),
- /*fatal=*/true);
- return;
- }
- // setup wrapped logger
- auto wrapped_logger =
- std::make_shared<AsyncMapConsumerLogger>(
- [logger](auto const& msg, bool fatal) {
- (*logger)(
- fmt::format("While checking tree "
- "exists:\n{}",
- msg),
- fatal);
- });
- // check that the desired tree ID is part of the
- // repo
- auto tree_check = git_repo->CheckTreeExists(
- key.hash, wrapped_logger);
- if (not tree_check) {
- // errors encountered
- return;
- }
- if (not *tree_check) {
- std::string out_str{};
- std::string err_str{};
- auto cmd_out = FileSystemManager::ReadFile(
- command_output->stdout_file);
- auto cmd_err = FileSystemManager::ReadFile(
- command_output->stderr_file);
- if (cmd_out) {
- out_str = *cmd_out;
- }
- if (cmd_err) {
- err_str = *cmd_err;
- }
- std::string output{};
- if (!out_str.empty() || !err_str.empty()) {
- output = fmt::format(
- ".\nOutput of command:\n{}{}",
- out_str,
- err_str);
- }
- (*logger)(
- fmt::format("Executing {} did not create "
- "specified tree {}{}",
- nlohmann::json(cmdline).dump(),
- key.hash,
- output),
- /*fatal=*/true);
- return;
- }
- auto target_path = tmp_dir->GetPath();
- // fetch all into Git cache
- auto just_git_repo =
- GitRepoRemote::Open(just_git_cas);
- if (not just_git_repo) {
- (*logger)(fmt::format("Could not open Git "
- "repository {}",
- target_path.string()),
- /*fatal=*/true);
- return;
- }
- // define temp repo path
- auto tmp_dir =
- StorageUtils::CreateTypedTmpDir("git-tree");
- ;
- if (not tmp_dir) {
- (*logger)(fmt::format("Could not create unique "
- "path for target {}",
- target_path.string()),
- /*fatal=*/true);
- return;
- }
- wrapped_logger =
- std::make_shared<AsyncMapConsumerLogger>(
- [logger, target_path](auto const& msg,
- bool fatal) {
- (*logger)(
- fmt::format(
- "While fetch via tmp repo "
- "for target {}:\n{}",
- target_path.string(),
- msg),
- fatal);
- });
- if (not just_git_repo->FetchViaTmpRepo(
- tmp_dir->GetPath(),
- target_path.string(),
- std::nullopt,
- git_bin,
- launcher,
- wrapped_logger)) {
- return;
- }
- // setup a wrapped_logger
- wrapped_logger =
- std::make_shared<AsyncMapConsumerLogger>(
- [logger, target_path](auto const& msg,
- bool fatal) {
- (*logger)(fmt::format(
- "While doing keep commit "
- "and setting Git tree "
- "for target {}:\n{}",
- target_path.string(),
- msg),
- fatal);
- });
- KeepCommitAndSetRoot(
- key,
- *op_result.result, // commit id
- tmp_dir,
- critical_git_op_map,
- ts,
- setter,
- wrapped_logger);
- },
- [logger, target_path = tmp_dir->GetPath()](
- auto const& msg, bool fatal) {
- (*logger)(
- fmt::format("While running critical Git op "
- "INITIAL_COMMIT for target {}:\n{}",
- target_path.string(),
- msg),
- fatal);
- });
- }
- else {
- // tree found, so return the git tree root as-is
- auto root = nlohmann::json::array(
- {key.ignore_special
- ? FileRoot::kGitTreeIgnoreSpecialMarker
- : FileRoot::kGitTreeMarker,
- key.hash});
- if (not key.absent) {
- root.emplace_back(StorageConfig::GitRoot().string());
- }
- (*setter)(std::pair(std::move(root), true));
+ {key.tree_info},
+ [key, setter](auto const& values) {
+ // tree is now in Git cache;
+ // get cache hit info
+ auto is_cache_hit = *values[0];
+ // set the workspace root
+ auto root = nlohmann::json::array(
+ {key.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ key.tree_info.hash});
+ if (not key.absent) {
+ root.emplace_back(StorageConfig::GitRoot().string());
}
+ (*setter)(std::pair(std::move(root), is_cache_hit));
},
- [logger, target_path = StorageConfig::GitRoot()](auto const& msg,
- bool fatal) {
- (*logger)(fmt::format("While running critical Git "
- "op ENSURE_INIT bare for "
- "target {}:\n{}",
- target_path.string(),
- msg),
+ [logger, tree_id = key.tree_info.hash](auto const& msg,
+ bool fatal) {
+ (*logger)(fmt::format(
+ "While ensuring git-tree {} is in Git cache:\n{}",
+ tree_id,
+ msg),
fatal);
});
};
diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp
index 04708e3a..89d1af08 100644
--- a/src/other_tools/root_maps/tree_id_git_map.hpp
+++ b/src/other_tools/root_maps/tree_id_git_map.hpp
@@ -19,26 +19,19 @@
#include <utility>
#include <vector>
-#include "nlohmann/json.hpp"
-#include "src/buildtool/execution_api/common/execution_api.hpp"
-#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
-#include "src/other_tools/ops_maps/import_to_git_map.hpp"
+#include "src/other_tools/ops_maps/git_tree_fetch_map.hpp"
#include "src/utils/cpp/hash_combine.hpp"
struct TreeIdInfo {
- std::string hash{}; /* key */
- std::map<std::string, std::string> env_vars{};
- std::vector<std::string> inherit_env{};
- std::vector<std::string> command{};
- // name of repository for which work is done; used in progress reporting
- std::string origin{};
+ GitTreeInfo tree_info{}; /* key */
// create root that ignores symlinks
bool ignore_special{}; /* key */
// create an absent root
bool absent{}; /* key */
[[nodiscard]] auto operator==(const TreeIdInfo& other) const -> bool {
- return hash == other.hash and ignore_special == other.ignore_special and
+ return tree_info == other.tree_info and
+ ignore_special == other.ignore_special and
absent == other.absent;
}
};
@@ -49,7 +42,7 @@ struct hash<TreeIdInfo> {
[[nodiscard]] auto operator()(const TreeIdInfo& ti) const noexcept
-> std::size_t {
size_t seed{};
- hash_combine<std::string>(&seed, ti.hash);
+ hash_combine<GitTreeInfo>(&seed, ti.tree_info);
hash_combine<bool>(&seed, ti.ignore_special);
hash_combine<bool>(&seed, ti.absent);
return seed;
@@ -58,17 +51,12 @@ struct hash<TreeIdInfo> {
} // namespace std
/// \brief Maps a known tree provided through a generic command to its
-/// workspace root and the information whether it was a cache it.
+/// workspace root and the information whether it was a cache hit.
using TreeIdGitMap =
AsyncMapConsumer<TreeIdInfo, std::pair<nlohmann::json, bool>>;
[[nodiscard]] auto CreateTreeIdGitMap(
- gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
- gsl::not_null<ImportToGitMap*> const& import_to_git_map,
- std::string const& git_bin,
- std::vector<std::string> const& launcher,
- IExecutionApi* local_api,
- IExecutionApi* remote_api,
+ gsl::not_null<GitTreeFetchMap*> const& git_tree_fetch_map,
std::size_t jobs) -> TreeIdGitMap;
#endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_TREE_ID_GIT_MAP_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 16:53:02 +0000