summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/other_tools/just_mr/setup.cpp12
-rw-r--r--src/other_tools/root_maps/TARGETS2
-rw-r--r--src/other_tools/root_maps/tree_id_git_map.cpp102
-rw-r--r--src/other_tools/root_maps/tree_id_git_map.hpp5
4 files changed, 109 insertions, 12 deletions
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 90070db1..6cd9462b 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -151,10 +151,14 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
&import_to_git_map,
&critical_git_op_map,
common_args.jobs);
- auto tree_id_git_map = CreateTreeIdGitMap(&critical_git_op_map,
- common_args.git_path->string(),
- *common_args.local_launcher,
- common_args.jobs);
+ auto tree_id_git_map =
+ CreateTreeIdGitMap(&critical_git_op_map,
+ &import_to_git_map,
+ common_args.git_path->string(),
+ *common_args.local_launcher,
+ local_api ? &(*local_api) : nullptr,
+ remote_api ? &(*remote_api) : nullptr,
+ common_args.jobs);
auto repos_to_setup_map = CreateReposToSetupMap(config,
main,
interactive,
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index 9340cd14..4669d30c 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -115,7 +115,9 @@
, "srcs": ["tree_id_git_map.cpp"]
, "deps":
[ ["@", "json", "", "json"]
+ , ["src/buildtool/execution_api/common", "common"]
, ["src/other_tools/ops_maps", "critical_git_op_map"]
+ , ["src/other_tools/ops_maps", "import_to_git_map"]
, ["src/utils/cpp", "hash_combine"]
]
, "stage": ["src", "other_tools", "root_maps"]
diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp
index 752f224d..a61ded3a 100644
--- a/src/other_tools/root_maps/tree_id_git_map.cpp
+++ b/src/other_tools/root_maps/tree_id_git_map.cpp
@@ -87,15 +87,22 @@ void KeepCommitAndSetRoot(
auto CreateTreeIdGitMap(
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
std::string const& git_bin,
std::vector<std::string> const& launcher,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
std::size_t jobs) -> TreeIdGitMap {
- auto tree_to_git = [critical_git_op_map, git_bin, launcher](
- auto ts,
- auto setter,
- auto logger,
- auto /*unused*/,
- auto const& key) {
+ auto tree_to_git = [critical_git_op_map,
+ import_to_git_map,
+ git_bin,
+ launcher,
+ local_api,
+ remote_api](auto ts,
+ auto setter,
+ auto logger,
+ auto /*unused*/,
+ auto const& key) {
// if root is absent, no work needs to be done
if (key.absent) {
auto root = nlohmann::json::array(
@@ -120,8 +127,16 @@ auto CreateTreeIdGitMap(
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
- [critical_git_op_map, git_bin, launcher, key, ts, setter, logger](
- auto const& values) {
+ [critical_git_op_map,
+ import_to_git_map,
+ git_bin,
+ launcher,
+ local_api,
+ remote_api,
+ key,
+ ts,
+ setter,
+ logger](auto const& values) {
GitOpValue op_result = *values[0];
// check flag
if (not op_result.result) {
@@ -155,6 +170,77 @@ auto CreateTreeIdGitMap(
}
if (not *tree_found) {
JustMRProgress::Instance().TaskTracker().Start(key.origin);
+ // check if tree is in remote CAS, if a remote is given
+ auto digest = ArtifactDigest{key.hash, 0, /*is_tree=*/true};
+ if (remote_api != nullptr and local_api != nullptr and
+ remote_api->IsAvailable(digest) and
+ remote_api->RetrieveToCas(
+ {Artifact::ObjectInfo{.digest = digest,
+ .type = ObjectType::Tree}},
+ local_api)) {
+ JustMRProgress::Instance().TaskTracker().Stop(
+ key.origin);
+ // Move tree from CAS to local Git storage
+ auto tmp_dir = StorageUtils::CreateTypedTmpDir(
+ "fetch-remote-git-tree");
+ if (not tmp_dir) {
+ (*logger)(fmt::format("Failed to create tmp "
+ "directory for copying "
+ "git-tree {} from remote CAS",
+ key.hash),
+ true);
+ return;
+ }
+ if (not local_api->RetrieveToPaths(
+ {Artifact::ObjectInfo{
+ .digest = digest,
+ .type = ObjectType::Tree}},
+ {tmp_dir->GetPath()})) {
+ (*logger)(
+ fmt::format("Failed to copy git-tree {} to {}",
+ key.hash,
+ tmp_dir->GetPath().string()),
+ true);
+ return;
+ }
+ CommitInfo c_info{tmp_dir->GetPath(), "tree", key.hash};
+ import_to_git_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(c_info)},
+ [tmp_dir, // keep tmp_dir alive
+ key,
+ setter,
+ logger](auto const& values) {
+ if (not values[0]->second) {
+ (*logger)("Importing to git failed",
+ /*fatal=*/true);
+ return;
+ }
+ // set the workspace root
+ auto root = nlohmann::json::array(
+ {key.ignore_special
+ ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ key.hash});
+ if (not key.absent) {
+ root.emplace_back(
+ StorageConfig::GitRoot().string());
+ }
+ (*setter)(std::pair(std::move(root), false));
+ },
+ [logger, tmp_dir, tree_id = key.hash](
+ auto const& msg, bool fatal) {
+ (*logger)(
+ fmt::format("While moving git-tree {} from "
+ "{} to local git:\n{}",
+ tree_id,
+ tmp_dir->GetPath().string(),
+ msg),
+ fatal);
+ });
+
+ return;
+ }
// create temporary location for command execution root
auto tmp_dir = StorageUtils::CreateTypedTmpDir("git-tree");
if (not tmp_dir) {
diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp
index 8041919d..04708e3a 100644
--- a/src/other_tools/root_maps/tree_id_git_map.hpp
+++ b/src/other_tools/root_maps/tree_id_git_map.hpp
@@ -20,7 +20,9 @@
#include <vector>
#include "nlohmann/json.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
+#include "src/other_tools/ops_maps/import_to_git_map.hpp"
#include "src/utils/cpp/hash_combine.hpp"
struct TreeIdInfo {
@@ -62,8 +64,11 @@ using TreeIdGitMap =
[[nodiscard]] auto CreateTreeIdGitMap(
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
std::string const& git_bin,
std::vector<std::string> const& launcher,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
std::size_t jobs) -> TreeIdGitMap;
#endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_TREE_ID_GIT_MAP_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 16:53:41 +0000