summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/other_tools/just_mr/main.cpp6
-rw-r--r--src/other_tools/repo_map/repos_to_setup_map.cpp63
-rw-r--r--src/other_tools/root_maps/TARGETS1
-rw-r--r--src/other_tools/root_maps/distdir_git_map.cpp133
-rw-r--r--src/other_tools/root_maps/distdir_git_map.hpp3
5 files changed, 108 insertions, 98 deletions
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index 3411349c..374b160b 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -968,8 +968,10 @@ void DefaultReachableRepositories(
&critical_git_op_map,
&import_to_git_map,
arguments.common.jobs);
- auto distdir_git_map = CreateDistdirGitMap(
- &import_to_git_map, &critical_git_op_map, arguments.common.jobs);
+ auto distdir_git_map = CreateDistdirGitMap(&content_cas_map,
+ &import_to_git_map,
+ &critical_git_op_map,
+ arguments.common.jobs);
auto repos_to_setup_map = CreateReposToSetupMap(config,
main,
interactive,
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index c0327d18..438d5a64 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -266,7 +266,6 @@ void FileCheckout(ExpressionPtr const& repo_desc,
void DistdirCheckout(ExpressionPtr const& repo_desc,
ExpressionPtr&& repos,
std::string const& repo_name,
- gsl::not_null<ContentCASMap*> const& content_cas_map,
gsl::not_null<DistdirGitMap*> const& distdir_git_map,
gsl::not_null<TaskSystem*> const& ts,
ReposToSetupMap::SetterPtr const& setter,
@@ -292,7 +291,7 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
// get distdir list
auto distdir_repos = repo_desc_repositories->get()->List();
// create list of archives to fetch
- std::vector<ArchiveContent> dist_repos_to_fetch{};
+ auto dist_repos_to_fetch = std::make_shared<std::vector<ArchiveContent>>();
for (auto const& dist_repo : distdir_repos) {
// get name of dist_repo
auto dist_repo_name = dist_repo->String();
@@ -410,51 +409,30 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
.string());
distdir_content->insert_or_assign(repo_distfile, archive.content);
// add to fetch list
- dist_repos_to_fetch.emplace_back(std::move(archive));
+ dist_repos_to_fetch->emplace_back(std::move(archive));
}
}
- // fetch the gathered distdir repos into CAS
- content_cas_map->ConsumeAfterKeysReady(
+ // get hash of distdir content
+ auto distdir_content_id =
+ HashFunction::ComputeBlobHash(nlohmann::json(*distdir_content).dump())
+ .HexString();
+ // get the WS root as git tree
+ DistdirInfo distdir_info = {
+ distdir_content_id, distdir_content, dist_repos_to_fetch};
+ distdir_git_map->ConsumeAfterKeysReady(
ts,
- dist_repos_to_fetch,
- [distdir_content = std::move(distdir_content),
- repos = std::move(repos),
- repo_name,
- distdir_git_map,
- ts,
- setter,
- logger]([[maybe_unused]] auto const& values) mutable {
- // repos are in CAS
- // get hash of distdir content
- auto distdir_content_id =
- HashFunction::ComputeBlobHash(
- nlohmann::json(*distdir_content).dump())
- .HexString();
- // get the WS root as git tree
- DistdirInfo distdir_info = {distdir_content_id, distdir_content};
- distdir_git_map->ConsumeAfterKeysReady(
- ts,
- {std::move(distdir_info)},
- [repos = std::move(repos), repo_name, setter, logger](
- auto const& values) {
- auto ws_root = *values[0];
- nlohmann::json cfg({});
- cfg["workspace_root"] = ws_root;
- SetReposTakeOver(&cfg, repos, repo_name);
- (*setter)(std::move(cfg));
- },
- [logger, repo_name](auto const& msg, bool fatal) {
- (*logger)(
- fmt::format("While setting the workspace root for "
- "repository {} of type distdir:\n{}",
- repo_name,
- msg),
- fatal);
- });
+ {std::move(distdir_info)},
+ [repos = std::move(repos), repo_name, setter, logger](
+ auto const& values) {
+ auto ws_root = *values[0];
+ nlohmann::json cfg({});
+ cfg["workspace_root"] = ws_root;
+ SetReposTakeOver(&cfg, repos, repo_name);
+ (*setter)(std::move(cfg));
},
[logger, repo_name](auto const& msg, bool fatal) {
- (*logger)(fmt::format("While fetching archives for distdir "
- "repository {}:\n{}",
+ (*logger)(fmt::format("While setting the workspace root for "
+ "repository {} of type distdir:\n{}",
repo_name,
msg),
fatal);
@@ -589,7 +567,6 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
DistdirCheckout(*resolved_repo_desc,
std::move(repos),
key,
- content_cas_map,
distdir_git_map,
ts,
setter,
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index 9cd243ba..64f44494 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -6,6 +6,7 @@
, "deps":
[ ["@", "json", "", "json"]
, ["src/other_tools/ops_maps", "import_to_git_map"]
+ , ["src/other_tools/ops_maps", "content_cas_map"]
]
, "stage": ["src", "other_tools", "root_maps"]
, "private-deps":
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
index e1d15c2d..4e75658e 100644
--- a/src/other_tools/root_maps/distdir_git_map.cpp
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -21,6 +21,7 @@
#include "src/buildtool/execution_api/local/file_storage.hpp"
#include "src/buildtool/execution_api/local/local_cas.hpp"
#include "src/other_tools/just_mr/utils.hpp"
+#include "src/other_tools/ops_maps/content_cas_map.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
#include "src/utils/cpp/tmp_dir.hpp"
@@ -49,15 +50,17 @@ namespace {
} // namespace
auto CreateDistdirGitMap(
+ gsl::not_null<ContentCASMap*> const& content_cas_map,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
std::size_t jobs) -> DistdirGitMap {
- auto distdir_to_git = [import_to_git_map, critical_git_op_map](
- auto ts,
- auto setter,
- auto logger,
- auto /* unused */,
- auto const& key) {
+ auto distdir_to_git = [content_cas_map,
+ import_to_git_map,
+ critical_git_op_map](auto ts,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
auto distdir_tree_id_file =
JustMR::Utils::GetDistdirTreeIDFile(key.content_id);
if (FileSystemManager::Exists(distdir_tree_id_file)) {
@@ -111,61 +114,85 @@ auto CreateDistdirGitMap(
});
}
else {
- // create the links to CAS
- auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("distdir");
- if (not tmp_dir) {
- (*logger)(fmt::format(
- "Failed to create tmp path for distdir target {}",
- key.content_id),
- /*fatal=*/true);
- return;
- }
- // link content from CAS into tmp dir
- if (not LinkToCAS(key.content_list, tmp_dir->GetPath())) {
- (*logger)(fmt::format("Failed to create links to CAS content!",
- key.content_id),
- /*fatal=*/true);
- return;
- }
- // do import to git
- CommitInfo c_info{tmp_dir->GetPath(), "distdir", key.content_id};
- import_to_git_map->ConsumeAfterKeysReady(
+ // fetch the gathered distdir repos into CAS
+ content_cas_map->ConsumeAfterKeysReady(
ts,
- {std::move(c_info)},
- [tmp_dir, // keep tmp_dir alive
- distdir_tree_id_file,
+ *key.repos_to_fetch,
+ [distdir_tree_id_file,
+ content_id = key.content_id,
+ content_list = key.content_list,
+ import_to_git_map,
+ ts,
setter,
- logger](auto const& values) {
- // check for errors
- if (not values[0]->second) {
- (*logger)("Importing to git failed",
+ logger]([[maybe_unused]] auto const& values) mutable {
+ // repos are in CAS
+ // create the links to CAS
+ auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("distdir");
+ if (not tmp_dir) {
+ (*logger)(fmt::format("Failed to create tmp path for "
+ "distdir target {}",
+ content_id),
/*fatal=*/true);
return;
}
- // only the tree is of interest
- std::string distdir_tree_id = values[0]->first;
- // write to tree id file
- if (not JustMR::Utils::WriteTreeIDFile(distdir_tree_id_file,
- distdir_tree_id)) {
- (*logger)(
- fmt::format("Failed to write tree id to file {}",
- distdir_tree_id_file.string()),
- /*fatal=*/true);
+ // link content from CAS into tmp dir
+ if (not LinkToCAS(content_list, tmp_dir->GetPath())) {
+ (*logger)(fmt::format(
+ "Failed to create links to CAS content!",
+ content_id),
+ /*fatal=*/true);
return;
}
- // set the workspace root
- (*setter)(nlohmann::json::array(
- {"git tree",
- distdir_tree_id,
- JustMR::Utils::GetGitCacheRoot().string()}));
+ // do import to git
+ CommitInfo c_info{
+ tmp_dir->GetPath(), "distdir", content_id};
+ import_to_git_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(c_info)},
+ [tmp_dir, // keep tmp_dir alive
+ distdir_tree_id_file,
+ setter,
+ logger](auto const& values) {
+ // check for errors
+ if (not values[0]->second) {
+ (*logger)("Importing to git failed",
+ /*fatal=*/true);
+ return;
+ }
+ // only the tree is of interest
+ std::string distdir_tree_id = values[0]->first;
+ // write to tree id file
+ if (not JustMR::Utils::WriteTreeIDFile(
+ distdir_tree_id_file, distdir_tree_id)) {
+ (*logger)(
+ fmt::format(
+ "Failed to write tree id to file {}",
+ distdir_tree_id_file.string()),
+ /*fatal=*/true);
+ return;
+ }
+ // set the workspace root
+ (*setter)(nlohmann::json::array(
+ {"git tree",
+ distdir_tree_id,
+ JustMR::Utils::GetGitCacheRoot().string()}));
+ },
+ [logger, target_path = tmp_dir->GetPath()](
+ auto const& msg, bool fatal) {
+ (*logger)(fmt::format("While importing target {} "
+ "to git:\n{}",
+ target_path.string(),
+ msg),
+ fatal);
+ });
},
- [logger, target_path = tmp_dir->GetPath()](auto const& msg,
- bool fatal) {
- (*logger)(
- fmt::format("While importing target {} to git:\n{}",
- target_path.string(),
- msg),
- fatal);
+ [logger, content_id = key.content_id](auto const& msg,
+ bool fatal) {
+ (*logger)(fmt::format("While fetching archives for distdir "
+ "content {}:\n{}",
+ content_id,
+ msg),
+ fatal);
});
}
};
diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp
index 91c26ad5..8afeb3d9 100644
--- a/src/other_tools/root_maps/distdir_git_map.hpp
+++ b/src/other_tools/root_maps/distdir_git_map.hpp
@@ -16,11 +16,13 @@
#define INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_DISTDIR_GIT_MAP_HPP
#include "nlohmann/json.hpp"
+#include "src/other_tools/ops_maps/content_cas_map.hpp"
#include "src/other_tools/ops_maps/import_to_git_map.hpp"
struct DistdirInfo {
std::string content_id; /* key */
std::shared_ptr<std::unordered_map<std::string, std::string>> content_list;
+ std::shared_ptr<std::vector<ArchiveContent>> repos_to_fetch;
[[nodiscard]] auto operator==(const DistdirInfo& other) const noexcept
-> bool {
@@ -33,6 +35,7 @@ struct DistdirInfo {
using DistdirGitMap = AsyncMapConsumer<DistdirInfo, nlohmann::json>;
[[nodiscard]] auto CreateDistdirGitMap(
+ gsl::not_null<ContentCASMap*> const& content_cas_map,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
std::size_t jobs) -> DistdirGitMap;
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 10:35:20 +0000