summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/other_tools/just_mr/TARGETS2
-rw-r--r--src/other_tools/just_mr/setup.cpp16
-rw-r--r--src/other_tools/just_mr/setup_utils.cpp22
-rw-r--r--src/other_tools/just_mr/setup_utils.hpp12
-rw-r--r--src/other_tools/ops_maps/content_cas_map.hpp6
-rw-r--r--src/other_tools/repo_map/repos_to_setup_map.cpp47
-rw-r--r--src/other_tools/root_maps/TARGETS3
-rw-r--r--src/other_tools/root_maps/commit_git_map.cpp81
-rw-r--r--src/other_tools/root_maps/commit_git_map.hpp8
-rw-r--r--src/other_tools/root_maps/content_git_map.cpp42
-rw-r--r--src/other_tools/root_maps/distdir_git_map.cpp33
-rw-r--r--src/other_tools/root_maps/distdir_git_map.hpp9
-rw-r--r--src/other_tools/root_maps/fpath_git_map.cpp33
-rw-r--r--src/other_tools/root_maps/fpath_git_map.hpp7
-rw-r--r--src/other_tools/root_maps/tree_id_git_map.cpp36
-rw-r--r--src/other_tools/root_maps/tree_id_git_map.hpp6
16 files changed, 265 insertions, 98 deletions
diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index e0a05163..a64f4c3c 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -85,6 +85,7 @@
[ ["src/buildtool/build_engine/expression", "expression_ptr_interface"]
, ["src/buildtool/build_engine/expression", "expression"]
, ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/serve_api/remote", "serve_api"]
, "cli"
]
, "stage": ["src", "other_tools", "just_mr"]
@@ -96,6 +97,7 @@
, ["src/buildtool/auth", "auth"]
, ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
, ["src/buildtool/execution_api/remote", "bazel"]
+ , ["src/buildtool/serve_api/remote", "config"]
]
}
, "fetch":
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 7b606fd3..ed09bb61 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -94,6 +94,10 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
IExecutionApi::Ptr local_api{remote_api ? std::make_unique<LocalApi>()
: nullptr};
+ // setup the API for serving tree of known commit
+ auto serve_api = JustMR::Utils::SetupServeApi(
+ common_args.remote_serve_address, auth_args);
+
// setup the required async maps
auto crit_git_op_ptr = std::make_shared<CriticalGitOpGuard>();
auto critical_git_op_map = CreateCriticalGitOpMap(crit_git_op_ptr);
@@ -110,11 +114,13 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
common_args.jobs);
auto resolve_symlinks_map = CreateResolveSymlinksMap();
- auto commit_git_map = CreateCommitGitMap(&critical_git_op_map,
- common_args.just_mr_paths,
- common_args.git_path->string(),
- *common_args.local_launcher,
- common_args.jobs);
+ auto commit_git_map =
+ CreateCommitGitMap(&critical_git_op_map,
+ common_args.just_mr_paths,
+ common_args.git_path->string(),
+ *common_args.local_launcher,
+ serve_api ? &(*serve_api) : nullptr,
+ common_args.jobs);
auto content_git_map = CreateContentGitMap(&content_cas_map,
&import_to_git_map,
&resolve_symlinks_map,
diff --git a/src/other_tools/just_mr/setup_utils.cpp b/src/other_tools/just_mr/setup_utils.cpp
index 28355e89..b325a71d 100644
--- a/src/other_tools/just_mr/setup_utils.cpp
+++ b/src/other_tools/just_mr/setup_utils.cpp
@@ -15,6 +15,9 @@
#include "src/other_tools/just_mr/setup_utils.hpp"
#include <fstream>
+#include <memory>
+#include <optional>
+#include <string>
#include <unordered_set>
#include "nlohmann/json.hpp"
@@ -24,6 +27,7 @@
#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/serve_api/remote/config.hpp"
#include "src/other_tools/just_mr/exit_codes.hpp"
namespace {
@@ -202,4 +206,22 @@ auto SetupRemoteApi(std::optional<std::string> const& remote_exec_addr,
return nullptr;
}
+auto SetupServeApi(std::optional<std::string> const& remote_serve_addr,
+ MultiRepoRemoteAuthArguments const& auth) -> ServeApi::Ptr {
+ if (remote_serve_addr) {
+ // setup authentication
+ SetupAuthConfig(auth);
+ // setup remote
+ if (not RemoteServeConfig::SetRemoteAddress(*remote_serve_addr)) {
+ Logger::Log(LogLevel::Error,
+ "setting remote serve service address '{}' failed.",
+ *remote_serve_addr);
+ std::exit(kExitConfigError);
+ }
+ auto address = RemoteServeConfig::RemoteAddress();
+ return std::make_unique<ServeApi>(address->host, address->port);
+ }
+ return nullptr;
+}
+
} // namespace JustMR::Utils
diff --git a/src/other_tools/just_mr/setup_utils.hpp b/src/other_tools/just_mr/setup_utils.hpp
index 2bc067cf..3294a7e5 100644
--- a/src/other_tools/just_mr/setup_utils.hpp
+++ b/src/other_tools/just_mr/setup_utils.hpp
@@ -24,6 +24,7 @@
#include "src/buildtool/build_engine/expression/configuration.hpp"
#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/serve_api/remote/serve_api.hpp"
#include "src/other_tools/just_mr/cli.hpp"
/* Setup-related constants and utilities for just-mr */
@@ -58,9 +59,14 @@ void DefaultReachableRepositories(
-> std::shared_ptr<Configuration>;
/// \brief Setup of a remote API based on just-mr arguments.
-auto SetupRemoteApi(std::optional<std::string> const& remote_exec_addr,
- MultiRepoRemoteAuthArguments const& auth)
- -> IExecutionApi::Ptr;
+[[nodiscard]] auto SetupRemoteApi(
+ std::optional<std::string> const& remote_exec_addr,
+ MultiRepoRemoteAuthArguments const& auth) -> IExecutionApi::Ptr;
+
+/// \brief Setup of a 'just serve' remote API based on just-mr arguments.
+[[nodiscard]] auto SetupServeApi(
+ std::optional<std::string> const& remote_serve_addr,
+ MultiRepoRemoteAuthArguments const& auth) -> ServeApi::Ptr;
} // namespace Utils
diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp
index b6413761..859fbe6b 100644
--- a/src/other_tools/ops_maps/content_cas_map.hpp
+++ b/src/other_tools/ops_maps/content_cas_map.hpp
@@ -47,11 +47,14 @@ struct ArchiveRepoInfo {
std::string subdir; /* key */
// create root based on "special" pragma value
std::optional<PragmaSpecial> pragma_special{std::nullopt}; /* key */
+ // create an absent root
+ bool absent{}; /* key */
[[nodiscard]] auto operator==(const ArchiveRepoInfo& other) const -> bool {
return archive == other.archive and repo_type == other.repo_type and
subdir == other.subdir and
- pragma_special == other.pragma_special;
+ pragma_special == other.pragma_special and
+ absent == other.absent;
}
};
@@ -83,6 +86,7 @@ struct hash<ArchiveRepoInfo> {
hash_combine<std::string>(&seed, ct.repo_type);
hash_combine<std::string>(&seed, ct.subdir);
hash_combine<std::optional<PragmaSpecial>>(&seed, ct.pragma_special);
+ hash_combine<bool>(&seed, ct.absent);
return seed;
}
};
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index 69addd88..5aefe3b1 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -102,6 +102,12 @@ void GitCheckout(ExpressionPtr const& repo_desc,
? std::make_optional(
kPragmaSpecialMap.at(pragma_special->get()->String()))
: std::nullopt;
+ // check "absent" pragma
+ auto pragma_absent =
+ repo_desc_pragma ? repo_desc_pragma->get()->At("absent") : std::nullopt;
+ auto pragma_absent_value = pragma_absent and
+ pragma_absent->get()->IsBool() and
+ pragma_absent->get()->Bool();
// populate struct
GitRepoInfo git_repo_info = {
.hash = repo_desc_commit->get()->String(),
@@ -109,7 +115,8 @@ void GitCheckout(ExpressionPtr const& repo_desc,
.branch = repo_desc_branch->get()->String(),
.subdir = subdir.empty() ? "." : subdir.string(),
.origin = repo_name,
- .ignore_special = pragma_special_value == PragmaSpecial::Ignore};
+ .ignore_special = pragma_special_value == PragmaSpecial::Ignore,
+ .absent = pragma_absent_value};
// get the WS root as git tree
commit_git_map->ConsumeAfterKeysReady(
ts,
@@ -192,6 +199,12 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
? std::make_optional(
kPragmaSpecialMap.at(pragma_special->get()->String()))
: std::nullopt;
+ // check "absent" pragma
+ auto pragma_absent =
+ repo_desc_pragma ? repo_desc_pragma->get()->At("absent") : std::nullopt;
+ auto pragma_absent_value = pragma_absent and
+ pragma_absent->get()->IsBool() and
+ pragma_absent->get()->Bool();
// populate struct
ArchiveRepoInfo archive_repo_info = {
.archive =
@@ -210,7 +223,8 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
.origin_from_distdir = false},
.repo_type = repo_type,
.subdir = subdir.empty() ? "." : subdir.string(),
- .pragma_special = pragma_special_value};
+ .pragma_special = pragma_special_value,
+ .absent = pragma_absent_value};
// get the WS root as git tree
content_git_map->ConsumeAfterKeysReady(
ts,
@@ -283,9 +297,17 @@ void FileCheckout(ExpressionPtr const& repo_desc,
pragma_special_value == PragmaSpecial::ResolveCompletely or
(pragma_to_git and pragma_to_git->get()->IsBool() and
pragma_to_git->get()->Bool())) {
+ // check "absent" pragma
+ auto pragma_absent = repo_desc_pragma
+ ? repo_desc_pragma->get()->At("absent")
+ : std::nullopt;
+ auto pragma_absent_value = pragma_absent and
+ pragma_absent->get()->IsBool() and
+ pragma_absent->get()->Bool();
// get the WS root as git tree
FpathInfo fpath_info = {.fpath = fpath,
- .pragma_special = pragma_special_value};
+ .pragma_special = pragma_special_value,
+ .absent = pragma_absent_value};
fpath_git_map->ConsumeAfterKeysReady(
ts,
{std::move(fpath_info)},
@@ -345,6 +367,13 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
/*fatal=*/true);
return;
}
+ // check "absent" pragma
+ auto repo_desc_pragma = repo_desc->At("pragma");
+ auto pragma_absent =
+ repo_desc_pragma ? repo_desc_pragma->get()->At("absent") : std::nullopt;
+ auto pragma_absent_value = pragma_absent and
+ pragma_absent->get()->IsBool() and
+ pragma_absent->get()->Bool();
// map of distfile to content
auto distdir_content =
std::make_shared<std::unordered_map<std::string, std::string>>();
@@ -483,7 +512,8 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
DistdirInfo distdir_info = {.content_id = distdir_content_id,
.content_list = distdir_content,
.repos_to_fetch = dist_repos_to_fetch,
- .origin = repo_name};
+ .origin = repo_name,
+ .absent = pragma_absent_value};
distdir_git_map->ConsumeAfterKeysReady(
ts,
{std::move(distdir_info)},
@@ -586,12 +616,19 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc,
? std::make_optional(
kPragmaSpecialMap.at(pragma_special->get()->String()))
: std::nullopt;
+ // check "absent" pragma
+ auto pragma_absent =
+ repo_desc_pragma ? repo_desc_pragma->get()->At("absent") : std::nullopt;
+ auto pragma_absent_value = pragma_absent and
+ pragma_absent->get()->IsBool() and
+ pragma_absent->get()->Bool();
// populate struct
TreeIdInfo tree_id_info = {
.hash = repo_desc_hash->get()->String(),
.env_vars = std::move(env),
.command = std::move(cmd),
- .ignore_special = pragma_special_value == PragmaSpecial::Ignore};
+ .ignore_special = pragma_special_value == PragmaSpecial::Ignore,
+ .absent = pragma_absent_value};
// get the WS root as git tree
tree_id_git_map->ConsumeAfterKeysReady(
ts,
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index 06ff9ce2..c7089bfc 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -29,7 +29,8 @@
, "hdrs": ["commit_git_map.hpp"]
, "srcs": ["commit_git_map.cpp"]
, "deps":
- [ ["src/other_tools/just_mr", "utils"]
+ [ ["src/buildtool/serve_api/remote", "serve_api"]
+ , ["src/other_tools/just_mr", "utils"]
, ["src/other_tools/ops_maps", "critical_git_op_map"]
, ["src/utils/cpp", "hash_combine"]
, ["@", "json", "", "json"]
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index 5a2f831e..64d686c1 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -39,6 +39,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
std::string const& git_bin,
std::vector<std::string> const& launcher,
+ ServeApi* serve_api,
gsl::not_null<TaskSystem*> const& ts,
CommitGitMap::SetterPtr const& ws_setter,
CommitGitMap::LoggerPtr const& logger) {
@@ -63,7 +64,39 @@ void EnsureCommit(GitRepoInfo const& repo_info,
}
if (not is_commit_present.value()) {
JustMRProgress::Instance().TaskTracker().Start(repo_info.origin);
- // if commit not there, fetch it
+ // check if commit is known to remote serve service, if asked for an
+ // absent root
+ if (repo_info.absent) {
+ if (serve_api != nullptr) {
+ if (auto tree_id = serve_api->RetrieveTreeFromCommit(
+ repo_info.hash, repo_info.subdir)) {
+ // set the workspace root as absent
+ JustMRProgress::Instance().TaskTracker().Stop(
+ repo_info.origin);
+ (*ws_setter)(std::pair(
+ nlohmann::json::array(
+ {repo_info.ignore_special
+ ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ *tree_id}),
+ false));
+ return;
+ }
+ // give warning
+ (*logger)(fmt::format("Tree at subdir {} for commit {} could "
+ "not be served",
+ repo_info.subdir,
+ repo_info.hash),
+ /*fatal=*/false);
+ }
+ else {
+ // give warning
+ (*logger)(
+ "Absent root requested, but no serve endpoint provided",
+ /*fatal=*/false);
+ }
+ }
+ // default to fetching it from network
auto tmp_dir = JustMR::Utils::CreateTypedTmpDir("fetch");
if (not tmp_dir) {
(*logger)("Failed to create fetch tmp directory!",
@@ -152,14 +185,15 @@ void EnsureCommit(GitRepoInfo const& repo_info,
}
// set the workspace root
JustMRProgress::Instance().TaskTracker().Stop(repo_info.origin);
- (*ws_setter)(
- std::pair(nlohmann::json::array(
- {repo_info.ignore_special
- ? FileRoot::kGitTreeIgnoreSpecialMarker
- : FileRoot::kGitTreeMarker,
- *subtree,
- repo_root}),
- false));
+ auto root = nlohmann::json::array(
+ {repo_info.ignore_special
+ ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ *subtree});
+ if (not repo_info.absent) {
+ root.emplace_back(repo_root);
+ }
+ (*ws_setter)(std::pair(std::move(root), false));
},
[logger, target_path = repo_root](auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git op "
@@ -184,13 +218,14 @@ void EnsureCommit(GitRepoInfo const& repo_info,
return;
}
// set the workspace root
- (*ws_setter)(std::pair(
- nlohmann::json::array({repo_info.ignore_special
- ? FileRoot::kGitTreeIgnoreSpecialMarker
- : FileRoot::kGitTreeMarker,
- *subtree,
- repo_root}),
- true));
+ auto root = nlohmann::json::array(
+ {repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ *subtree});
+ if (not repo_info.absent) {
+ root.emplace_back(repo_root);
+ }
+ (*ws_setter)(std::pair(std::move(root), true));
}
}
@@ -202,15 +237,17 @@ auto CreateCommitGitMap(
JustMR::PathsPtr const& just_mr_paths,
std::string const& git_bin,
std::vector<std::string> const& launcher,
+ ServeApi* serve_api,
std::size_t jobs) -> CommitGitMap {
auto commit_to_git = [critical_git_op_map,
just_mr_paths,
git_bin,
- launcher](auto ts,
- auto setter,
- auto logger,
- auto /* unused */,
- auto const& key) {
+ launcher,
+ serve_api](auto ts,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
// get root for repo (making sure that if repo is a path, it is
// absolute)
std::string fetch_repo = key.repo_url;
@@ -240,6 +277,7 @@ auto CreateCommitGitMap(
critical_git_op_map,
git_bin,
launcher,
+ serve_api,
ts,
setter,
logger](auto const& values) {
@@ -266,6 +304,7 @@ auto CreateCommitGitMap(
critical_git_op_map,
git_bin,
launcher,
+ serve_api,
ts,
setter,
wrapped_logger);
diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp
index 1e702e3a..00a607cb 100644
--- a/src/other_tools/root_maps/commit_git_map.hpp
+++ b/src/other_tools/root_maps/commit_git_map.hpp
@@ -19,6 +19,7 @@
#include <utility>
#include "nlohmann/json.hpp"
+#include "src/buildtool/serve_api/remote/serve_api.hpp"
#include "src/other_tools/just_mr/utils.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
#include "src/utils/cpp/hash_combine.hpp"
@@ -33,10 +34,13 @@ struct GitRepoInfo {
std::string origin{};
// create root that ignores symlinks
bool ignore_special{}; /* key */
+ // create an absent root
+ bool absent{}; /* key */
[[nodiscard]] auto operator==(const GitRepoInfo& other) const -> bool {
return hash == other.hash and subdir == other.subdir and
- ignore_special == other.ignore_special;
+ ignore_special == other.ignore_special and
+ absent == other.absent;
}
};
@@ -49,6 +53,7 @@ struct hash<GitRepoInfo> {
hash_combine<std::string>(&seed, ct.hash);
hash_combine<std::string>(&seed, ct.subdir);
hash_combine<bool>(&seed, ct.ignore_special);
+ hash_combine<bool>(&seed, ct.absent);
return seed;
}
};
@@ -64,6 +69,7 @@ using CommitGitMap =
JustMR::PathsPtr const& just_mr_paths,
std::string const& git_bin,
std::vector<std::string> const& launcher,
+ ServeApi* serve_api,
std::size_t jobs) -> CommitGitMap;
#endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp
index 64c3db75..9930984e 100644
--- a/src/other_tools/root_maps/content_git_map.cpp
+++ b/src/other_tools/root_maps/content_git_map.cpp
@@ -46,6 +46,7 @@ void ResolveContentTree(
std::string const& tree_hash,
bool is_cache_hit,
std::optional<PragmaSpecial> const& pragma_special,
+ bool absent,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<TaskSystem*> const& ts,
ContentGitMap::SetterPtr const& ws_setter,
@@ -65,12 +66,12 @@ void ResolveContentTree(
return;
}
// set the workspace root
- (*ws_setter)(std::pair(
- nlohmann::json::array({FileRoot::kGitTreeMarker,
- *resolved_tree_id,
- StorageConfig::GitRoot().string()}),
- true // it is definitely a cache hit
- ));
+ auto root = nlohmann::json::array(
+ {FileRoot::kGitTreeMarker, *resolved_tree_id});
+ if (not absent) {
+ root.emplace_back(StorageConfig::GitRoot().string());
+ }
+ (*ws_setter)(std::pair(std::move(root), true));
}
else {
// resolve tree
@@ -84,6 +85,7 @@ void ResolveContentTree(
tree_hash,
tree_id_file,
is_cache_hit,
+ absent,
ws_setter,
logger](auto const& hashes) {
if (not hashes[0]) {
@@ -115,12 +117,12 @@ void ResolveContentTree(
return;
}
// set the workspace root
- (*ws_setter)(
- std::pair(nlohmann::json::array(
- {FileRoot::kGitTreeMarker,
- resolved_tree.id,
- StorageConfig::GitRoot().string()}),
- is_cache_hit));
+ auto root = nlohmann::json::array(
+ {FileRoot::kGitTreeMarker, resolved_tree.id});
+ if (not absent) {
+ root.emplace_back(StorageConfig::GitRoot().string());
+ }
+ (*ws_setter)(std::pair(std::move(root), is_cache_hit));
},
[logger, content](auto const& msg, bool fatal) {
(*logger)(fmt::format("While resolving symlinks for "
@@ -133,11 +135,12 @@ void ResolveContentTree(
}
else {
// set the workspace root as-is
- (*ws_setter)(std::pair(
- nlohmann::json::array({FileRoot::kGitTreeMarker,
- tree_hash,
- StorageConfig::GitRoot().string()}),
- is_cache_hit));
+ auto root =
+ nlohmann::json::array({FileRoot::kGitTreeMarker, tree_hash});
+ if (not absent) {
+ root.emplace_back(StorageConfig::GitRoot().string());
+ }
+ (*ws_setter)(std::pair(std::move(root), is_cache_hit));
}
}
@@ -187,6 +190,7 @@ auto CreateContentGitMap(
subdir = key.subdir,
content = key.archive.content,
pragma_special = key.pragma_special,
+ absent = key.absent,
resolve_symlinks_map,
ts,
setter,
@@ -226,6 +230,7 @@ auto CreateContentGitMap(
*subtree_hash,
true, /*is_cache_hit*/
pragma_special,
+ absent,
resolve_symlinks_map,
ts,
setter,
@@ -251,6 +256,7 @@ auto CreateContentGitMap(
content_id = key.archive.content,
subdir = key.subdir,
pragma_special = key.pragma_special,
+ absent = key.absent,
import_to_git_map,
resolve_symlinks_map,
ts,
@@ -294,6 +300,7 @@ auto CreateContentGitMap(
content_id,
subdir,
pragma_special,
+ absent,
resolve_symlinks_map,
ts,
setter,
@@ -355,6 +362,7 @@ auto CreateContentGitMap(
*subtree_hash,
false, /*is_cache_hit*/
pragma_special,
+ absent,
resolve_symlinks_map,
ts,
setter,
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
index a0971478..a132cf47 100644
--- a/src/other_tools/root_maps/distdir_git_map.cpp
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -90,8 +90,10 @@ auto CreateDistdirGitMap(
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
- [distdir_tree_id = *distdir_tree_id, setter, logger](
- auto const& values) {
+ [distdir_tree_id = *distdir_tree_id,
+ absent = key.absent,
+ setter,
+ logger](auto const& values) {
GitOpValue op_result = *values[0];
// check flag
if (not op_result.result) {
@@ -101,12 +103,12 @@ auto CreateDistdirGitMap(
}
// subdir is ".", so no need to deal with the Git cache
// set the workspace root
- (*setter)(
- std::pair(nlohmann::json::array(
- {FileRoot::kGitTreeMarker,
- distdir_tree_id,
- StorageConfig::GitRoot().string()}),
- true));
+ auto root = nlohmann::json::array(
+ {FileRoot::kGitTreeMarker, distdir_tree_id});
+ if (not absent) {
+ root.emplace_back(StorageConfig::GitRoot().string());
+ }
+ (*setter)(std::pair(std::move(root), true));
},
[logger, target_path = StorageConfig::GitRoot()](
auto const& msg, bool fatal) {
@@ -126,6 +128,7 @@ auto CreateDistdirGitMap(
[distdir_tree_id_file,
content_id = key.content_id,
content_list = key.content_list,
+ absent = key.absent,
import_to_git_map,
ts,
setter,
@@ -156,6 +159,7 @@ auto CreateDistdirGitMap(
{std::move(c_info)},
[tmp_dir, // keep tmp_dir alive
distdir_tree_id_file,
+ absent,
setter,
logger](auto const& values) {
// check for errors
@@ -177,12 +181,13 @@ auto CreateDistdirGitMap(
return;
}
// set the workspace root
- (*setter)(std::pair(
- nlohmann::json::array(
- {FileRoot::kGitTreeMarker,
- distdir_tree_id,
- StorageConfig::GitRoot().string()}),
- false));
+ auto root = nlohmann::json::array(
+ {FileRoot::kGitTreeMarker, distdir_tree_id});
+ if (not absent) {
+ root.emplace_back(
+ StorageConfig::GitRoot().string());
+ }
+ (*setter)(std::pair(std::move(root), false));
},
[logger, target_path = tmp_dir->GetPath()](
auto const& msg, bool fatal) {
diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp
index c440253b..4c282157 100644
--- a/src/other_tools/root_maps/distdir_git_map.hpp
+++ b/src/other_tools/root_maps/distdir_git_map.hpp
@@ -27,10 +27,12 @@ struct DistdirInfo {
std::shared_ptr<std::vector<ArchiveContent>> repos_to_fetch;
// name of repository for which work is done; used in progress reporting
std::string origin;
+ // create an absent root
+ bool absent{}; /* key */
[[nodiscard]] auto operator==(const DistdirInfo& other) const noexcept
-> bool {
- return content_id == other.content_id;
+ return content_id == other.content_id and absent == other.absent;
}
};
@@ -51,7 +53,10 @@ template <>
struct hash<DistdirInfo> {
[[nodiscard]] auto operator()(const DistdirInfo& dd) const noexcept
-> std::size_t {
- return std::hash<std::string>{}(dd.content_id);
+ size_t seed{};
+ hash_combine<std::string>(&seed, dd.content_id);
+ hash_combine<bool>(&seed, dd.absent);
+ return seed;
}
};
} // namespace std
diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp
index f0a12fd4..87f0ea52 100644
--- a/src/other_tools/root_maps/fpath_git_map.cpp
+++ b/src/other_tools/root_maps/fpath_git_map.cpp
@@ -24,11 +24,11 @@
namespace {
void ResolveFilePathTree(
-
std::string const& repo_root,
std::string const& target_path,
std::string const& tree_hash,
std::optional<PragmaSpecial> const& pragma_special,
+ bool absent,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<TaskSystem*> const& ts,
FilePathGitMap::SetterPtr const& ws_setter,
@@ -48,8 +48,12 @@ void ResolveFilePathTree(
return;
}
// set the workspace root
- (*ws_setter)(nlohmann::json::array(
- {FileRoot::kGitTreeMarker, *resolved_tree_id, repo_root}));
+ auto root = nlohmann::json::array(
+ {FileRoot::kGitTreeMarker, *resolved_tree_id});
+ if (not absent) {
+ root.emplace_back(repo_root);
+ }
+ (*ws_setter)(std::move(root));
}
else {
// resolve tree
@@ -63,6 +67,7 @@ void ResolveFilePathTree(
tree_hash,
repo_root,
tree_id_file,
+ absent,
ws_setter,
logger](auto const& hashes) {
if (not hashes[0]) {
@@ -94,10 +99,12 @@ void ResolveFilePathTree(
return;
}
// set the workspace root
- (*ws_setter)(
- nlohmann::json::array({FileRoot::kGitTreeMarker,
- resolved_tree.id,
- repo_root}));
+ auto root = nlohmann::json::array(
+ {FileRoot::kGitTreeMarker, resolved_tree.id});
+ if (not absent) {
+ root.emplace_back(repo_root);
+ }
+ (*ws_setter)(std::move(root));
},
[logger, target_path](auto const& msg, bool fatal) {
(*logger)(fmt::format(
@@ -110,8 +117,12 @@ void ResolveFilePathTree(
}
else {
// set the workspace root as-is
- (*ws_setter)(nlohmann::json::array(
- {FileRoot::kGitTreeMarker, tree_hash, repo_root}));
+ auto root =
+ nlohmann::json::array({FileRoot::kGitTreeMarker, tree_hash});
+ if (not absent) {
+ root.emplace_back(repo_root);
+ }
+ (*ws_setter)(std::move(root));
}
}
@@ -174,6 +185,7 @@ auto CreateFilePathGitMap(
{std::move(op_key)},
[fpath = key.fpath,
pragma_special = key.pragma_special,
+ absent = key.absent,
git_cas = std::move(git_cas),
repo_root = std::move(*repo_root),
resolve_symlinks_map,
@@ -216,6 +228,7 @@ auto CreateFilePathGitMap(
fpath.string(),
*tree_hash,
pragma_special,
+ absent,
resolve_symlinks_map,
ts,
setter,
@@ -267,6 +280,7 @@ auto CreateFilePathGitMap(
[tmp_dir,
fpath = key.fpath,
pragma_special = key.pragma_special,
+ absent = key.absent,
resolve_symlinks_map,
ts,
setter,
@@ -284,6 +298,7 @@ auto CreateFilePathGitMap(
fpath.string(),
tree,
pragma_special,
+ absent,
resolve_symlinks_map,
ts,
setter,
diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp
index 39cae653..6063aa8d 100644
--- a/src/other_tools/root_maps/fpath_git_map.hpp
+++ b/src/other_tools/root_maps/fpath_git_map.hpp
@@ -26,10 +26,14 @@ struct FpathInfo {
std::filesystem::path fpath{}; /* key */
// create root based on "special" pragma value
std::optional<PragmaSpecial> pragma_special{std::nullopt}; /* key */
+ // create an absent root
+ bool absent{}; /* key */
[[nodiscard]] auto operator==(const FpathInfo& other) const noexcept
-> bool {
- return fpath == other.fpath and pragma_special == other.pragma_special;
+ return fpath == other.fpath and
+ pragma_special == other.pragma_special and
+ absent == other.absent;
}
};
@@ -51,6 +55,7 @@ struct hash<FpathInfo> {
size_t seed{};
hash_combine<std::filesystem::path>(&seed, ct.fpath);
hash_combine<std::optional<PragmaSpecial>>(&seed, ct.pragma_special);
+ hash_combine<bool>(&seed, ct.absent);
return seed;
}
};
diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp
index 920ed615..34da3121 100644
--- a/src/other_tools/root_maps/tree_id_git_map.cpp
+++ b/src/other_tools/root_maps/tree_id_git_map.cpp
@@ -58,16 +58,17 @@ void KeepCommitAndSetRoot(
/*fatal=*/true);
return;
}
+ JustMRProgress::Instance().TaskTracker().Stop(tree_id_info.origin);
// set the workspace root
- JustMRProgress::Instance().TaskTracker().Start(tree_id_info.origin);
- (*ws_setter)(
- std::pair(nlohmann::json::array(
- {tree_id_info.ignore_special
- ? FileRoot::kGitTreeIgnoreSpecialMarker
- : FileRoot::kGitTreeMarker,
- tree_id_info.hash,
- StorageConfig::GitRoot().string()}),
- false));
+ auto root = nlohmann::json::array(
+ {tree_id_info.ignore_special
+ ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ tree_id_info.hash});
+ if (not tree_id_info.absent) {
+ root.emplace_back(StorageConfig::GitRoot().string());
+ }
+ (*ws_setter)(std::pair(std::move(root), false));
},
[logger, commit, target_path = tmp_dir->GetPath()](auto const& msg,
bool fatal) {
@@ -345,14 +346,15 @@ auto CreateTreeIdGitMap(
}
else {
// tree found, so return the git tree root as-is
- (*setter)(std::pair(
- nlohmann::json::array(
- {key.ignore_special
- ? FileRoot::kGitTreeIgnoreSpecialMarker
- : FileRoot::kGitTreeMarker,
- key.hash,
- StorageConfig::GitRoot().string()}),
- true));
+ auto root = nlohmann::json::array(
+ {key.ignore_special
+ ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ key.hash});
+ if (not key.absent) {
+ root.emplace_back(StorageConfig::GitRoot().string());
+ }
+ (*setter)(std::pair(std::move(root), true));
}
},
[logger, target_path = StorageConfig::GitRoot()](auto const& msg,
diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp
index ca5fb9fb..3c029117 100644
--- a/src/other_tools/root_maps/tree_id_git_map.hpp
+++ b/src/other_tools/root_maps/tree_id_git_map.hpp
@@ -30,9 +30,12 @@ struct TreeIdInfo {
std::string origin{};
// create root that ignores symlinks
bool ignore_special{}; /* key */
+ // create an absent root
+ bool absent{}; /* key */
[[nodiscard]] auto operator==(const TreeIdInfo& other) const -> bool {
- return hash == other.hash and ignore_special == other.ignore_special;
+ return hash == other.hash and ignore_special == other.ignore_special and
+ absent == other.absent;
}
};
@@ -44,6 +47,7 @@ struct hash<TreeIdInfo> {
size_t seed{};
hash_combine<std::string>(&seed, ti.hash);
hash_combine<bool>(&seed, ti.ignore_special);
+ hash_combine<bool>(&seed, ti.absent);
return seed;
}
};
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 16:40:22 +0000