summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/main/serve.cpp75
-rw-r--r--src/buildtool/serve_api/remote/serve_api.cpp5
-rw-r--r--src/buildtool/serve_api/remote/serve_api.hpp3
-rw-r--r--src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp11
-rw-r--r--src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp4
-rw-r--r--src/buildtool/serve_api/serve_service/TARGETS10
-rw-r--r--src/buildtool/serve_api/serve_service/just_serve.proto31
-rw-r--r--src/buildtool/serve_api/serve_service/target_level_cache_server.cpp76
-rw-r--r--src/buildtool/serve_api/serve_service/target_level_cache_server.hpp14
-rw-r--r--src/other_tools/just_mr/cli.hpp6
-rw-r--r--src/other_tools/just_mr/setup.cpp5
-rw-r--r--src/other_tools/repo_map/repos_to_setup_map.cpp42
-rw-r--r--src/other_tools/repo_map/repos_to_setup_map.hpp1
-rw-r--r--src/other_tools/root_maps/TARGETS2
-rw-r--r--src/other_tools/root_maps/commit_git_map.cpp195
-rw-r--r--src/other_tools/root_maps/commit_git_map.hpp6
16 files changed, 419 insertions, 67 deletions
diff --git a/src/buildtool/main/serve.cpp b/src/buildtool/main/serve.cpp
index da7ed62a..74d78bdc 100644
--- a/src/buildtool/main/serve.cpp
+++ b/src/buildtool/main/serve.cpp
@@ -57,6 +57,18 @@ void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs) {
clargs->serve.config.string());
std::exit(kExitFailure);
}
+ // read local build root
+ auto local_root = serve_config["local build root"];
+ if (local_root.IsNotNull()) {
+ if (not local_root->IsString()) {
+ Logger::Log(LogLevel::Error,
+ "just serve: configuration-file provided local root "
+ "has to be a string, but found {}",
+ local_root->ToString());
+ std::exit(kExitFailure);
+ }
+ clargs->endpoint.local_root = local_root->String();
+ }
// read paths of additional lookup repositories
auto repositories = serve_config["repositories"];
if (repositories.IsNotNull()) {
@@ -176,6 +188,30 @@ void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs) {
}
clargs->auth.tls_ca_cert = cacert->String();
}
+ // read the TLS client certificate
+ auto client_cert = auth_args->Get("client cert", Expression::none_t{});
+ if (client_cert.IsNotNull()) {
+ if (not client_cert->IsString()) {
+ Logger::Log(LogLevel::Error,
+ "Configuration-provided TLS client certificate has "
+ "to be a string, but found {}",
+ client_cert->ToString());
+ std::exit(kExitFailure);
+ }
+ clargs->cauth.tls_client_cert = client_cert->String();
+ }
+ // read the TLS client key
+ auto client_key = auth_args->Get("client key", Expression::none_t{});
+ if (client_key.IsNotNull()) {
+ if (not client_key->IsString()) {
+ Logger::Log(LogLevel::Error,
+ "Configuration-provided TLS client key has to be a "
+ "string, but found {}",
+ client_key->ToString());
+ std::exit(kExitFailure);
+ }
+ clargs->cauth.tls_client_key = client_key->String();
+ }
}
// read remote service arguments
auto remote_service = serve_config["remote service"];
@@ -271,6 +307,45 @@ void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs) {
clargs->sauth.tls_server_key = server_key->String();
}
}
+ // read execution endpoint arguments
+ auto exec_endpoint = serve_config["execution endpoint"];
+ if (exec_endpoint.IsNotNull()) {
+ if (not exec_endpoint->IsMap()) {
+ Logger::Log(LogLevel::Error,
+ "just-serve: configuration-file provided execution "
+ "endpoint has to be a map, but found {}",
+ exec_endpoint->ToString());
+ std::exit(kExitFailure);
+ }
+ // read the compatible flag
+ auto compatible =
+ exec_endpoint->Get("compatible", Expression::none_t{});
+ if (compatible.IsNotNull()) {
+ if (not compatible->IsBool()) {
+ Logger::Log(LogLevel::Error,
+ "just-serve: expected execution endpoint "
+ "compatible to be a flag, but found {}",
+ compatible->ToString());
+ std::exit(kExitFailure);
+ }
+ // compatibility is set immediately if flag is true
+ if (compatible->Bool()) {
+ Compatibility::SetCompatible();
+ }
+ }
+ // read the address
+ auto address = exec_endpoint->Get("address", Expression::none_t{});
+ if (address.IsNotNull()) {
+ if (not address->IsString()) {
+ Logger::Log(LogLevel::Error,
+ "Configuration-provided execution endpoint address "
+ "has to be a string, but found {}",
+ address->ToString());
+ std::exit(kExitFailure);
+ }
+ clargs->endpoint.remote_execution_address = address->String();
+ }
+ }
}
#endif // BOOTSTRAP_BUILD_TOOL
diff --git a/src/buildtool/serve_api/remote/serve_api.cpp b/src/buildtool/serve_api/remote/serve_api.cpp
index 67b5a8d4..7b57d7d0 100644
--- a/src/buildtool/serve_api/remote/serve_api.cpp
+++ b/src/buildtool/serve_api/remote/serve_api.cpp
@@ -24,7 +24,8 @@ ServeApi::ServeApi(ServeApi&& other) noexcept = default;
ServeApi::~ServeApi() = default;
auto ServeApi::RetrieveTreeFromCommit(std::string const& commit,
- std::string const& subdir)
+ std::string const& subdir,
+ bool sync_tree)
-> std::optional<std::string> {
- return tlc_->ServeCommitTree(commit, subdir);
+ return tlc_->ServeCommitTree(commit, subdir, sync_tree);
}
diff --git a/src/buildtool/serve_api/remote/serve_api.hpp b/src/buildtool/serve_api/remote/serve_api.hpp
index f1724cf9..d3fc29d1 100644
--- a/src/buildtool/serve_api/remote/serve_api.hpp
+++ b/src/buildtool/serve_api/remote/serve_api.hpp
@@ -37,7 +37,8 @@ class ServeApi final {
~ServeApi();
[[nodiscard]] auto RetrieveTreeFromCommit(std::string const& commit,
- std::string const& subdir = ".")
+ std::string const& subdir = ".",
+ bool sync_tree = false)
-> std::optional<std::string>;
private:
diff --git a/src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp b/src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp
index e09b3d03..3109d4d9 100644
--- a/src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp
+++ b/src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp
@@ -24,11 +24,13 @@ ServeTargetLevelCacheClient::ServeTargetLevelCacheClient(
}
auto ServeTargetLevelCacheClient::ServeCommitTree(std::string const& commit_id,
- std::string const& subdir)
+ std::string const& subdir,
+ bool sync_tree)
-> std::optional<std::string> {
justbuild::just_serve::ServeCommitTreeRequest request{};
request.set_commit(commit_id);
request.set_subdir(subdir);
+ request.set_sync_tree(sync_tree);
grpc::ClientContext context;
justbuild::just_serve::ServeCommitTreeResponse response;
@@ -38,8 +40,11 @@ auto ServeTargetLevelCacheClient::ServeCommitTree(std::string const& commit_id,
LogStatus(&logger_, LogLevel::Debug, status);
return std::nullopt;
}
- if (response.status().code() != grpc::StatusCode::OK) {
- LogStatus(&logger_, LogLevel::Debug, response.status());
+ if (response.status() !=
+ ::justbuild::just_serve::ServeCommitTreeStatus::OK) {
+ logger_.Emit(LogLevel::Debug,
+ "ServeCommitTree response returned with {}",
+ static_cast<int>(response.status()));
return std::nullopt;
}
return response.tree();
diff --git a/src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp b/src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp
index fa89c24b..a9b2a3d2 100644
--- a/src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp
+++ b/src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp
@@ -31,9 +31,11 @@ class ServeTargetLevelCacheClient {
/// \brief Retrieve the Git tree of a given commit, if known by the remote.
/// \param[in] commit_id Hash of the Git commit to look up.
/// \param[in] subdir Relative path of the tree inside commit.
+ /// \param[in] sync_tree Sync tree to the remote-execution end point
/// \returns The hash of the tree if commit found, nullopt otherwise.
[[nodiscard]] auto ServeCommitTree(std::string const& commit_id,
- std::string const& subdir)
+ std::string const& subdir,
+ bool sync_tree)
-> std::optional<std::string>;
private:
diff --git a/src/buildtool/serve_api/serve_service/TARGETS b/src/buildtool/serve_api/serve_service/TARGETS
index 93155ae7..984806c9 100644
--- a/src/buildtool/serve_api/serve_service/TARGETS
+++ b/src/buildtool/serve_api/serve_service/TARGETS
@@ -12,10 +12,18 @@
, "hdrs": ["target_level_cache_server.hpp"]
, "srcs": ["target_level_cache_server.cpp"]
, "proto": ["just_serve_proto"]
- , "deps": [["src/buildtool/logging", "logging"]]
+ , "deps":
+ [ ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/execution_api/remote", "config"]
+ ]
, "stage": ["src", "buildtool", "serve_api", "serve_service"]
, "private-deps":
[ ["@", "fmt", "", "fmt"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
+ , ["src/buildtool/execution_api/local", "local"]
+ , ["src/buildtool/execution_api/remote", "bazel"]
, ["src/buildtool/file_system", "git_repo"]
, ["src/buildtool/serve_api/remote", "config"]
, ["src/buildtool/storage", "config"]
diff --git a/src/buildtool/serve_api/serve_service/just_serve.proto b/src/buildtool/serve_api/serve_service/just_serve.proto
index 3eca2954..c20f77aa 100644
--- a/src/buildtool/serve_api/serve_service/just_serve.proto
+++ b/src/buildtool/serve_api/serve_service/just_serve.proto
@@ -16,8 +16,6 @@ syntax = "proto3";
package justbuild.just_serve;
-import "google/rpc/status.proto";
-
// A request message for
// [TargetLevelCache.ServeCommitTree][justbuild.just_serve.TargetLevelCache.ServeCommitTree].
message ServeCommitTreeRequest {
@@ -26,21 +24,36 @@ message ServeCommitTreeRequest {
// Relative path of requested tree with respect to the commit root.
string subdir = 2;
+
+ // If set to true and the tree is found, it will be uploaded to the
+ // remote-execution end point.
+ bool sync_tree = 3;
}
+enum ServeCommitTreeStatus{
+ // All good
+ OK = 0;
+
+ // Failed to upload tree remotely
+ SYNC_ERROR = 1;
+
+ // Tree not found
+ NOT_FOUND = 2;
+
+ // Internaly, something is very broken
+ INTERNAL_ERROR = 3;
+}
+
+
// A response message for
// [TargetLevelCache.ServeCommitTree][justbuild.just_serve.TargetLevelCache.ServeCommitTree].
message ServeCommitTreeResponse {
// The requested Git tree hash.
string tree = 1;
- // If the status has a code other than `OK`, it indicates that the tree hash
- // could not be computed. In this case, the `tree` field is optional.
- //
- // If the status code is `NOT_FOUND`, it indicates that either the commit was
- // not found, or the commit root tree does not contain the given relative
- // path.
- google.rpc.Status status = 2;
+ // If the status has a code `OK` or `SYNC_ERROR`, the tree is correct.
+ // For any other value, the `tree` field is not set.
+ ServeCommitTreeStatus status = 2;
}
// Services for improved interaction with the target-level cache.
diff --git a/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp b/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp
index 688a5c32..e0451c63 100644
--- a/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp
+++ b/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp
@@ -15,10 +15,28 @@
#include "src/buildtool/serve_api/serve_service/target_level_cache_server.hpp"
#include "fmt/core.h"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_api/local/local_api.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
#include "src/buildtool/file_system/git_repo.hpp"
#include "src/buildtool/serve_api/remote/config.hpp"
#include "src/buildtool/storage/config.hpp"
+auto TargetLevelCacheService::CreateExecutionApi(
+ std::optional<RemoteExecutionConfig::ServerAddress> const& address)
+ -> gsl::not_null<IExecutionApi::Ptr> {
+ if (address) {
+ ExecutionConfiguration config;
+ config.skip_cache_lookup = false;
+
+ return std::make_unique<BazelApi>(
+ "remote-execution", address->host, address->port, config);
+ }
+ return std::make_unique<LocalApi>();
+}
+
auto TargetLevelCacheService::GetTreeFromCommit(
std::filesystem::path const& repo_path,
std::string const& commit,
@@ -53,26 +71,74 @@ auto TargetLevelCacheService::ServeCommitTree(
const ::justbuild::just_serve::ServeCommitTreeRequest* request,
::justbuild::just_serve::ServeCommitTreeResponse* response)
-> ::grpc::Status {
+ using ServeCommitTreeStatus =
+ ::justbuild::just_serve::ServeCommitTreeStatus;
auto const& commit{request->commit()};
auto const& subdir{request->subdir()};
// try in local build root Git cache
if (auto tree_id = GetTreeFromCommit(
StorageConfig::GitRoot(), commit, subdir, logger_)) {
+ if (request->sync_tree()) {
+ // sync tree with remote CAS
+ auto digest = ArtifactDigest{*tree_id, 0, /*is_tree=*/true};
+ auto repo = RepositoryConfig{};
+ if (not repo.SetGitCAS(StorageConfig::GitRoot())) {
+ auto str = fmt::format("Failed to SetGitCAS at {}",
+ StorageConfig::GitRoot().string());
+ logger_->Emit(LogLevel::Debug, str);
+ response->set_status(ServeCommitTreeStatus::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ auto git_api = GitApi{&repo};
+ if (not git_api.RetrieveToCas(
+ {Artifact::ObjectInfo{.digest = digest,
+ .type = ObjectType::Tree}},
+ &(*remote_api_))) {
+ auto str = fmt::format("Failed to sync tree {}", *tree_id);
+ logger_->Emit(LogLevel::Debug, str);
+ *(response->mutable_tree()) = std::move(*tree_id);
+ response->set_status(ServeCommitTreeStatus::SYNC_ERROR);
+ return ::grpc::Status::OK;
+ }
+ }
+ // set response
*(response->mutable_tree()) = std::move(*tree_id);
- response->mutable_status()->CopyFrom(google::rpc::Status{});
+ response->set_status(ServeCommitTreeStatus::OK);
return ::grpc::Status::OK;
}
// try given extra repositories, in order
for (auto const& path : RemoteServeConfig::KnownRepositories()) {
if (auto tree_id = GetTreeFromCommit(path, commit, subdir, logger_)) {
+
+ if (request->sync_tree()) {
+ // sync tree with remote CAS
+ auto digest = ArtifactDigest{*tree_id, 0, /*is_tree=*/true};
+ auto repo = RepositoryConfig{};
+ if (not repo.SetGitCAS(path)) {
+ auto str =
+ fmt::format("Failed to SetGitCAS at {}", path.string());
+ logger_->Emit(LogLevel::Debug, str);
+ response->set_status(ServeCommitTreeStatus::INTERNAL_ERROR);
+ return ::grpc::Status::OK;
+ }
+ auto git_api = GitApi{&repo};
+ if (not git_api.RetrieveToCas(
+ {Artifact::ObjectInfo{.digest = digest,
+ .type = ObjectType::Tree}},
+ &(*remote_api_))) {
+ auto str = fmt::format("Failed to sync tree {}", *tree_id);
+ logger_->Emit(LogLevel::Debug, str);
+ *(response->mutable_tree()) = std::move(*tree_id);
+ response->set_status(ServeCommitTreeStatus::SYNC_ERROR);
+ }
+ }
+ // set response
*(response->mutable_tree()) = std::move(*tree_id);
- response->mutable_status()->CopyFrom(google::rpc::Status{});
+ response->set_status(ServeCommitTreeStatus::OK);
return ::grpc::Status::OK;
}
}
// commit not found
- google::rpc::Status status;
- status.set_code(grpc::StatusCode::NOT_FOUND);
- response->mutable_status()->CopyFrom(status);
+ response->set_status(ServeCommitTreeStatus::NOT_FOUND);
return ::grpc::Status::OK;
}
diff --git a/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp b/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp
index 80d09647..826d4031 100644
--- a/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp
+++ b/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp
@@ -21,7 +21,10 @@
#include <string>
#include <vector>
+#include "gsl/gsl"
#include "justbuild/just_serve/just_serve.grpc.pb.h"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/remote/config.hpp"
#include "src/buildtool/logging/logger.hpp"
class TargetLevelCacheService final
@@ -46,6 +49,17 @@ class TargetLevelCacheService final
private:
std::shared_ptr<Logger> logger_{std::make_shared<Logger>("serve-service")};
+ // remote execution endpoint
+ gsl::not_null<IExecutionApi::Ptr> const remote_api_{
+ CreateExecutionApi(RemoteExecutionConfig::RemoteAddress())};
+ // local api
+ gsl::not_null<IExecutionApi::Ptr> const local_api_{
+ CreateExecutionApi(std::nullopt)};
+
+ [[nodiscard]] static auto CreateExecutionApi(
+ std::optional<RemoteExecutionConfig::ServerAddress> const& address)
+ -> gsl::not_null<IExecutionApi::Ptr>;
+
[[nodiscard]] static auto GetTreeFromCommit(
std::filesystem::path const& repo_path,
std::string const& commit,
diff --git a/src/other_tools/just_mr/cli.hpp b/src/other_tools/just_mr/cli.hpp
index 22c401ba..328ab666 100644
--- a/src/other_tools/just_mr/cli.hpp
+++ b/src/other_tools/just_mr/cli.hpp
@@ -48,6 +48,7 @@ struct MultiRepoCommonArguments {
std::optional<std::string> remote_execution_address;
std::optional<bool> compatible{std::nullopt};
std::optional<std::string> remote_serve_address;
+ bool fetch_absent{false};
};
struct MultiRepoLogArguments {
@@ -192,6 +193,11 @@ static inline void SetupMultiRepoCommonArguments(
clargs->remote_serve_address,
"Address of a remote 'just serve' service.")
->type_name("NAME:PORT");
+ app->add_flag("--fetch-absent",
+ clargs->fetch_absent,
+ "Do not produce absent roots. For Git repositories, try to "
+ "fetch served commit trees from the remote execution "
+ "endpoint before reverting to the network.");
}
static inline auto SetupMultiRepoLogArguments(
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index ed09bb61..54295c6b 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -116,10 +116,14 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
auto commit_git_map =
CreateCommitGitMap(&critical_git_op_map,
+ &import_to_git_map,
common_args.just_mr_paths,
common_args.git_path->string(),
*common_args.local_launcher,
serve_api ? &(*serve_api) : nullptr,
+ local_api ? &(*local_api) : nullptr,
+ remote_api ? &(*remote_api) : nullptr,
+ common_args.fetch_absent,
common_args.jobs);
auto content_git_map = CreateContentGitMap(&content_cas_map,
&import_to_git_map,
@@ -147,6 +151,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
&fpath_git_map,
&distdir_git_map,
&tree_id_git_map,
+ common_args.fetch_absent,
common_args.jobs);
// set up progress observer
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index 5aefe3b1..f51e48ec 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -150,6 +150,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
std::string const& repo_name,
std::string const& repo_type,
gsl::not_null<ContentGitMap*> const& content_git_map,
+ bool fetch_absent,
gsl::not_null<TaskSystem*> const& ts,
ReposToSetupMap::SetterPtr const& setter,
ReposToSetupMap::LoggerPtr const& logger) {
@@ -224,7 +225,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
.repo_type = repo_type,
.subdir = subdir.empty() ? "." : subdir.string(),
.pragma_special = pragma_special_value,
- .absent = pragma_absent_value};
+ .absent = not fetch_absent and pragma_absent_value};
// get the WS root as git tree
content_git_map->ConsumeAfterKeysReady(
ts,
@@ -258,6 +259,7 @@ void FileCheckout(ExpressionPtr const& repo_desc,
ExpressionPtr&& repos,
std::string const& repo_name,
gsl::not_null<FilePathGitMap*> const& fpath_git_map,
+ bool fetch_absent,
gsl::not_null<TaskSystem*> const& ts,
ReposToSetupMap::SetterPtr const& setter,
ReposToSetupMap::LoggerPtr const& logger) {
@@ -305,9 +307,10 @@ void FileCheckout(ExpressionPtr const& repo_desc,
pragma_absent->get()->IsBool() and
pragma_absent->get()->Bool();
// get the WS root as git tree
- FpathInfo fpath_info = {.fpath = fpath,
- .pragma_special = pragma_special_value,
- .absent = pragma_absent_value};
+ FpathInfo fpath_info = {
+ .fpath = fpath,
+ .pragma_special = pragma_special_value,
+ .absent = not fetch_absent and pragma_absent_value};
fpath_git_map->ConsumeAfterKeysReady(
ts,
{std::move(fpath_info)},
@@ -349,6 +352,7 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
ExpressionPtr&& repos,
std::string const& repo_name,
gsl::not_null<DistdirGitMap*> const& distdir_git_map,
+ bool fetch_absent,
gsl::not_null<TaskSystem*> const& ts,
ReposToSetupMap::SetterPtr const& setter,
ReposToSetupMap::LoggerPtr const& logger) {
@@ -509,11 +513,12 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
HashFunction::ComputeBlobHash(nlohmann::json(*distdir_content).dump())
.HexString();
// get the WS root as git tree
- DistdirInfo distdir_info = {.content_id = distdir_content_id,
- .content_list = distdir_content,
- .repos_to_fetch = dist_repos_to_fetch,
- .origin = repo_name,
- .absent = pragma_absent_value};
+ DistdirInfo distdir_info = {
+ .content_id = distdir_content_id,
+ .content_list = distdir_content,
+ .repos_to_fetch = dist_repos_to_fetch,
+ .origin = repo_name,
+ .absent = not fetch_absent and pragma_absent_value};
distdir_git_map->ConsumeAfterKeysReady(
ts,
{std::move(distdir_info)},
@@ -545,6 +550,7 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc,
ExpressionPtr&& repos,
std::string const& repo_name,
gsl::not_null<TreeIdGitMap*> const& tree_id_git_map,
+ bool fetch_absent,
gsl::not_null<TaskSystem*> const& ts,
ReposToSetupMap::SetterPtr const& setter,
ReposToSetupMap::LoggerPtr const& logger) {
@@ -628,7 +634,7 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc,
.env_vars = std::move(env),
.command = std::move(cmd),
.ignore_special = pragma_special_value == PragmaSpecial::Ignore,
- .absent = pragma_absent_value};
+ .absent = not fetch_absent and pragma_absent_value};
// get the WS root as git tree
tree_id_git_map->ConsumeAfterKeysReady(
ts,
@@ -665,6 +671,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
gsl::not_null<FilePathGitMap*> const& fpath_git_map,
gsl::not_null<DistdirGitMap*> const& distdir_git_map,
gsl::not_null<TreeIdGitMap*> const& tree_id_git_map,
+ bool fetch_absent,
std::size_t jobs) -> ReposToSetupMap {
auto setup_repo = [config,
main,
@@ -673,11 +680,12 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
content_git_map,
fpath_git_map,
distdir_git_map,
- tree_id_git_map](auto ts,
- auto setter,
- auto logger,
- auto /* unused */,
- auto const& key) {
+ tree_id_git_map,
+ fetch_absent](auto ts,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
auto repos = (*config)["repositories"];
if (main && (key == *main) && interactive) {
// no repository checkout required
@@ -765,6 +773,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
key,
repo_type_str,
content_git_map,
+ fetch_absent,
ts,
setter,
wrapped_logger);
@@ -775,6 +784,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
std::move(repos),
key,
fpath_git_map,
+ fetch_absent,
ts,
setter,
wrapped_logger);
@@ -785,6 +795,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
std::move(repos),
key,
distdir_git_map,
+ fetch_absent,
ts,
setter,
wrapped_logger);
@@ -795,6 +806,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
std::move(repos),
key,
tree_id_git_map,
+ fetch_absent,
ts,
setter,
wrapped_logger);
diff --git a/src/other_tools/repo_map/repos_to_setup_map.hpp b/src/other_tools/repo_map/repos_to_setup_map.hpp
index 3306800d..1fbf9687 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.hpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.hpp
@@ -34,6 +34,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
gsl::not_null<FilePathGitMap*> const& fpath_git_map,
gsl::not_null<DistdirGitMap*> const& distdir_git_map,
gsl::not_null<TreeIdGitMap*> const& tree_id_git_map,
+ bool fetch_absent,
std::size_t jobs) -> ReposToSetupMap;
#endif // INCLUDED_SRC_OTHER_TOOLS_REPO_MAP_REPOS_TO_SETUP_MAP_HPP
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index c7089bfc..b2cd1dfb 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -30,8 +30,10 @@
, "srcs": ["commit_git_map.cpp"]
, "deps":
[ ["src/buildtool/serve_api/remote", "serve_api"]
+ , ["src/buildtool/execution_api/common", "common"]
, ["src/other_tools/just_mr", "utils"]
, ["src/other_tools/ops_maps", "critical_git_op_map"]
+ , ["src/other_tools/ops_maps", "import_to_git_map"]
, ["src/utils/cpp", "hash_combine"]
, ["@", "json", "", "json"]
]
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index 64d686c1..883fe697 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -37,9 +37,13 @@ void EnsureCommit(GitRepoInfo const& repo_info,
std::filesystem::path const& repo_root,
GitCASPtr const& git_cas,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi* serve_api,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
+ bool fetch_absent,
gsl::not_null<TaskSystem*> const& ts,
CommitGitMap::SetterPtr const& ws_setter,
CommitGitMap::LoggerPtr const& logger) {
@@ -69,31 +73,146 @@ void EnsureCommit(GitRepoInfo const& repo_info,
if (repo_info.absent) {
if (serve_api != nullptr) {
if (auto tree_id = serve_api->RetrieveTreeFromCommit(
- repo_info.hash, repo_info.subdir)) {
- // set the workspace root as absent
- JustMRProgress::Instance().TaskTracker().Stop(
- repo_info.origin);
- (*ws_setter)(std::pair(
- nlohmann::json::array(
- {repo_info.ignore_special
- ? FileRoot::kGitTreeIgnoreSpecialMarker
- : FileRoot::kGitTreeMarker,
- *tree_id}),
- false));
- return;
+ repo_info.hash, repo_info.subdir, fetch_absent)) {
+ if (not fetch_absent) {
+ // set the workspace root as absent
+ JustMRProgress::Instance().TaskTracker().Stop(
+ repo_info.origin);
+ (*ws_setter)(std::pair(
+ nlohmann::json::array(
+ {repo_info.ignore_special
+ ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ *tree_id}),
+ false));
+ return;
+ }
+ // verify if we know the tree already locally
+ auto wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [logger, tree = *tree_id](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While verifying presence of "
+ "tree {}:\n{}",
+ tree,
+ msg),
+ fatal);
+ });
+ auto tree_present =
+ git_repo->CheckTreeExists(*tree_id, wrapped_logger);
+ if (not tree_present) {
+ return;
+ }
+ if (*tree_present) {
+ JustMRProgress::Instance().TaskTracker().Stop(
+ repo_info.origin);
+ (*ws_setter)(std::pair(
+ nlohmann::json::array(
+ {repo_info.ignore_special
+ ? FileRoot::kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ *tree_id,
+ StorageConfig::GitRoot().string()}),
+ false));
+ return;
+ }
+ // try to get the tree from remote execution endpoint
+ auto digest = ArtifactDigest{*tree_id, 0, /*is_tree=*/true};
+ if (remote_api != nullptr and local_api != nullptr and
+ remote_api->RetrieveToCas(
+ {Artifact::ObjectInfo{.digest = digest,
+ .type = ObjectType::Tree}},
+ local_api)) {
+ JustMRProgress::Instance().TaskTracker().Stop(
+ repo_info.origin);
+ // Move tree from CAS to local git storage
+ auto tmp_dir = JustMR::Utils::CreateTypedTmpDir(
+ "fetch-absent-root");
+ if (not tmp_dir) {
+ (*logger)(
+ fmt::format(
+ "Failed to create tmp directory after "
+ "fetching tree {} for absent commit {}",
+ repo_info.hash,
+ *tree_id),
+ true);
+ return;
+ }
+ if (not local_api->RetrieveToPaths(
+ {Artifact::ObjectInfo{
+ .digest = digest,
+ .type = ObjectType::Tree}},
+ {tmp_dir->GetPath()})) {
+ (*logger)(fmt::format("Failed to copy fetchted "
+ "tree {} to {}",
+ *tree_id,
+ tmp_dir->GetPath().string()),
+ true);
+ return;
+ }
+ CommitInfo c_info{tmp_dir->GetPath(), "tree", *tree_id};
+ import_to_git_map->ConsumeAfterKeysReady(
+ ts,
+ {std::move(c_info)},
+ [tmp_dir, // keep tmp_dir alive
+ repo_info,
+ tree_id,
+ logger,
+ ws_setter](auto const& values) {
+ if (not values[0]->second) {
+ (*logger)("Importing to git failed",
+ /*fatal=*/true);
+ return;
+ }
+ // set the workspace root as present
+ (*ws_setter)(std::pair(
+ nlohmann::json::array(
+ {repo_info.ignore_special
+ ? FileRoot::
+ kGitTreeIgnoreSpecialMarker
+ : FileRoot::kGitTreeMarker,
+ *tree_id,
+ StorageConfig::GitRoot().string()}),
+ false));
+ },
+ [logger, tmp_dir, tree_id](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format("While moving tree {} from {} "
+ "to local git:\n{}",
+ *tree_id,
+ tmp_dir->GetPath().string(),
+ msg),
+ fatal);
+ });
+
+ return;
+ }
+ // just serve should have made the tree available in the
+ // remote CAS, so log this attempt and revert to network
+ (*logger)(fmt::format("Tree {} marked as served, but not "
+ "found on remote",
+ *tree_id),
+ /*fatal=*/false);
+ }
+ else {
+ // give warning
+ (*logger)(
+ fmt::format("Tree at subdir {} for commit {} could "
+ "not be served",
+ repo_info.subdir,
+ repo_info.hash),
+ /*fatal=*/false);
}
- // give warning
- (*logger)(fmt::format("Tree at subdir {} for commit {} could "
- "not be served",
- repo_info.subdir,
- repo_info.hash),
- /*fatal=*/false);
}
else {
- // give warning
- (*logger)(
- "Absent root requested, but no serve endpoint provided",
- /*fatal=*/false);
+ if (not fetch_absent) {
+ // give warning
+ (*logger)(
+ "Absent root requested, but no serve endpoint provided",
+ /*fatal=*/false);
+ }
}
}
// default to fetching it from network
@@ -151,7 +270,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
- [git_cas, repo_info, repo_root, ws_setter, logger](
+ [git_cas, repo_info, repo_root, fetch_absent, ws_setter, logger](
auto const& values) {
GitOpValue op_result = *values[0];
// check flag
@@ -190,7 +309,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
*subtree});
- if (not repo_info.absent) {
+ if (fetch_absent or not repo_info.absent) {
root.emplace_back(repo_root);
}
(*ws_setter)(std::pair(std::move(root), false));
@@ -222,7 +341,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
{repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
*subtree});
- if (not repo_info.absent) {
+ if (fetch_absent or not repo_info.absent) {
root.emplace_back(repo_root);
}
(*ws_setter)(std::pair(std::move(root), true));
@@ -234,20 +353,28 @@ void EnsureCommit(GitRepoInfo const& repo_info,
/// \brief Create a CommitGitMap object
auto CreateCommitGitMap(
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
JustMR::PathsPtr const& just_mr_paths,
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi* serve_api,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
+ bool fetch_absent,
std::size_t jobs) -> CommitGitMap {
auto commit_to_git = [critical_git_op_map,
+ import_to_git_map,
just_mr_paths,
git_bin,
launcher,
- serve_api](auto ts,
- auto setter,
- auto logger,
- auto /* unused */,
- auto const& key) {
+ serve_api,
+ local_api,
+ remote_api,
+ fetch_absent](auto ts,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
// get root for repo (making sure that if repo is a path, it is
// absolute)
std::string fetch_repo = key.repo_url;
@@ -275,9 +402,13 @@ auto CreateCommitGitMap(
[key,
repo_root,
critical_git_op_map,
+ import_to_git_map,
git_bin,
launcher,
serve_api,
+ local_api,
+ remote_api,
+ fetch_absent,
ts,
setter,
logger](auto const& values) {
@@ -302,9 +433,13 @@ auto CreateCommitGitMap(
repo_root,
op_result.git_cas,
critical_git_op_map,
+ import_to_git_map,
git_bin,
launcher,
serve_api,
+ local_api,
+ remote_api,
+ fetch_absent,
ts,
setter,
wrapped_logger);
diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp
index 00a607cb..4f59a56f 100644
--- a/src/other_tools/root_maps/commit_git_map.hpp
+++ b/src/other_tools/root_maps/commit_git_map.hpp
@@ -19,9 +19,11 @@
#include <utility>
#include "nlohmann/json.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/serve_api/remote/serve_api.hpp"
#include "src/other_tools/just_mr/utils.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
+#include "src/other_tools/ops_maps/import_to_git_map.hpp"
#include "src/utils/cpp/hash_combine.hpp"
struct GitRepoInfo {
@@ -66,10 +68,14 @@ using CommitGitMap =
[[nodiscard]] auto CreateCommitGitMap(
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
JustMR::PathsPtr const& just_mr_paths,
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi* serve_api,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
+ bool fetch_absent,
std::size_t jobs) -> CommitGitMap;
#endif // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 00:50:19 +0000