summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Cristian Sarbu <paul.cristian.sarbu@huawei.com>2024-10-21 12:51:08 +0200
committerPaul Cristian Sarbu <paul.cristian.sarbu@huawei.com>2024-10-25 13:00:43 +0200
commit6242ef6fff52116398913a40634a71292616c126 (patch)
tree56da133aa074278c71f9489fd637d2aa5c4d1800
parent0675a0daf093442860d117afb060e5456e37c7e2 (diff)
downloadjustbuild-6242ef6fff52116398913a40634a71292616c126.tar.gz
just-mr and SourceTree: Use new Git execution api instance
In just-mr: to instantiate the new Git api instance, both storage configs, as well as the compatible storage, need to be passed to the maps. While there, use more explicit naming schemes for the storage and CAS instances used. In serve: also acquire gc locks for the local storages when needed to instantiate the new Git api, which now has access to the CAS. In all these instances we also pass, as needed, the local api, which currently still operates only in native mode. This makes no difference currently, but will ensure less changes needed when the future compatible-aware local api will be used instead.
-rw-r--r--src/buildtool/serve_api/serve_service/TARGETS2
-rw-r--r--src/buildtool/serve_api/serve_service/source_tree.cpp29
-rw-r--r--src/other_tools/just_mr/fetch.cpp26
-rw-r--r--src/other_tools/just_mr/setup.cpp138
-rw-r--r--src/other_tools/ops_maps/TARGETS3
-rw-r--r--src/other_tools/ops_maps/git_tree_fetch_map.cpp218
-rw-r--r--src/other_tools/ops_maps/git_tree_fetch_map.hpp5
-rw-r--r--src/other_tools/root_maps/TARGETS9
-rw-r--r--src/other_tools/root_maps/commit_git_map.cpp277
-rw-r--r--src/other_tools/root_maps/commit_git_map.hpp5
-rw-r--r--src/other_tools/root_maps/content_git_map.cpp350
-rw-r--r--src/other_tools/root_maps/content_git_map.hpp7
-rw-r--r--src/other_tools/root_maps/distdir_git_map.cpp82
-rw-r--r--src/other_tools/root_maps/distdir_git_map.hpp6
-rw-r--r--src/other_tools/root_maps/fpath_git_map.cpp160
-rw-r--r--src/other_tools/root_maps/fpath_git_map.hpp6
-rw-r--r--src/other_tools/root_maps/root_utils.cpp25
-rw-r--r--src/other_tools/root_maps/root_utils.hpp19
-rw-r--r--src/other_tools/root_maps/tree_id_git_map.cpp113
-rw-r--r--src/other_tools/root_maps/tree_id_git_map.hpp5
20 files changed, 960 insertions, 525 deletions
diff --git a/src/buildtool/serve_api/serve_service/TARGETS b/src/buildtool/serve_api/serve_service/TARGETS
index 6b5afe1c..954a487a 100644
--- a/src/buildtool/serve_api/serve_service/TARGETS
+++ b/src/buildtool/serve_api/serve_service/TARGETS
@@ -33,7 +33,7 @@
, ["src/buildtool/common", "common"]
, ["src/buildtool/common", "protocol_traits"]
, ["src/buildtool/crypto", "hash_function"]
- , ["src/buildtool/execution_api/git", "git"]
+ , ["src/buildtool/execution_api/serve", "mr_git_api"]
, ["src/buildtool/file_system", "git_repo"]
, ["src/buildtool/logging", "log_level"]
, ["src/buildtool/multithreading", "async_map_utils"]
diff --git a/src/buildtool/serve_api/serve_service/source_tree.cpp b/src/buildtool/serve_api/serve_service/source_tree.cpp
index 3099de77..8bc06c39 100644
--- a/src/buildtool/serve_api/serve_service/source_tree.cpp
+++ b/src/buildtool/serve_api/serve_service/source_tree.cpp
@@ -26,7 +26,7 @@
#include "src/buildtool/common/artifact_digest_factory.hpp"
#include "src/buildtool/common/protocol_traits.hpp"
#include "src/buildtool/crypto/hash_function.hpp"
-#include "src/buildtool/execution_api/git/git_api.hpp"
+#include "src/buildtool/execution_api/serve/mr_git_api.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/file_system/git_repo.hpp"
#include "src/buildtool/logging/log_level.hpp"
@@ -288,6 +288,23 @@ auto SourceTreeService::SyncGitEntryToCas(
std::string const& object_hash,
std::filesystem::path const& repo_path) const noexcept
-> std::remove_cvref_t<decltype(TResponse::OK)> {
+ // get gc locks for the local storages
+ auto native_lock =
+ GarbageCollector::SharedLock(*native_context_->storage_config);
+ if (not native_lock) {
+ logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock");
+ return TResponse::INTERNAL_ERROR;
+ }
+ std::optional<LockFile> compat_lock = std::nullopt;
+ if (compat_context_ != nullptr) {
+ compat_lock =
+ GarbageCollector::SharedLock(*compat_context_->storage_config);
+ if (not compat_lock) {
+ logger_->Emit(LogLevel::Error, "Could not acquire gc SharedLock");
+ return TResponse::INTERNAL_ERROR;
+ }
+ }
+
auto const hash_type =
native_context_->storage_config->hash_function.GetType();
if (IsTreeObject(kType) and not ProtocolTraits::IsTreeAllowed(hash_type)) {
@@ -308,11 +325,17 @@ auto SourceTreeService::SyncGitEntryToCas(
auto const digest = ArtifactDigestFactory::Create(
hash_type, object_hash, 0, IsTreeObject(kType));
if (not digest) {
- logger_->Emit(LogLevel::Error, "{}", digest.error());
+ logger_->Emit(LogLevel::Error, "SyncGitEntryToCas: {}", digest.error());
return TResponse::INTERNAL_ERROR;
}
- auto git_api = GitApi{&repo};
+ auto const is_compat = compat_context_ != nullptr;
+ auto git_api =
+ MRGitApi{&repo,
+ native_context_->storage_config,
+ is_compat ? &*compat_context_->storage_config : nullptr,
+ is_compat ? &*compat_context_->storage : nullptr,
+ is_compat ? &*apis_.local : nullptr};
if (not git_api.RetrieveToCas(
{Artifact::ObjectInfo{.digest = *digest, .type = kType}},
*apis_.remote)) {
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index 04c6c013..bdca1331 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -491,18 +491,20 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
&native_storage_config,
common_args.jobs);
- auto git_tree_fetch_map =
- CreateGitTreeFetchMap(&critical_git_op_map,
- &import_to_git_map,
- common_args.git_path->string(),
- *common_args.local_launcher,
- serve ? &*serve : nullptr,
- &native_storage_config,
- &(*apis.local),
- has_remote_api ? &*apis.remote : nullptr,
- fetch_args.backup_to_remote,
- &progress,
- common_args.jobs);
+ auto git_tree_fetch_map = CreateGitTreeFetchMap(
+ &critical_git_op_map,
+ &import_to_git_map,
+ common_args.git_path->string(),
+ *common_args.local_launcher,
+ serve ? &*serve : nullptr,
+ &native_storage_config,
+ compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
+ compat_storage != nullptr ? &*compat_storage : nullptr,
+ &(*apis.local),
+ has_remote_api ? &*apis.remote : nullptr,
+ fetch_args.backup_to_remote,
+ &progress,
+ common_args.jobs);
// set up progress observer
std::atomic<bool> done{false};
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 012629c2..8b464fce 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -293,51 +293,58 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
&native_storage_config,
common_args.jobs);
- auto git_tree_fetch_map =
- CreateGitTreeFetchMap(&critical_git_op_map,
- &import_to_git_map,
- common_args.git_path->string(),
- *common_args.local_launcher,
- serve ? &*serve : nullptr,
- &native_storage_config,
- &(*apis.local),
- has_remote_api ? &*apis.remote : nullptr,
- false, /* backup_to_remote */
- &progress,
- common_args.jobs);
+ auto git_tree_fetch_map = CreateGitTreeFetchMap(
+ &critical_git_op_map,
+ &import_to_git_map,
+ common_args.git_path->string(),
+ *common_args.local_launcher,
+ serve ? &*serve : nullptr,
+ &native_storage_config,
+ compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
+ compat_storage != nullptr ? &*compat_storage : nullptr,
+ &(*apis.local),
+ has_remote_api ? &*apis.remote : nullptr,
+ false, /* backup_to_remote */
+ &progress,
+ common_args.jobs);
auto resolve_symlinks_map = CreateResolveSymlinksMap();
- auto commit_git_map =
- CreateCommitGitMap(&critical_git_op_map,
- &import_to_git_map,
- common_args.just_mr_paths,
- common_args.alternative_mirrors,
- common_args.git_path->string(),
- *common_args.local_launcher,
- serve ? &*serve : nullptr,
- &native_storage_config,
- &(*apis.local),
- has_remote_api ? &*apis.remote : nullptr,
- common_args.fetch_absent,
- &progress,
- common_args.jobs);
-
- auto content_git_map =
- CreateContentGitMap(&content_cas_map,
- &import_to_git_map,
- common_args.just_mr_paths,
- common_args.alternative_mirrors,
- common_args.ca_info,
- &resolve_symlinks_map,
- &critical_git_op_map,
- serve ? &*serve : nullptr,
- &native_storage_config,
- &native_storage,
- has_remote_api ? &*apis.remote : nullptr,
- common_args.fetch_absent,
- &progress,
- common_args.jobs);
+ auto commit_git_map = CreateCommitGitMap(
+ &critical_git_op_map,
+ &import_to_git_map,
+ common_args.just_mr_paths,
+ common_args.alternative_mirrors,
+ common_args.git_path->string(),
+ *common_args.local_launcher,
+ serve ? &*serve : nullptr,
+ &native_storage_config,
+ compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
+ compat_storage != nullptr ? &*compat_storage : nullptr,
+ &(*apis.local),
+ has_remote_api ? &*apis.remote : nullptr,
+ common_args.fetch_absent,
+ &progress,
+ common_args.jobs);
+
+ auto content_git_map = CreateContentGitMap(
+ &content_cas_map,
+ &import_to_git_map,
+ common_args.just_mr_paths,
+ common_args.alternative_mirrors,
+ common_args.ca_info,
+ &resolve_symlinks_map,
+ &critical_git_op_map,
+ serve ? &*serve : nullptr,
+ &native_storage_config,
+ compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
+ &native_storage,
+ compat_storage != nullptr ? &*compat_storage : nullptr,
+ has_remote_api ? &*apis.local : nullptr, // only needed if remote given
+ has_remote_api ? &*apis.remote : nullptr,
+ common_args.fetch_absent,
+ &progress,
+ common_args.jobs);
auto foreign_file_git_map =
CreateForeignFileGitMap(&content_cas_map,
@@ -355,33 +362,40 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
&resolve_symlinks_map,
serve ? &*serve : nullptr,
&native_storage_config,
+ compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
+ compat_storage != nullptr ? &*compat_storage : nullptr,
+ has_remote_api ? &*apis.local : nullptr, // only needed if remote given
has_remote_api ? &*apis.remote : nullptr,
common_args.jobs,
multi_repo_tool_name,
common_args.just_path ? common_args.just_path->string()
: kDefaultJustPath);
- auto distdir_git_map =
- CreateDistdirGitMap(&content_cas_map,
- &import_to_git_map,
- &critical_git_op_map,
- serve ? &*serve : nullptr,
- &native_storage_config,
- &native_storage,
- &(*apis.local),
- has_remote_api ? &*apis.remote : nullptr,
- common_args.jobs);
+ auto distdir_git_map = CreateDistdirGitMap(
+ &content_cas_map,
+ &import_to_git_map,
+ &critical_git_op_map,
+ serve ? &*serve : nullptr,
+ &native_storage_config,
+ compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
+ &native_storage,
+ compat_storage != nullptr ? &*compat_storage : nullptr,
+ &(*apis.local),
+ has_remote_api ? &*apis.remote : nullptr,
+ common_args.jobs);
- auto tree_id_git_map =
- CreateTreeIdGitMap(&git_tree_fetch_map,
- &critical_git_op_map,
- &import_to_git_map,
- common_args.fetch_absent,
- serve ? &*serve : nullptr,
- &native_storage_config,
- &(*apis.local),
- has_remote_api ? &*apis.remote : nullptr,
- common_args.jobs);
+ auto tree_id_git_map = CreateTreeIdGitMap(
+ &git_tree_fetch_map,
+ &critical_git_op_map,
+ &import_to_git_map,
+ common_args.fetch_absent,
+ serve ? &*serve : nullptr,
+ &native_storage_config,
+ compat_storage_config != nullptr ? &*compat_storage_config : nullptr,
+ compat_storage != nullptr ? &*compat_storage : nullptr,
+ &(*apis.local),
+ has_remote_api ? &*apis.remote : nullptr,
+ common_args.jobs);
auto repos_to_setup_map = CreateReposToSetupMap(config,
main,
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS
index da8be93f..96d9db57 100644
--- a/src/other_tools/ops_maps/TARGETS
+++ b/src/other_tools/ops_maps/TARGETS
@@ -115,6 +115,7 @@
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/serve_api/remote", "serve_api"]
, ["src/buildtool/storage", "config"]
+ , ["src/buildtool/storage", "storage"]
, ["src/other_tools/just_mr/progress_reporting", "progress"]
, ["src/other_tools/ops_maps", "critical_git_op_map"]
, ["src/other_tools/ops_maps", "import_to_git_map"]
@@ -125,7 +126,7 @@
, ["src/buildtool/common", "common"]
, ["src/buildtool/common", "config"]
, ["src/buildtool/common", "protocol_traits"]
- , ["src/buildtool/execution_api/git", "git"]
+ , ["src/buildtool/execution_api/serve", "mr_git_api"]
, ["src/buildtool/file_system", "file_system_manager"]
, ["src/buildtool/multithreading", "task_system"]
, ["src/buildtool/system", "system_command"]
diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.cpp b/src/other_tools/ops_maps/git_tree_fetch_map.cpp
index 1bfc25ad..6f1b5556 100644
--- a/src/other_tools/ops_maps/git_tree_fetch_map.cpp
+++ b/src/other_tools/ops_maps/git_tree_fetch_map.cpp
@@ -23,7 +23,7 @@
#include "src/buildtool/common/protocol_traits.hpp"
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/execution_api/common/execution_common.hpp"
-#include "src/buildtool/execution_api/git/git_api.hpp"
+#include "src/buildtool/execution_api/serve/mr_git_api.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/multithreading/task_system.hpp"
#include "src/buildtool/system/system_command.hpp"
@@ -32,13 +32,21 @@
namespace {
void BackupToRemote(ArtifactDigest const& digest,
- StorageConfig const& storage_config,
+ StorageConfig const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const& remote_api,
GitTreeFetchMap::LoggerPtr const& logger) {
// try to back up to remote CAS
auto repo = RepositoryConfig{};
- if (repo.SetGitCAS(storage_config.GitRoot())) {
- auto git_api = GitApi{&repo};
+ if (repo.SetGitCAS(native_storage_config.GitRoot())) {
+ auto git_api =
+ MRGitApi{&repo,
+ &native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ compat_storage_config != nullptr ? &*local_api : nullptr};
if (not git_api.RetrieveToCas(
{Artifact::ObjectInfo{.digest = digest,
.type = ObjectType::Tree}},
@@ -53,24 +61,28 @@ void BackupToRemote(ArtifactDigest const& digest,
else {
// give a warning
(*logger)(fmt::format("Failed to SetGitCAS at {}",
- storage_config.GitRoot().string()),
+ native_storage_config.GitRoot().string()),
/*fatal=*/false);
}
}
/// \brief Moves the root tree from local CAS to the Git cache and sets the
/// root.
-void MoveCASTreeToGit(ArtifactDigest const& digest,
- gsl::not_null<ImportToGitMap*> const& import_to_git_map,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<IExecutionApi const*> const& local_api,
- IExecutionApi const* remote_api,
- bool backup_to_remote,
- gsl::not_null<TaskSystem*> const& ts,
- GitTreeFetchMap::SetterPtr const& setter,
- GitTreeFetchMap::LoggerPtr const& logger) {
+void MoveCASTreeToGit(
+ ArtifactDigest const& digest,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ gsl::not_null<IExecutionApi const*> const& local_api,
+ IExecutionApi const* remote_api,
+ bool backup_to_remote,
+ gsl::not_null<TaskSystem*> const& ts,
+ GitTreeFetchMap::SetterPtr const& setter,
+ GitTreeFetchMap::LoggerPtr const& logger) {
// Move tree from CAS to local Git storage
- auto tmp_dir = storage_config->CreateTypedTmpDir("fetch-remote-git-tree");
+ auto tmp_dir =
+ native_storage_config->CreateTypedTmpDir("fetch-remote-git-tree");
if (not tmp_dir) {
(*logger)(fmt::format("Failed to create tmp directory for copying "
"git-tree {} from remote CAS",
@@ -93,7 +105,10 @@ void MoveCASTreeToGit(ArtifactDigest const& digest,
{std::move(c_info)},
[tmp_dir, // keep tmp_dir alive
digest,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
backup_to_remote,
setter,
@@ -105,7 +120,13 @@ void MoveCASTreeToGit(ArtifactDigest const& digest,
}
// backup to remote if needed and in compatibility mode
if (backup_to_remote and remote_api != nullptr) {
- BackupToRemote(digest, *storage_config, *remote_api, logger);
+ BackupToRemote(digest,
+ *native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
+ *remote_api,
+ logger);
}
(*setter)(false /*no cache hit*/);
},
@@ -119,15 +140,19 @@ void MoveCASTreeToGit(ArtifactDigest const& digest,
});
}
-void TagAndSetRoot(ArtifactDigest const& digest,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
- IExecutionApi const* remote_api,
- bool backup_to_remote,
- gsl::not_null<TaskSystem*> const& ts,
- GitTreeFetchMap::SetterPtr const& setter,
- GitTreeFetchMap::LoggerPtr const& logger) {
- auto repo = storage_config->GitRoot();
+void TagAndSetRoot(
+ ArtifactDigest const& digest,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<IExecutionApi const*> const& local_api,
+ IExecutionApi const* remote_api,
+ bool backup_to_remote,
+ gsl::not_null<TaskSystem*> const& ts,
+ GitTreeFetchMap::SetterPtr const& setter,
+ GitTreeFetchMap::LoggerPtr const& logger) {
+ auto repo = native_storage_config->GitRoot();
GitOpKey op_key = {.params =
{
repo, // target_path
@@ -138,8 +163,15 @@ void TagAndSetRoot(ArtifactDigest const& digest,
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
- [digest, backup_to_remote, storage_config, remote_api, logger, setter](
- auto const& values) {
+ [digest,
+ backup_to_remote,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
+ remote_api,
+ logger,
+ setter](auto const& values) {
GitOpValue op_result = *values[0];
if (not op_result.result) {
(*logger)("Tree tagging failed",
@@ -148,7 +180,13 @@ void TagAndSetRoot(ArtifactDigest const& digest,
}
// backup to remote if needed and in compatibility mode
if (backup_to_remote and remote_api != nullptr) {
- BackupToRemote(digest, *storage_config, *remote_api, logger);
+ BackupToRemote(digest,
+ *native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
+ *remote_api,
+ logger);
}
(*setter)(false /*no cache hit*/);
},
@@ -165,15 +203,18 @@ void TagAndSetRoot(ArtifactDigest const& digest,
void TakeTreeFromOlderGeneration(
std::size_t generation,
ArtifactDigest const& digest,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
GitCASPtr const& git_cas,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
bool backup_to_remote,
gsl::not_null<TaskSystem*> const& ts,
GitTreeFetchMap::SetterPtr const& setter,
GitTreeFetchMap::LoggerPtr const& logger) {
- auto source = storage_config->GitGenerationRoot(generation);
+ auto source = native_storage_config->GitGenerationRoot(generation);
GitOpKey op_key = {.params =
{
source, // target_path
@@ -187,13 +228,16 @@ void TakeTreeFromOlderGeneration(
[digest,
git_cas,
critical_git_op_map,
+ local_api,
remote_api,
backup_to_remote,
ts,
setter,
logger,
source,
- storage_config](auto const& values) {
+ native_storage_config,
+ compat_storage_config,
+ compat_storage](auto const& values) {
GitOpValue op_result = *values[0];
if (not op_result.result) {
(*logger)("Tree tagging failed", /*fatal=*/true);
@@ -214,12 +258,15 @@ void TakeTreeFromOlderGeneration(
fatal);
});
if (not git_repo->LocalFetchViaTmpRepo(
- *storage_config, source, tag, fetch_logger)) {
+ *native_storage_config, source, tag, fetch_logger)) {
return;
}
TagAndSetRoot(digest,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
critical_git_op_map,
+ local_api,
remote_api,
backup_to_remote,
ts,
@@ -244,7 +291,9 @@ auto CreateGitTreeFetchMap(
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
bool backup_to_remote,
@@ -255,7 +304,9 @@ auto CreateGitTreeFetchMap(
git_bin,
launcher,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
local_api,
remote_api,
backup_to_remote,
@@ -266,15 +317,16 @@ auto CreateGitTreeFetchMap(
auto const& key) {
// check whether tree exists already in Git cache;
// ensure Git cache exists
- GitOpKey op_key = {.params =
- {
- storage_config->GitRoot(), // target_path
- "", // git_hash
- std::nullopt, // message
- std::nullopt, // source_path
- true // init_bare
- },
- .op_type = GitOpType::ENSURE_INIT};
+ GitOpKey op_key = {
+ .params =
+ {
+ native_storage_config->GitRoot(), // target_path
+ "", // git_hash
+ std::nullopt, // message
+ std::nullopt, // source_path
+ true // init_bare
+ },
+ .op_type = GitOpType::ENSURE_INIT};
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
@@ -283,7 +335,9 @@ auto CreateGitTreeFetchMap(
git_bin,
launcher,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
local_api,
remote_api,
backup_to_remote,
@@ -303,9 +357,10 @@ auto CreateGitTreeFetchMap(
auto git_repo = GitRepoRemote::Open(
op_result.git_cas); // link fake repo to odb
if (not git_repo) {
- (*logger)(fmt::format("Could not open repository {}",
- storage_config->GitRoot().string()),
- /*fatal=*/true);
+ (*logger)(
+ fmt::format("Could not open repository {}",
+ native_storage_config->GitRoot().string()),
+ /*fatal=*/true);
return;
}
// setup wrapped logger
@@ -327,7 +382,10 @@ auto CreateGitTreeFetchMap(
// backup to remote if needed and in native mode
if (backup_to_remote and remote_api != nullptr) {
BackupToRemote(ArtifactDigest{key.tree_hash, 0},
- *storage_config,
+ *native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
*remote_api,
logger);
}
@@ -338,9 +396,10 @@ auto CreateGitTreeFetchMap(
// Check older generations for presence of the tree
for (std::size_t generation = 1;
- generation < storage_config->num_generations;
+ generation < native_storage_config->num_generations;
generation++) {
- auto old = storage_config->GitGenerationRoot(generation);
+ auto old =
+ native_storage_config->GitGenerationRoot(generation);
if (FileSystemManager::IsDirectory(old)) {
auto old_repo = GitRepo::Open(old);
auto no_logging =
@@ -353,9 +412,12 @@ auto CreateGitTreeFetchMap(
TakeTreeFromOlderGeneration(
generation,
ArtifactDigest{key.tree_hash, 0},
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
op_result.git_cas,
critical_git_op_map,
+ local_api,
remote_api,
backup_to_remote,
ts,
@@ -373,7 +435,9 @@ auto CreateGitTreeFetchMap(
// import tree to Git cache
MoveCASTreeToGit(digest,
import_to_git_map,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
local_api,
remote_api,
backup_to_remote,
@@ -402,7 +466,9 @@ auto CreateGitTreeFetchMap(
MoveCASTreeToGit(
digest,
import_to_git_map,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
local_api,
remote_api,
false, // tree already in remote, so ignore backing up
@@ -414,7 +480,7 @@ auto CreateGitTreeFetchMap(
}
// create temporary location for command execution root
auto content_dir =
- storage_config->CreateTypedTmpDir("git-tree");
+ native_storage_config->CreateTypedTmpDir("git-tree");
if (not content_dir) {
(*logger)(
"Failed to create execution root tmp directory for "
@@ -423,7 +489,8 @@ auto CreateGitTreeFetchMap(
return;
}
// create temporary location for storing command result files
- auto out_dir = storage_config->CreateTypedTmpDir("git-tree");
+ auto out_dir =
+ native_storage_config->CreateTypedTmpDir("git-tree");
if (not out_dir) {
(*logger)(
"Failed to create results tmp directory for tree id "
@@ -454,7 +521,7 @@ auto CreateGitTreeFetchMap(
}
// create temporary location for the import repository
auto repo_dir =
- storage_config->CreateTypedTmpDir("import-repo");
+ native_storage_config->CreateTypedTmpDir("import-repo");
if (not repo_dir) {
(*logger)(
"Failed to create tmp directory for import repository",
@@ -484,7 +551,10 @@ auto CreateGitTreeFetchMap(
key,
git_bin,
launcher,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
backup_to_remote,
progress,
@@ -558,14 +628,15 @@ auto CreateGitTreeFetchMap(
auto just_git_repo = GitRepoRemote::Open(just_git_cas);
if (not just_git_repo) {
(*logger)(
- fmt::format("Could not open Git repository {}",
- storage_config->GitRoot().string()),
+ fmt::format(
+ "Could not open Git repository {}",
+ native_storage_config->GitRoot().string()),
/*fatal=*/true);
return;
}
// define temp repo path
- auto tmp_dir =
- storage_config->CreateTypedTmpDir("git-tree");
+ auto tmp_dir = native_storage_config->CreateTypedTmpDir(
+ "git-tree");
;
if (not tmp_dir) {
(*logger)(fmt::format("Could not create unique "
@@ -586,7 +657,7 @@ auto CreateGitTreeFetchMap(
fatal);
});
if (not just_git_repo->FetchViaTmpRepo(
- *storage_config,
+ *native_storage_config,
target_path.string(),
std::nullopt,
key.inherit_env,
@@ -612,7 +683,8 @@ auto CreateGitTreeFetchMap(
GitOpKey op_key = {
.params =
{
- storage_config->GitRoot(), // target_path
+ native_storage_config
+ ->GitRoot(), // target_path
*op_result.result, // git_hash
"Keep referenced tree alive" // message
},
@@ -621,7 +693,10 @@ auto CreateGitTreeFetchMap(
ts,
{std::move(op_key)},
[remote_api,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
backup_to_remote,
key,
progress,
@@ -640,7 +715,10 @@ auto CreateGitTreeFetchMap(
remote_api != nullptr) {
BackupToRemote(
ArtifactDigest{key.tree_hash, 0},
- *storage_config,
+ *native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
*remote_api,
logger);
}
@@ -649,7 +727,7 @@ auto CreateGitTreeFetchMap(
},
[logger,
commit = *op_result.result,
- target_path = storage_config->GitRoot()](
+ target_path = native_storage_config->GitRoot()](
auto const& msg, bool fatal) {
(*logger)(
fmt::format("While running critical Git op "
@@ -671,8 +749,8 @@ auto CreateGitTreeFetchMap(
fatal);
});
},
- [logger, target_path = storage_config->GitRoot()](auto const& msg,
- bool fatal) {
+ [logger, target_path = native_storage_config->GitRoot()](
+ auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git op "
"ENSURE_INIT bare for target {}:\n{}",
target_path.string(),
diff --git a/src/other_tools/ops_maps/git_tree_fetch_map.hpp b/src/other_tools/ops_maps/git_tree_fetch_map.hpp
index e5949099..7ff6b301 100644
--- a/src/other_tools/ops_maps/git_tree_fetch_map.hpp
+++ b/src/other_tools/ops_maps/git_tree_fetch_map.hpp
@@ -27,6 +27,7 @@
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/serve_api/remote/serve_api.hpp"
#include "src/buildtool/storage/config.hpp"
+#include "src/buildtool/storage/storage.hpp"
#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
#include "src/other_tools/ops_maps/import_to_git_map.hpp"
@@ -65,7 +66,9 @@ using GitTreeFetchMap = AsyncMapConsumer<GitTreeInfo, bool>;
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
bool backup_to_remote,
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index 83ab83de..61e16cb7 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -45,6 +45,7 @@
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/serve_api/remote", "serve_api"]
, ["src/buildtool/storage", "config"]
+ , ["src/buildtool/storage", "storage"]
, ["src/other_tools/just_mr", "mirrors"]
, ["src/other_tools/just_mr/progress_reporting", "progress"]
, ["src/other_tools/ops_maps", "critical_git_op_map"]
@@ -80,6 +81,7 @@
, ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"]
, ["src/buildtool/serve_api/remote", "serve_api"]
, ["src/buildtool/storage", "config"]
+ , ["src/buildtool/storage", "storage"]
, ["src/other_tools/just_mr", "utils"]
, ["src/other_tools/ops_maps", "import_to_git_map"]
, ["src/utils/cpp", "hash_combine"]
@@ -169,6 +171,7 @@
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/serve_api/remote", "serve_api"]
, ["src/buildtool/storage", "config"]
+ , ["src/buildtool/storage", "storage"]
, ["src/other_tools/ops_maps", "critical_git_op_map"]
, ["src/other_tools/ops_maps", "git_tree_fetch_map"]
, ["src/other_tools/ops_maps", "import_to_git_map"]
@@ -181,7 +184,7 @@
, ["src/buildtool/common", "common"]
, ["src/buildtool/common", "config"]
, ["src/buildtool/crypto", "hash_info"]
- , ["src/buildtool/execution_api/git", "git"]
+ , ["src/buildtool/execution_api/serve", "mr_git_api"]
, ["src/buildtool/file_system", "file_root"]
]
}
@@ -195,6 +198,8 @@
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/multithreading", "async_map_consumer"]
, ["src/buildtool/serve_api/remote", "serve_api"]
+ , ["src/buildtool/storage", "config"]
+ , ["src/buildtool/storage", "storage"]
]
, "stage": ["src", "other_tools", "root_maps"]
, "private-deps":
@@ -203,7 +208,7 @@
, ["src/buildtool/common", "common"]
, ["src/buildtool/common", "config"]
, ["src/buildtool/crypto", "hash_function"]
- , ["src/buildtool/execution_api/git", "git"]
+ , ["src/buildtool/execution_api/serve", "mr_git_api"]
, ["src/buildtool/file_system", "object_type"]
]
}
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index e6463d57..f221f63f 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -49,23 +49,29 @@ namespace {
}
[[nodiscard]] auto IsCacheGitRoot(
- StorageConfig const& storage_config,
+ StorageConfig const& native_storage_config,
std::filesystem::path const& repo_root) noexcept -> bool {
return std::filesystem::absolute(ToNormalPath(repo_root)) ==
- std::filesystem::absolute(ToNormalPath(storage_config.GitRoot()));
+ std::filesystem::absolute(
+ ToNormalPath(native_storage_config.GitRoot()));
}
/// \brief Helper function for ensuring the serve endpoint, if given, has the
/// root if it was marked absent.
/// It guarantees the logger is called exactly once with fatal on failure, and
/// the setter on success.
-void EnsureRootAsAbsent(std::string const& tree_id,
- std::filesystem::path const& repo_root,
- GitRepoInfo const& repo_info,
- ServeApi const* serve,
- IExecutionApi const* remote_api,
- CommitGitMap::SetterPtr const& ws_setter,
- CommitGitMap::LoggerPtr const& logger) {
+void EnsureRootAsAbsent(
+ std::string const& tree_id,
+ std::filesystem::path const& repo_root,
+ GitRepoInfo const& repo_info,
+ ServeApi const* serve,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ gsl::not_null<IExecutionApi const*> const& local_api,
+ IExecutionApi const* remote_api,
+ CommitGitMap::SetterPtr const& ws_setter,
+ CommitGitMap::LoggerPtr const& logger) {
// this is an absent root
if (serve != nullptr) {
// check if the serve endpoint has this root
@@ -116,6 +122,10 @@ void EnsureRootAsAbsent(std::string const& tree_id,
if (not EnsureAbsentRootOnServe(*serve,
tree_id,
repo_root,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ &*local_api,
remote_api,
logger,
true /*no_sync_is_fatal*/)) {
@@ -146,7 +156,7 @@ void EnsureRootAsAbsent(std::string const& tree_id,
void WriteIdFileAndSetWSRoot(std::string const& root_tree_id,
std::string const& subdir,
bool ignore_special,
- StorageConfig const& storage_config,
+ StorageConfig const& native_storage_config,
GitCASPtr const& git_cas,
std::filesystem::path const& tree_id_file,
CommitGitMap::SetterPtr const& ws_setter,
@@ -163,7 +173,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id,
auto git_repo = GitRepoRemote::Open(git_cas); // link fake repo to odb
if (not git_repo) {
(*logger)(fmt::format("Could not open cache object database {}",
- storage_config.GitRoot().string()),
+ native_storage_config.GitRoot().string()),
/*fatal=*/true);
return;
}
@@ -186,7 +196,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id,
? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
*tree_id,
- storage_config.GitRoot().string()}),
+ native_storage_config.GitRoot().string()}),
false));
}
@@ -269,7 +279,7 @@ void TagAndSetRoot(std::filesystem::path const& repo_root,
void TakeCommitFromOlderGeneration(
std::filesystem::path const& source,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
std::filesystem::path const& repo_root,
GitRepoInfo const& repo_info,
GitCASPtr const& git_cas,
@@ -291,7 +301,7 @@ void TakeCommitFromOlderGeneration(
[logger,
git_cas,
repo_root,
- storage_config,
+ native_storage_config,
source,
repo_info,
critical_git_op_map,
@@ -321,7 +331,7 @@ void TakeCommitFromOlderGeneration(
fatal);
});
if (not git_repo->LocalFetchViaTmpRepo(
- *storage_config, source, tag, fetch_logger)) {
+ *native_storage_config, source, tag, fetch_logger)) {
return;
}
TagAndSetRoot(repo_root,
@@ -349,7 +359,7 @@ void NetworkFetchAndSetPresentRoot(
std::string const& fetch_repo,
MirrorsPtr const& additional_mirrors,
GitCASPtr const& git_cas,
- StorageConfig const& storage_config,
+ StorageConfig const& native_storage_config,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
std::string const& git_bin,
std::vector<std::string> const& launcher,
@@ -410,7 +420,7 @@ void NetworkFetchAndSetPresentRoot(
err_messages += fmt::format(
"While attempting fetch from URL {}:\n{}\n", mirror, msg);
});
- if (git_repo->FetchViaTmpRepo(storage_config,
+ if (git_repo->FetchViaTmpRepo(native_storage_config,
mirror,
repo_info.branch,
repo_info.inherit_env,
@@ -456,7 +466,7 @@ void NetworkFetchAndSetPresentRoot(
return;
}
// if witnessing repository is the Git cache, then also tag the commit
- if (IsCacheGitRoot(storage_config, repo_root)) {
+ if (IsCacheGitRoot(native_storage_config, repo_root)) {
TagAndSetRoot(repo_root,
repo_info,
true,
@@ -505,24 +515,27 @@ void NetworkFetchAndSetPresentRoot(
/// the root.
/// It guarantees the logger is called exactly once with fatal on failure, and
/// the setter on success.
-void EnsureCommit(GitRepoInfo const& repo_info,
- std::filesystem::path const& repo_root,
- std::string const& fetch_repo,
- MirrorsPtr const& additional_mirrors,
- GitCASPtr const& git_cas,
- gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
- gsl::not_null<ImportToGitMap*> const& import_to_git_map,
- std::string const& git_bin,
- std::vector<std::string> const& launcher,
- ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<IExecutionApi const*> const& local_api,
- IExecutionApi const* remote_api,
- bool fetch_absent,
- gsl::not_null<JustMRProgress*> const& progress,
- gsl::not_null<TaskSystem*> const& ts,
- CommitGitMap::SetterPtr const& ws_setter,
- CommitGitMap::LoggerPtr const& logger) {
+void EnsureCommit(
+ GitRepoInfo const& repo_info,
+ std::filesystem::path const& repo_root,
+ std::string const& fetch_repo,
+ MirrorsPtr const& additional_mirrors,
+ GitCASPtr const& git_cas,
+ gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ gsl::not_null<ImportToGitMap*> const& import_to_git_map,
+ std::string const& git_bin,
+ std::vector<std::string> const& launcher,
+ ServeApi const* serve,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ gsl::not_null<IExecutionApi const*> const& local_api,
+ IExecutionApi const* remote_api,
+ bool fetch_absent,
+ gsl::not_null<JustMRProgress*> const& progress,
+ gsl::not_null<TaskSystem*> const& ts,
+ CommitGitMap::SetterPtr const& ws_setter,
+ CommitGitMap::LoggerPtr const& logger) {
// link fake repo to odb
auto git_repo = GitRepoRemote::Open(git_cas);
if (not git_repo) {
@@ -543,8 +556,8 @@ void EnsureCommit(GitRepoInfo const& repo_info,
return;
}
if (not is_commit_present.value()) {
- auto tree_id_file =
- StorageUtils::GetCommitTreeIDFile(*storage_config, repo_info.hash);
+ auto tree_id_file = StorageUtils::GetCommitTreeIDFile(
+ *native_storage_config, repo_info.hash);
// Check if we have stored a file association between commit and tree;
// if an association file exists, the respective tree MUST be in the
// Git cache
@@ -557,18 +570,20 @@ void EnsureCommit(GitRepoInfo const& repo_info,
/*fatal=*/true);
return;
}
- auto just_git_cas = GitCAS::Open(storage_config->GitRoot());
+ auto just_git_cas = GitCAS::Open(native_storage_config->GitRoot());
if (not just_git_cas) {
- (*logger)(fmt::format("Could not open Git cache database {}",
- storage_config->GitRoot().string()),
- /*fatal=*/true);
+ (*logger)(
+ fmt::format("Could not open Git cache database {}",
+ native_storage_config->GitRoot().string()),
+ /*fatal=*/true);
return;
}
auto just_git_repo = GitRepo::Open(just_git_cas);
if (not just_git_repo) {
- (*logger)(fmt::format("Could not open Git cache repository {}",
- storage_config->GitRoot().string()),
- /*fatal=*/true);
+ (*logger)(
+ fmt::format("Could not open Git cache repository {}",
+ native_storage_config->GitRoot().string()),
+ /*fatal=*/true);
return;
}
// extract the subdir tree
@@ -590,13 +605,18 @@ void EnsureCommit(GitRepoInfo const& repo_info,
// set the workspace root
if (repo_info.absent and not fetch_absent) {
// try by all available means to generate & set the absent root
- EnsureRootAsAbsent(*tree_id,
- storage_config->GitRoot(),
- repo_info,
- serve,
- remote_api,
- ws_setter,
- logger);
+ EnsureRootAsAbsent(
+ *tree_id,
+ native_storage_config->GitRoot(), /*repo_root*/
+ repo_info,
+ serve,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
+ remote_api,
+ ws_setter,
+ logger);
}
else {
// this root is present
@@ -606,7 +626,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
*tree_id,
- storage_config->GitRoot().string()}),
+ native_storage_config->GitRoot().string()}),
/*is_cache_hit=*/false));
}
// done!
@@ -615,9 +635,9 @@ void EnsureCommit(GitRepoInfo const& repo_info,
// Check older generations for presence of the commit
for (std::size_t generation = 1;
- generation < storage_config->num_generations;
+ generation < native_storage_config->num_generations;
generation++) {
- auto old = storage_config->GitGenerationRoot(generation);
+ auto old = native_storage_config->GitGenerationRoot(generation);
if (FileSystemManager::IsDirectory(old)) {
auto old_repo = GitRepo::Open(old);
auto no_logging = std::make_shared<AsyncMapConsumerLogger>(
@@ -627,7 +647,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
old_repo->CheckCommitExists(repo_info.hash, no_logging);
if (check_result and *check_result) {
TakeCommitFromOlderGeneration(old,
- storage_config,
+ native_storage_config,
repo_root,
repo_info,
git_cas,
@@ -687,16 +707,16 @@ void EnsureCommit(GitRepoInfo const& repo_info,
if (serve_result) {
auto const& root_tree_id = *serve_result;
// verify if we know the tree already in the local Git cache
- GitOpKey op_key = {
- .params =
- {
- storage_config->GitRoot(), // target_path
- "", // git_hash
- std::nullopt, // message
- std::nullopt, // source_path
- true // init_bare
- },
- .op_type = GitOpType::ENSURE_INIT};
+ GitOpKey op_key = {.params =
+ {
+ native_storage_config
+ ->GitRoot(), // target_path
+ "", // git_hash
+ std::nullopt, // message
+ std::nullopt, // source_path
+ true // init_bare
+ },
+ .op_type = GitOpType::ENSURE_INIT};
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
@@ -706,7 +726,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
repo_root,
fetch_repo,
additional_mirrors,
- storage_config,
+ native_storage_config,
git_cas,
critical_git_op_map,
import_to_git_map,
@@ -730,28 +750,31 @@ void EnsureCommit(GitRepoInfo const& repo_info,
GitRepoRemote::Open(op_result.git_cas);
if (not just_git_repo) {
(*logger)(
- fmt::format(
- "Could not open Git "
- "cache repository "
- "{}",
- storage_config->GitRoot().string()),
+ fmt::format("Could not open Git "
+ "cache repository "
+ "{}",
+ native_storage_config->GitRoot()
+ .string()),
/*fatal=*/true);
return;
}
// check tree existence
- auto wrapped_logger = std::make_shared<
- AsyncMapConsumerLogger>(
- [logger, storage_config, tree = root_tree_id](
- auto const& msg, bool fatal) {
- (*logger)(
- fmt::format(
- "While verifying presence of "
- "tree {} in repository {}:\n{}",
- tree,
- storage_config->GitRoot().string(),
- msg),
- fatal);
- });
+ auto wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [logger,
+ native_storage_config,
+ tree = root_tree_id](auto const& msg,
+ bool fatal) {
+ (*logger)(
+ fmt::format(
+ "While verifying presence of "
+ "tree {} in repository {}:\n{}",
+ tree,
+ native_storage_config->GitRoot()
+ .string(),
+ msg),
+ fatal);
+ });
auto tree_present = just_git_repo->CheckTreeExists(
root_tree_id, wrapped_logger);
if (not tree_present) {
@@ -765,7 +788,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
root_tree_id,
repo_info.subdir,
repo_info.ignore_special,
- *storage_config,
+ *native_storage_config,
op_result.git_cas,
tree_id_file,
ws_setter,
@@ -776,7 +799,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
// now check if the tree is in the local checkout,
// if this checkout is not our Git cache; this can
// save an unnecessary remote CAS call
- if (not IsCacheGitRoot(*storage_config,
+ if (not IsCacheGitRoot(*native_storage_config,
repo_root)) {
auto git_repo = GitRepoRemote::Open(git_cas);
if (not git_repo) {
@@ -855,7 +878,8 @@ void EnsureCommit(GitRepoInfo const& repo_info,
// try to get root tree from remote CAS
auto const root_digest =
ArtifactDigestFactory::Create(
- storage_config->hash_function.GetType(),
+ native_storage_config->hash_function
+ .GetType(),
root_tree_id,
0,
/*is_tree=*/true);
@@ -868,7 +892,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
progress->TaskTracker().Stop(repo_info.origin);
// Move tree from local CAS to local Git storage
auto tmp_dir =
- storage_config->CreateTypedTmpDir(
+ native_storage_config->CreateTypedTmpDir(
"fetch-absent-root");
if (not tmp_dir) {
(*logger)(
@@ -901,7 +925,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
{std::move(c_info)},
[tmp_dir, // keep tmp_dir alive
root_tree_id,
- storage_config,
+ native_storage_config,
subdir = repo_info.subdir,
ignore_special = repo_info.ignore_special,
just_git_cas = op_result.git_cas,
@@ -930,14 +954,15 @@ void EnsureCommit(GitRepoInfo const& repo_info,
// write association to id file, get
// subdir tree, and set the workspace
// root as present
- WriteIdFileAndSetWSRoot(root_tree_id,
- subdir,
- ignore_special,
- *storage_config,
- just_git_cas,
- tree_id_file,
- ws_setter,
- logger);
+ WriteIdFileAndSetWSRoot(
+ root_tree_id,
+ subdir,
+ ignore_special,
+ *native_storage_config,
+ just_git_cas,
+ tree_id_file,
+ ws_setter,
+ logger);
},
[logger, tmp_dir, root_tree_id](
auto const& msg, bool fatal) {
@@ -961,22 +986,24 @@ void EnsureCommit(GitRepoInfo const& repo_info,
root_tree_id),
/*fatal=*/false);
- NetworkFetchAndSetPresentRoot(repo_info,
- repo_root,
- fetch_repo,
- additional_mirrors,
- git_cas,
- *storage_config,
- critical_git_op_map,
- git_bin,
- launcher,
- fetch_absent,
- progress,
- ts,
- ws_setter,
- logger);
+ NetworkFetchAndSetPresentRoot(
+ repo_info,
+ repo_root,
+ fetch_repo,
+ additional_mirrors,
+ git_cas,
+ *native_storage_config,
+ critical_git_op_map,
+ git_bin,
+ launcher,
+ fetch_absent,
+ progress,
+ ts,
+ ws_setter,
+ logger);
},
- [logger, target_path = storage_config->GitRoot()](
+ [logger,
+ target_path = native_storage_config->GitRoot()](
auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git "
"op ENSURE_INIT bare for "
@@ -1007,7 +1034,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
fetch_repo,
additional_mirrors,
git_cas,
- *storage_config,
+ *native_storage_config,
critical_git_op_map,
git_bin,
launcher,
@@ -1041,6 +1068,10 @@ void EnsureCommit(GitRepoInfo const& repo_info,
repo_root,
repo_info,
serve,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ws_setter,
logger);
@@ -1070,7 +1101,9 @@ auto CreateCommitGitMap(
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
bool fetch_absent,
@@ -1083,7 +1116,9 @@ auto CreateCommitGitMap(
git_bin,
launcher,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
local_api,
remote_api,
fetch_absent,
@@ -1100,7 +1135,7 @@ auto CreateCommitGitMap(
fetch_repo = std::filesystem::absolute(*fetch_repo_path).string();
}
std::filesystem::path repo_root = StorageUtils::GetGitRoot(
- *storage_config, just_mr_paths, fetch_repo);
+ *native_storage_config, just_mr_paths, fetch_repo);
// ensure git repo
// define Git operation to be done
GitOpKey op_key = {
@@ -1126,7 +1161,9 @@ auto CreateCommitGitMap(
git_bin,
launcher,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
local_api,
remote_api,
fetch_absent,
@@ -1161,7 +1198,9 @@ auto CreateCommitGitMap(
git_bin,
launcher,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
local_api,
remote_api,
fetch_absent,
diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp
index 1c5e9f6b..fad04614 100644
--- a/src/other_tools/root_maps/commit_git_map.hpp
+++ b/src/other_tools/root_maps/commit_git_map.hpp
@@ -27,6 +27,7 @@
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/serve_api/remote/serve_api.hpp"
#include "src/buildtool/storage/config.hpp"
+#include "src/buildtool/storage/storage.hpp"
#include "src/other_tools/just_mr/mirrors.hpp"
#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
@@ -84,7 +85,9 @@ using CommitGitMap =
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
bool fetch_absent,
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp
index 43222763..ecae60ac 100644
--- a/src/other_tools/root_maps/content_git_map.cpp
+++ b/src/other_tools/root_maps/content_git_map.cpp
@@ -52,7 +52,10 @@ void EnsureRootAsAbsent(
std::string const& tree_id,
ArchiveRepoInfo const& key,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
bool is_cache_hit,
ContentGitMap::SetterPtr const& ws_setter,
@@ -114,7 +117,11 @@ void EnsureRootAsAbsent(
if (not EnsureAbsentRootOnServe(
*serve,
tree_id,
- storage_config->GitRoot(),
+ native_storage_config->GitRoot(), /*repo_path*/
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
logger,
/*no_sync_is_fatal=*/true)) {
@@ -137,12 +144,17 @@ void EnsureRootAsAbsent(
// the tree is known locally, so we can upload it to remote
// CAS for the serve endpoint to retrieve it and set up the
// root
- if (not EnsureAbsentRootOnServe(*serve,
- tree_id,
- storage_config->GitRoot(),
- remote_api,
- logger,
- /*no_sync_is_fatal=*/true)) {
+ if (not EnsureAbsentRootOnServe(
+ *serve,
+ tree_id,
+ native_storage_config->GitRoot(), /*repo_root*/
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
+ remote_api,
+ logger,
+ /*no_sync_is_fatal=*/true)) {
return;
}
}
@@ -171,7 +183,10 @@ void ResolveContentTree(
bool is_cache_hit,
bool is_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
@@ -181,7 +196,7 @@ void ResolveContentTree(
if (key.pragma_special) {
// get the resolved tree
auto tree_id_file = StorageUtils::GetResolvedTreeIDFile(
- *storage_config, tree_hash, *key.pragma_special);
+ *native_storage_config, tree_hash, *key.pragma_special);
if (FileSystemManager::Exists(tree_id_file)) {
// read resolved tree id
auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file);
@@ -197,18 +212,22 @@ void ResolveContentTree(
EnsureRootAsAbsent(*resolved_tree_id,
key,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
is_cache_hit,
ws_setter,
logger);
}
else {
- (*ws_setter)(std::pair(
- nlohmann::json::array({FileRoot::kGitTreeMarker,
- *resolved_tree_id,
- storage_config->GitRoot().string()}),
- /*is_cache_hit=*/is_cache_hit));
+ (*ws_setter)(
+ std::pair(nlohmann::json::array(
+ {FileRoot::kGitTreeMarker,
+ *resolved_tree_id,
+ native_storage_config->GitRoot().string()}),
+ /*is_cache_hit=*/is_cache_hit));
}
}
else {
@@ -229,7 +248,10 @@ void ResolveContentTree(
key,
is_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ts,
ws_setter,
@@ -239,7 +261,8 @@ void ResolveContentTree(
GitOpKey op_key = {
.params =
{
- storage_config->GitRoot(), // target_path
+ native_storage_config
+ ->GitRoot(), // target_path
resolved_tree_id, // git_hash
"Keep referenced tree alive" // message
},
@@ -252,7 +275,10 @@ void ResolveContentTree(
tree_id_file,
is_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
is_cache_hit,
ws_setter,
@@ -279,7 +305,10 @@ void ResolveContentTree(
EnsureRootAsAbsent(resolved_tree_id,
key,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
is_cache_hit,
ws_setter,
@@ -290,11 +319,13 @@ void ResolveContentTree(
nlohmann::json::array(
{FileRoot::kGitTreeMarker,
resolved_tree_id,
- storage_config->GitRoot().string()}),
+ native_storage_config->GitRoot()
+ .string()}),
/*is_cache_hit=*/is_cache_hit));
}
},
- [logger, target_path = storage_config->GitRoot()](
+ [logger,
+ target_path = native_storage_config->GitRoot()](
auto const& msg, bool fatal) {
(*logger)(
fmt::format("While running critical Git op "
@@ -320,18 +351,22 @@ void ResolveContentTree(
EnsureRootAsAbsent(tree_hash,
key,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
is_cache_hit,
ws_setter,
logger);
}
else {
- (*ws_setter)(std::pair(
- nlohmann::json::array({FileRoot::kGitTreeMarker,
- tree_hash,
- storage_config->GitRoot().string()}),
- /*is_cache_hit=*/is_cache_hit));
+ (*ws_setter)(
+ std::pair(nlohmann::json::array(
+ {FileRoot::kGitTreeMarker,
+ tree_hash,
+ native_storage_config->GitRoot().string()}),
+ /*is_cache_hit=*/is_cache_hit));
}
}
}
@@ -346,7 +381,10 @@ void WriteIdFileAndSetWSRoot(
std::filesystem::path const& archive_tree_id_file,
bool is_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
@@ -391,7 +429,10 @@ void WriteIdFileAndSetWSRoot(
false, /*is_cache_hit*/
is_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
critical_git_op_map,
resolve_symlinks_map,
@@ -409,7 +450,10 @@ void ExtractAndImportToGit(
std::filesystem::path const& archive_tree_id_file,
bool is_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
@@ -418,7 +462,7 @@ void ExtractAndImportToGit(
ContentGitMap::SetterPtr const& setter,
ContentGitMap::LoggerPtr const& logger) {
// extract archive
- auto tmp_dir = storage_config->CreateTypedTmpDir(key.repo_type);
+ auto tmp_dir = native_storage_config->CreateTypedTmpDir(key.repo_type);
if (not tmp_dir) {
(*logger)(fmt::format("Failed to create tmp path for {} target {}",
key.repo_type,
@@ -447,7 +491,10 @@ void ExtractAndImportToGit(
key,
is_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
critical_git_op_map,
resolve_symlinks_map,
@@ -469,7 +516,10 @@ void ExtractAndImportToGit(
archive_tree_id_file,
is_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
critical_git_op_map,
resolve_symlinks_map,
@@ -487,13 +537,13 @@ void ExtractAndImportToGit(
}
auto IdFileExistsInOlderGeneration(
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
ArchiveRepoInfo const& key) -> std::optional<std::size_t> {
for (std::size_t generation = 1;
- generation < storage_config->num_generations;
+ generation < native_storage_config->num_generations;
generation++) {
auto archive_tree_id_file =
- StorageUtils::GetArchiveTreeIDFile(*storage_config,
+ StorageUtils::GetArchiveTreeIDFile(*native_storage_config,
key.repo_type,
key.archive.content_hash.Hash(),
generation);
@@ -509,9 +559,12 @@ void HandleLocallyKnownTree(
std::filesystem::path const& archive_tree_id_file,
bool fetch_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<TaskSystem*> const& ts,
ContentGitMap::SetterPtr const& setter,
@@ -528,11 +581,11 @@ void HandleLocallyKnownTree(
// define Git operation to be done
GitOpKey op_key = {.params =
{
- storage_config->GitRoot(), // target_path
- "", // git_hash
- std::nullopt, // message
- std::nullopt, // source_path
- true // init_bare
+ native_storage_config->GitRoot(), // target_path
+ "", // git_hash
+ std::nullopt, // message
+ std::nullopt, // source_path
+ true // init_bare
},
.op_type = GitOpType::ENSURE_INIT};
critical_git_op_map->ConsumeAfterKeysReady(
@@ -542,7 +595,10 @@ void HandleLocallyKnownTree(
key,
fetch_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
critical_git_op_map,
resolve_symlinks_map,
@@ -585,7 +641,10 @@ void HandleLocallyKnownTree(
/*is_cache_hit = */ true,
/*is_absent = */ (key.absent and not fetch_absent),
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
critical_git_op_map,
resolve_symlinks_map,
@@ -593,8 +652,8 @@ void HandleLocallyKnownTree(
setter,
logger);
},
- [logger, target_path = storage_config->GitRoot()](auto const& msg,
- bool fatal) {
+ [logger, target_path = native_storage_config->GitRoot()](
+ auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git "
"op ENSURE_INIT for "
"target {}:\n{}",
@@ -609,9 +668,12 @@ void HandleKnownInOlderGenerationAfterImport(
const std::string& tree_id,
bool fetch_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<TaskSystem*> const& ts,
ContentGitMap::SetterPtr const& setter,
@@ -619,7 +681,7 @@ void HandleKnownInOlderGenerationAfterImport(
// Now that we have the tree persisted in the git repository of the youngest
// generation; hence we can write the map-entry.
auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile(
- *storage_config, key.repo_type, key.archive.content_hash.Hash());
+ *native_storage_config, key.repo_type, key.archive.content_hash.Hash());
if (not StorageUtils::WriteTreeIDFile(archive_tree_id_file, tree_id)) {
(*logger)(fmt::format("Failed to write tree id to file {}",
archive_tree_id_file.string()),
@@ -632,9 +694,12 @@ void HandleKnownInOlderGenerationAfterImport(
archive_tree_id_file,
fetch_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
@@ -649,10 +714,12 @@ void HandleKnownInOlderGenerationAfterTaggingAndInit(
std::filesystem::path const& source,
bool fetch_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<Storage const*> const& storage,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<TaskSystem*> const& ts,
ContentGitMap::SetterPtr const& setter,
@@ -672,14 +739,14 @@ void HandleKnownInOlderGenerationAfterTaggingAndInit(
fatal);
});
if (not git_repo->LocalFetchViaTmpRepo(
- *storage_config, source, tag, fetch_logger)) {
+ *native_storage_config, source, tag, fetch_logger)) {
return;
}
GitOpKey op_key = {.params =
{
- storage_config->GitRoot(), // target_path
- tree_id, // git_hash
- "Keep referenced tree alive" // message
+ native_storage_config->GitRoot(), // target_path
+ tree_id, // git_hash
+ "Keep referenced tree alive" // message
},
.op_type = GitOpType::KEEP_TREE};
critical_git_op_map->ConsumeAfterKeysReady(
@@ -690,10 +757,12 @@ void HandleKnownInOlderGenerationAfterTaggingAndInit(
git_cas,
fetch_absent,
serve,
- storage_config,
- storage,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
@@ -709,9 +778,12 @@ void HandleKnownInOlderGenerationAfterTaggingAndInit(
tree_id,
fetch_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
@@ -732,21 +804,23 @@ void HandleKnownInOlderGenerationAfterTagging(
std::filesystem::path const& source,
bool fetch_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<Storage const*> const& storage,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<TaskSystem*> const& ts,
ContentGitMap::SetterPtr const& setter,
ContentGitMap::LoggerPtr const& logger) {
GitOpKey op_key = {.params =
{
- storage_config->GitRoot(), // target_path
- "", // git_hash
- std::nullopt, // message
- std::nullopt, // source_path
- true // init_bare
+ native_storage_config->GitRoot(), // target_path
+ "", // git_hash
+ std::nullopt, // message
+ std::nullopt, // source_path
+ true // init_bare
},
.op_type = GitOpType::ENSURE_INIT};
critical_git_op_map->ConsumeAfterKeysReady(
@@ -758,10 +832,12 @@ void HandleKnownInOlderGenerationAfterTagging(
source,
fetch_absent,
serve,
- storage_config,
- storage,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
@@ -780,17 +856,19 @@ void HandleKnownInOlderGenerationAfterTagging(
source,
fetch_absent,
serve,
- storage_config,
- storage,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
logger);
},
- [logger, target_path = storage_config->GitRoot()](auto const& msg,
- bool fatal) {
+ [logger, target_path = native_storage_config->GitRoot()](
+ auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git op "
"ENSURE_INIT for target {}:\n{}",
target_path.string(),
@@ -804,16 +882,18 @@ void HandleKnownInOlderGeneration(
std::size_t generation,
bool fetch_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<Storage const*> const& storage,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<TaskSystem*> const& ts,
ContentGitMap::SetterPtr const& setter,
ContentGitMap::LoggerPtr const& logger) {
auto archive_tree_id_file =
- StorageUtils::GetArchiveTreeIDFile(*storage_config,
+ StorageUtils::GetArchiveTreeIDFile(*native_storage_config,
key.repo_type,
key.archive.content_hash.Hash(),
generation);
@@ -824,7 +904,7 @@ void HandleKnownInOlderGeneration(
/*fatal=*/true);
return;
}
- auto source = storage_config->GitGenerationRoot(generation);
+ auto source = native_storage_config->GitGenerationRoot(generation);
GitOpKey op_key = {.params =
{
@@ -841,10 +921,12 @@ void HandleKnownInOlderGeneration(
source,
fetch_absent,
serve,
- storage_config,
- storage,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
@@ -861,10 +943,12 @@ void HandleKnownInOlderGeneration(
source,
fetch_absent,
serve,
- storage_config,
- storage,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
@@ -891,8 +975,11 @@ auto CreateContentGitMap(
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<Storage const*> const& storage,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ gsl::not_null<Storage const*> const& native_storage,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
bool fetch_absent,
gsl::not_null<JustMRProgress*> const& progress,
@@ -905,8 +992,11 @@ auto CreateContentGitMap(
additional_mirrors,
ca_info,
serve,
- storage,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ native_storage,
+ compat_storage,
+ local_api,
remote_api,
fetch_absent,
progress](auto ts,
@@ -914,31 +1004,38 @@ auto CreateContentGitMap(
auto logger,
auto /* unused */,
auto const& key) {
- auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile(
- *storage_config, key.repo_type, key.archive.content_hash.Hash());
+ auto archive_tree_id_file =
+ StorageUtils::GetArchiveTreeIDFile(*native_storage_config,
+ key.repo_type,
+ key.archive.content_hash.Hash());
if (FileSystemManager::Exists(archive_tree_id_file)) {
HandleLocallyKnownTree(key,
archive_tree_id_file,
fetch_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
logger);
}
- else if (auto generation =
- IdFileExistsInOlderGeneration(storage_config, key)) {
+ else if (auto generation = IdFileExistsInOlderGeneration(
+ native_storage_config, key)) {
HandleKnownInOlderGeneration(key,
*generation,
fetch_absent,
serve,
- storage_config,
- storage,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
resolve_symlinks_map,
critical_git_op_map,
+ local_api,
remote_api,
ts,
setter,
@@ -981,16 +1078,19 @@ auto CreateContentGitMap(
// a serve endpoint exists we can upload it the root ourselves;
// check if content already in CAS
- auto const& cas = storage->CAS();
+ auto const& native_cas = native_storage->CAS();
auto const digest = ArtifactDigest{key.archive.content_hash, 0};
if (auto content_cas_path =
- cas.BlobPath(digest, /*is_executable=*/false)) {
+ native_cas.BlobPath(digest, /*is_executable=*/false)) {
ExtractAndImportToGit(key,
*content_cas_path,
archive_tree_id_file,
/*is_absent = */ true,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
critical_git_op_map,
import_to_git_map,
@@ -1007,11 +1107,11 @@ auto CreateContentGitMap(
GitOpKey op_key = {
.params =
{
- storage_config->GitRoot(), // target_path
- "", // git_hash
- std::nullopt, // message
- std::nullopt, // source_path
- true // init_bare
+ native_storage_config->GitRoot(), // target_path
+ "", // git_hash
+ std::nullopt, // message
+ std::nullopt, // source_path
+ true // init_bare
},
.op_type = GitOpType::ENSURE_INIT};
critical_git_op_map->ConsumeAfterKeysReady(
@@ -1027,8 +1127,11 @@ auto CreateContentGitMap(
additional_mirrors,
ca_info,
serve,
- storage,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ native_storage,
+ compat_storage,
+ local_api,
remote_api,
progress,
ts,
@@ -1068,11 +1171,12 @@ auto CreateContentGitMap(
// blob check failed
return;
}
- auto const& cas = storage->CAS();
+ auto const& native_cas = native_storage->CAS();
if (res.second) {
// blob found; add it to CAS
- if (not cas.StoreBlob(*res.second,
- /*is_executable=*/false)) {
+ if (not native_cas.StoreBlob(
+ *res.second,
+ /*is_executable=*/false)) {
(*logger)(fmt::format(
"Failed to store content "
"{} to local CAS",
@@ -1080,14 +1184,17 @@ auto CreateContentGitMap(
/*fatal=*/true);
return;
}
- if (auto content_cas_path = cas.BlobPath(
+ if (auto content_cas_path = native_cas.BlobPath(
digest, /*is_executable=*/false)) {
ExtractAndImportToGit(key,
*content_cas_path,
archive_tree_id_file,
/*is_absent=*/true,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
critical_git_op_map,
import_to_git_map,
@@ -1115,17 +1222,20 @@ auto CreateContentGitMap(
.filename()
.string());
StorageUtils::AddDistfileToCAS(
- *storage, repo_distfile, just_mr_paths);
+ *native_storage, repo_distfile, just_mr_paths);
// check if content is in CAS now
- if (auto content_cas_path =
- cas.BlobPath(digest, /*is_executable=*/false)) {
+ if (auto content_cas_path = native_cas.BlobPath(
+ digest, /*is_executable=*/false)) {
progress->TaskTracker().Stop(key.archive.origin);
ExtractAndImportToGit(key,
*content_cas_path,
archive_tree_id_file,
/*is_absent=*/true,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
critical_git_op_map,
import_to_git_map,
@@ -1142,7 +1252,7 @@ auto CreateContentGitMap(
key.archive.content_hash.Hash()),
/*fatal=*/true);
},
- [logger, target_path = storage_config->GitRoot()](
+ [logger, target_path = native_storage_config->GitRoot()](
auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git op "
"ENSURE_INIT for target {}:\n{}",
@@ -1158,8 +1268,8 @@ auto CreateContentGitMap(
{key.archive},
[archive_tree_id_file,
key,
- storage,
- storage_config,
+ native_storage_config,
+ native_storage,
critical_git_op_map,
import_to_git_map,
resolve_symlinks_map,
@@ -1167,11 +1277,12 @@ auto CreateContentGitMap(
setter,
logger]([[maybe_unused]] auto const& values) {
// content is in local CAS now
- auto const& cas = storage->CAS();
+ auto const& native_cas = native_storage->CAS();
auto content_cas_path =
- cas.BlobPath(
- ArtifactDigest{key.archive.content_hash, 0},
- /*is_executable=*/false)
+ native_cas
+ .BlobPath(
+ ArtifactDigest{key.archive.content_hash, 0},
+ /*is_executable=*/false)
.value();
// root can only be present, so default all arguments
// that refer to a serve endpoint
@@ -1180,7 +1291,10 @@ auto CreateContentGitMap(
archive_tree_id_file,
/*is_absent=*/false,
/*serve=*/nullptr,
- storage_config,
+ native_storage_config,
+ /*compat_storage_config=*/nullptr,
+ /*compat_storage=*/nullptr,
+ /*local_api=*/nullptr,
/*remote_api=*/nullptr,
critical_git_op_map,
import_to_git_map,
diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp
index e969f20a..52e6929e 100644
--- a/src/other_tools/root_maps/content_git_map.hpp
+++ b/src/other_tools/root_maps/content_git_map.hpp
@@ -47,8 +47,11 @@ using ContentGitMap =
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<Storage const*> const& storage,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ gsl::not_null<Storage const*> const& native_storage,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
bool fetch_absent,
gsl::not_null<JustMRProgress*> const& progress,
diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp
index a19889b2..82c0c623 100644
--- a/src/other_tools/root_maps/distdir_git_map.cpp
+++ b/src/other_tools/root_maps/distdir_git_map.cpp
@@ -74,15 +74,15 @@ namespace {
/// the setter on success.
void ImportFromCASAndSetRoot(
DistdirInfo const& key,
- StorageConfig const& storage_config,
- Storage const& storage,
+ StorageConfig const& native_storage_config,
+ Storage const& native_storage,
std::filesystem::path const& distdir_tree_id_file,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
gsl::not_null<TaskSystem*> const& ts,
DistdirGitMap::SetterPtr const& setter,
DistdirGitMap::LoggerPtr const& logger) {
// create the links to CAS
- auto tmp_dir = storage_config.CreateTypedTmpDir("distdir");
+ auto tmp_dir = native_storage_config.CreateTypedTmpDir("distdir");
if (not tmp_dir) {
(*logger)(fmt::format("Failed to create tmp path for "
"distdir target {}",
@@ -91,7 +91,7 @@ void ImportFromCASAndSetRoot(
return;
}
// link content from CAS into tmp dir
- if (not LinkToCAS(storage, key.content_list, tmp_dir->GetPath())) {
+ if (not LinkToCAS(native_storage, key.content_list, tmp_dir->GetPath())) {
(*logger)(fmt::format("Failed to create links to CAS content!",
key.content_id),
/*fatal=*/true);
@@ -104,7 +104,7 @@ void ImportFromCASAndSetRoot(
{std::move(c_info)},
[tmp_dir, // keep tmp_dir alive
distdir_tree_id_file,
- git_root = storage_config.GitRoot().string(),
+ git_root = native_storage_config.GitRoot().string(),
setter,
logger](auto const& values) {
// check for errors
@@ -145,8 +145,10 @@ auto CreateDistdirGitMap(
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<Storage const*> const& storage,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ gsl::not_null<Storage const*> const& native_storage,
+ Storage const* compat_storage,
gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
std::size_t jobs) -> DistdirGitMap {
@@ -154,16 +156,18 @@ auto CreateDistdirGitMap(
import_to_git_map,
critical_git_op_map,
serve,
- storage,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ native_storage,
+ compat_storage,
local_api,
remote_api](auto ts,
auto setter,
auto logger,
auto /* unused */,
auto const& key) {
- auto distdir_tree_id_file =
- StorageUtils::GetDistdirTreeIDFile(*storage_config, key.content_id);
+ auto distdir_tree_id_file = StorageUtils::GetDistdirTreeIDFile(
+ *native_storage_config, key.content_id);
if (FileSystemManager::Exists(distdir_tree_id_file)) {
// read distdir_tree_id from file tree_id_file
auto distdir_tree_id =
@@ -179,11 +183,11 @@ auto CreateDistdirGitMap(
GitOpKey op_key = {
.params =
{
- storage_config->GitRoot(), // target_path
- "", // git_hash
- std::nullopt, // message
- std::nullopt, // source_path
- true // init_bare
+ native_storage_config->GitRoot(), // target_path
+ "", // git_hash
+ std::nullopt, // message
+ std::nullopt, // source_path
+ true // init_bare
},
.op_type = GitOpType::ENSURE_INIT};
critical_git_op_map->ConsumeAfterKeysReady(
@@ -193,7 +197,10 @@ auto CreateDistdirGitMap(
content_id = key.content_id,
key,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
setter,
logger](auto const& values) {
@@ -268,7 +275,12 @@ auto CreateDistdirGitMap(
if (not EnsureAbsentRootOnServe(
*serve,
distdir_tree_id,
- storage_config->GitRoot(),
+ native_storage_config
+ ->GitRoot(), /*repo_root*/
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ &*local_api,
remote_api,
logger,
true /*no_sync_is_fatal*/)) {
@@ -293,15 +305,15 @@ auto CreateDistdirGitMap(
}
else {
// set root as present
- (*setter)(
- std::pair(nlohmann::json::array(
- {FileRoot::kGitTreeMarker,
- distdir_tree_id,
- storage_config->GitRoot().string()}),
- /*is_cache_hit=*/true));
+ (*setter)(std::pair(
+ nlohmann::json::array(
+ {FileRoot::kGitTreeMarker,
+ distdir_tree_id,
+ native_storage_config->GitRoot().string()}),
+ /*is_cache_hit=*/true));
}
},
- [logger, target_path = storage_config->GitRoot()](
+ [logger, target_path = native_storage_config->GitRoot()](
auto const& msg, bool fatal) {
(*logger)(fmt::format("While running critical Git op "
"ENSURE_INIT for target {}:\n{}",
@@ -415,6 +427,10 @@ auto CreateDistdirGitMap(
*serve,
tree_id,
/*repo_path=*/"",
+ native_storage_config,
+ /*compat_storage_config=*/nullptr,
+ /*compat_storage=*/nullptr,
+ /*local_api=*/nullptr,
/*remote_api=*/nullptr,
logger,
/*no_sync_is_fatal=*/true)) {
@@ -452,6 +468,10 @@ auto CreateDistdirGitMap(
*serve,
tree_id,
/*repo_path=*/"",
+ native_storage_config,
+ /*compat_storage_config=*/nullptr,
+ /*compat_storage=*/nullptr,
+ /*local_api=*/nullptr,
/*remote_api=*/nullptr,
logger,
/*no_sync_is_fatal=*/true)) {
@@ -487,8 +507,8 @@ auto CreateDistdirGitMap(
// first, look in the local CAS
if (digest and local_api->IsAvailable({*digest})) {
ImportFromCASAndSetRoot(key,
- *storage_config,
- *storage,
+ *native_storage_config,
+ *native_storage,
distdir_tree_id_file,
import_to_git_map,
ts,
@@ -541,15 +561,15 @@ auto CreateDistdirGitMap(
[distdir_tree_id_file,
key,
import_to_git_map,
+ native_storage_config,
+ native_storage,
ts,
- storage,
- storage_config,
setter,
logger]([[maybe_unused]] auto const& values) {
// archive blobs are in CAS
ImportFromCASAndSetRoot(key,
- *storage_config,
- *storage,
+ *native_storage_config,
+ *native_storage,
distdir_tree_id_file,
import_to_git_map,
ts,
diff --git a/src/other_tools/root_maps/distdir_git_map.hpp b/src/other_tools/root_maps/distdir_git_map.hpp
index 9244f341..2dc678f8 100644
--- a/src/other_tools/root_maps/distdir_git_map.hpp
+++ b/src/other_tools/root_maps/distdir_git_map.hpp
@@ -57,8 +57,10 @@ using DistdirGitMap =
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
- gsl::not_null<Storage const*> const& storage,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ gsl::not_null<Storage const*> const& native_storage,
+ Storage const* compat_storage,
gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
std::size_t jobs) -> DistdirGitMap;
diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp
index 023dedf8..ca8bdd52 100644
--- a/src/other_tools/root_maps/fpath_git_map.cpp
+++ b/src/other_tools/root_maps/fpath_git_map.cpp
@@ -31,13 +31,18 @@ namespace {
/// \brief Does the serve endpoint checks and sets the workspace root.
/// It guarantees the logger is called exactly once with fatal on failure, and
/// the setter on success.
-void CheckServeAndSetRoot(std::string const& tree_id,
- std::string const& repo_root,
- bool absent,
- ServeApi const* serve,
- IExecutionApi const* remote_api,
- FilePathGitMap::SetterPtr const& ws_setter,
- FilePathGitMap::LoggerPtr const& logger) {
+void CheckServeAndSetRoot(
+ std::string const& tree_id,
+ std::string const& repo_root,
+ bool absent,
+ ServeApi const* serve,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
+ IExecutionApi const* remote_api,
+ FilePathGitMap::SetterPtr const& ws_setter,
+ FilePathGitMap::LoggerPtr const& logger) {
// if serve endpoint is given, try to ensure it has this tree available to
// be able to build against it. If root is not absent, do not fail if we
// don't have a suitable remote endpoint, but warn user nonetheless.
@@ -63,6 +68,10 @@ void CheckServeAndSetRoot(std::string const& tree_id,
if (not EnsureAbsentRootOnServe(*serve,
tree_id,
repo_root,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
logger,
/*no_sync_is_fatal=*/absent)) {
@@ -99,7 +108,10 @@ void ResolveFilePathTree(
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
gsl::not_null<TaskSystem*> const& ts,
FilePathGitMap::SetterPtr const& ws_setter,
@@ -107,7 +119,7 @@ void ResolveFilePathTree(
if (pragma_special) {
// get the resolved tree
auto tree_id_file = StorageUtils::GetResolvedTreeIDFile(
- *storage_config, tree_hash, *pragma_special);
+ *native_storage_config, tree_hash, *pragma_special);
if (FileSystemManager::Exists(tree_id_file)) {
// read resolved tree id
auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file);
@@ -122,9 +134,13 @@ void ResolveFilePathTree(
// available to be able to build against it; the tree is resolved,
// so it is in our Git cache
CheckServeAndSetRoot(*resolved_tree_id,
- storage_config->GitRoot().string(),
+ native_storage_config->GitRoot().string(),
absent,
serve,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ws_setter,
logger);
@@ -145,7 +161,10 @@ void ResolveFilePathTree(
tree_id_file,
absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ts,
ws_setter,
@@ -155,7 +174,8 @@ void ResolveFilePathTree(
GitOpKey op_key = {
.params =
{
- storage_config->GitRoot(), // target_path
+ native_storage_config
+ ->GitRoot(), // target_path
resolved_tree_id, // git_hash
"Keep referenced tree alive" // message
},
@@ -167,7 +187,10 @@ void ResolveFilePathTree(
tree_id_file,
absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ws_setter,
logger](auto const& values) {
@@ -193,14 +216,19 @@ void ResolveFilePathTree(
// it; the resolved tree is in the Git cache
CheckServeAndSetRoot(
resolved_tree_id,
- storage_config->GitRoot().string(),
+ native_storage_config->GitRoot().string(),
absent,
serve,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ws_setter,
logger);
},
- [logger, target_path = storage_config->GitRoot()](
+ [logger,
+ target_path = native_storage_config->GitRoot()](
auto const& msg, bool fatal) {
(*logger)(
fmt::format("While running critical Git op "
@@ -223,8 +251,17 @@ void ResolveFilePathTree(
// tree needs no further processing;
// if serve endpoint is given, try to ensure it has this tree available
// to be able to build against it
- CheckServeAndSetRoot(
- tree_hash, repo_root, absent, serve, remote_api, ws_setter, logger);
+ CheckServeAndSetRoot(tree_hash,
+ repo_root,
+ absent,
+ serve,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
+ remote_api,
+ ws_setter,
+ logger);
}
}
@@ -236,7 +273,10 @@ auto CreateFilePathGitMap(
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
std::size_t jobs,
std::string const& multi_repo_tool_name,
@@ -246,7 +286,10 @@ auto CreateFilePathGitMap(
import_to_git_map,
resolve_symlinks_map,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
multi_repo_tool_name,
build_tool_name](auto ts,
@@ -285,7 +328,10 @@ auto CreateFilePathGitMap(
critical_git_op_map,
resolve_symlinks_map,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ts,
setter,
@@ -324,16 +370,16 @@ auto CreateFilePathGitMap(
// resolve tree and set workspace root; tree gets resolved
// from source repo into the Git cache, which we first need
// to ensure is initialized
- GitOpKey op_key = {
- .params =
- {
- storage_config->GitRoot(), // target_path
- "", // git_hash
- std::nullopt, // message
- std::nullopt, // source_path
- true // init_bare
- },
- .op_type = GitOpType::ENSURE_INIT};
+ GitOpKey op_key = {.params =
+ {
+ native_storage_config
+ ->GitRoot(), // target_path
+ "", // git_hash
+ std::nullopt, // message
+ std::nullopt, // source_path
+ true // init_bare
+ },
+ .op_type = GitOpType::ENSURE_INIT};
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
@@ -346,7 +392,10 @@ auto CreateFilePathGitMap(
critical_git_op_map,
resolve_symlinks_map,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ts,
setter,
@@ -369,13 +418,17 @@ auto CreateFilePathGitMap(
critical_git_op_map,
resolve_symlinks_map,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ts,
setter,
logger);
},
- [logger, target_path = storage_config->GitRoot()](
+ [logger,
+ target_path = native_storage_config->GitRoot()](
auto const& msg, bool fatal) {
(*logger)(
fmt::format("While running critical Git op "
@@ -408,7 +461,7 @@ auto CreateFilePathGitMap(
/*fatal=*/false);
}
// it's not a git repo, so import it to git cache
- auto tmp_dir = storage_config->CreateTypedTmpDir("file");
+ auto tmp_dir = native_storage_config->CreateTypedTmpDir("file");
if (not tmp_dir) {
(*logger)("Failed to create import-to-git tmp directory!",
/*fatal=*/true);
@@ -437,7 +490,10 @@ auto CreateFilePathGitMap(
critical_git_op_map,
resolve_symlinks_map,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
remote_api,
ts,
setter,
@@ -452,21 +508,25 @@ auto CreateFilePathGitMap(
std::string tree = values[0]->first;
// resolve tree and set workspace root;
// we work on the Git CAS directly
- ResolveFilePathTree(storage_config->GitRoot().string(),
- fpath.string(),
- tree,
- pragma_special,
- values[0]->second, /*source_cas*/
- values[0]->second, /*target_cas*/
- absent,
- critical_git_op_map,
- resolve_symlinks_map,
- serve,
- storage_config,
- remote_api,
- ts,
- setter,
- logger);
+ ResolveFilePathTree(
+ native_storage_config->GitRoot().string(),
+ fpath.string(),
+ tree,
+ pragma_special,
+ values[0]->second, /*source_cas*/
+ values[0]->second, /*target_cas*/
+ absent,
+ critical_git_op_map,
+ resolve_symlinks_map,
+ serve,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api,
+ remote_api,
+ ts,
+ setter,
+ logger);
},
[logger, target_path = key.fpath](auto const& msg, bool fatal) {
(*logger)(
diff --git a/src/other_tools/root_maps/fpath_git_map.hpp b/src/other_tools/root_maps/fpath_git_map.hpp
index cb13da5c..ad2f4c39 100644
--- a/src/other_tools/root_maps/fpath_git_map.hpp
+++ b/src/other_tools/root_maps/fpath_git_map.hpp
@@ -27,6 +27,7 @@
#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp"
#include "src/buildtool/serve_api/remote/serve_api.hpp"
#include "src/buildtool/storage/config.hpp"
+#include "src/buildtool/storage/storage.hpp"
#include "src/other_tools/just_mr/utils.hpp"
#include "src/other_tools/ops_maps/import_to_git_map.hpp"
#include "src/utils/cpp/hash_combine.hpp"
@@ -56,7 +57,10 @@ using FilePathGitMap = AsyncMapConsumer<FpathInfo, nlohmann::json>;
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
std::size_t jobs,
std::string const& multi_repo_tool_name,
diff --git a/src/other_tools/root_maps/root_utils.cpp b/src/other_tools/root_maps/root_utils.cpp
index 9dd7badf..0e61c548 100644
--- a/src/other_tools/root_maps/root_utils.cpp
+++ b/src/other_tools/root_maps/root_utils.cpp
@@ -20,7 +20,7 @@
#include "src/buildtool/common/artifact_digest_factory.hpp"
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/crypto/hash_function.hpp"
-#include "src/buildtool/execution_api/git/git_api.hpp"
+#include "src/buildtool/execution_api/serve/mr_git_api.hpp"
#include "src/buildtool/file_system/object_type.hpp"
auto CheckServeHasAbsentRoot(ServeApi const& serve,
@@ -37,12 +37,17 @@ auto CheckServeHasAbsentRoot(ServeApi const& serve,
return std::nullopt;
}
-auto EnsureAbsentRootOnServe(ServeApi const& serve,
- std::string const& tree_id,
- std::filesystem::path const& repo_path,
- IExecutionApi const* remote_api,
- AsyncMapConsumerLoggerPtr const& logger,
- bool no_sync_is_fatal) -> bool {
+auto EnsureAbsentRootOnServe(
+ ServeApi const& serve,
+ std::string const& tree_id,
+ std::filesystem::path const& repo_path,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
+ IExecutionApi const* remote_api,
+ AsyncMapConsumerLoggerPtr const& logger,
+ bool no_sync_is_fatal) -> bool {
if (remote_api != nullptr) {
// upload tree to remote CAS
auto repo = RepositoryConfig{};
@@ -55,7 +60,11 @@ auto EnsureAbsentRootOnServe(ServeApi const& serve,
auto const digest = ArtifactDigestFactory::Create(
HashFunction::Type::GitSHA1, tree_id, 0, /*is_tree=*/true);
- auto git_api = GitApi{&repo};
+ auto git_api = MRGitApi{&repo,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ local_api};
if (not digest or not git_api.RetrieveToCas(
{Artifact::ObjectInfo{.digest = *digest,
.type = ObjectType::Tree}},
diff --git a/src/other_tools/root_maps/root_utils.hpp b/src/other_tools/root_maps/root_utils.hpp
index c055633f..2d3ec6a7 100644
--- a/src/other_tools/root_maps/root_utils.hpp
+++ b/src/other_tools/root_maps/root_utils.hpp
@@ -22,6 +22,8 @@
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/multithreading/async_map_consumer.hpp"
#include "src/buildtool/serve_api/remote/serve_api.hpp"
+#include "src/buildtool/storage/config.hpp"
+#include "src/buildtool/storage/storage.hpp"
/// \brief Calls the ServeApi to check whether the serve endpoint has the given
/// tree available to build against.
@@ -43,9 +45,16 @@
/// used by given remote execution endpoint!
/// \param tree_id The Git-tree identifier.
/// \param repo_path Local witnessing Git repository for the tree.
-/// \param remote_api Optional API of the remote-execution endpoint. If nullopt,
-/// skip the upload to the remote CAS; this assumes prior knowledge which
-/// guarantees the tree given by tree_id exists in the remote CAS for the
+/// \param native_storage_config Configuration of the native local storage.
+/// \param compat_storage_config Optional configuration of the compatible local
+/// storage, if it was set up.
+/// \param compat_storage Optional compatible local storage, if it was set up.
+/// \param local_api Optional API that knows how to communicate with the
+/// remote-execution endpoint specified by parameter remote_api, if given. In
+/// particular, it is expected to be provided if the remote is compatible.
+/// \param remote_api Optional API of the remote-execution endpoint.
+/// If nullopt, skip the upload to the remote CAS; this assumes prior knowledge
+/// which guarantees the tree given by tree_id exists in the remote CAS for the
/// duration of the subsequent serve API call; this option should be used
/// carefully, but does result in less remote communication.
/// \param logger An AsyncMapConsumer logger instance.
@@ -58,6 +67,10 @@
ServeApi const& serve,
std::string const& tree_id,
std::filesystem::path const& repo_path,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ IExecutionApi const* local_api,
IExecutionApi const* remote_api,
AsyncMapConsumerLoggerPtr const& logger,
bool no_sync_is_fatal) -> bool;
diff --git a/src/other_tools/root_maps/tree_id_git_map.cpp b/src/other_tools/root_maps/tree_id_git_map.cpp
index 76b29045..7b3099ae 100644
--- a/src/other_tools/root_maps/tree_id_git_map.cpp
+++ b/src/other_tools/root_maps/tree_id_git_map.cpp
@@ -18,7 +18,7 @@
#include "src/buildtool/common/artifact_digest.hpp"
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/crypto/hash_info.hpp"
-#include "src/buildtool/execution_api/git/git_api.hpp"
+#include "src/buildtool/execution_api/serve/mr_git_api.hpp"
#include "src/buildtool/file_system/file_root.hpp"
#include "src/other_tools/root_maps/root_utils.hpp"
@@ -26,18 +26,27 @@ namespace {
/// \brief Guarantees it terminates by either calling the setter or calling the
/// logger with fatal.
-void UploadToServeAndSetRoot(ServeApi const& serve,
- StorageConfig const& storage_config,
- std::string const& tree_id,
- ArtifactDigest const& digest,
- IExecutionApi const& remote_api,
- bool ignore_special,
- TreeIdGitMap::SetterPtr const& setter,
- TreeIdGitMap::LoggerPtr const& logger) {
+void UploadToServeAndSetRoot(
+ ServeApi const& serve,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
+ std::string const& tree_id,
+ ArtifactDigest const& digest,
+ gsl::not_null<IExecutionApi const*> const& local_api,
+ IExecutionApi const& remote_api,
+ bool ignore_special,
+ TreeIdGitMap::SetterPtr const& setter,
+ TreeIdGitMap::LoggerPtr const& logger) {
// upload to remote CAS
auto repo_config = RepositoryConfig{};
- if (repo_config.SetGitCAS(storage_config.GitRoot())) {
- auto git_api = GitApi{&repo_config};
+ if (repo_config.SetGitCAS(native_storage_config->GitRoot())) {
+ auto git_api =
+ MRGitApi{&repo_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
+ compat_storage_config != nullptr ? &*local_api : nullptr};
if (not git_api.RetrieveToCas(
{Artifact::ObjectInfo{.digest = digest,
.type = ObjectType::Tree}},
@@ -51,7 +60,7 @@ void UploadToServeAndSetRoot(ServeApi const& serve,
}
else {
(*logger)(fmt::format("Failed to SetGitCAS at {}",
- storage_config.GitRoot().string()),
+ native_storage_config->GitRoot().string()),
/*fatal=*/true);
return;
}
@@ -60,6 +69,10 @@ void UploadToServeAndSetRoot(ServeApi const& serve,
if (EnsureAbsentRootOnServe(serve,
tree_id,
/*repo_path=*/"",
+ native_storage_config,
+ /*compat_storage_config=*/nullptr,
+ /*compat_storage=*/nullptr,
+ /*local_api=*/nullptr,
/*remote_api=*/nullptr,
logger,
/*no_sync_is_fatal=*/true)) {
@@ -77,7 +90,9 @@ void UploadToServeAndSetRoot(ServeApi const& serve,
/// logger with fatal.
void MoveCASTreeToGitAndProcess(
ServeApi const& serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
std::string const& tree_id,
ArtifactDigest const& digest,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
@@ -88,7 +103,8 @@ void MoveCASTreeToGitAndProcess(
TreeIdGitMap::SetterPtr const& setter,
TreeIdGitMap::LoggerPtr const& logger) {
// Move tree from CAS to local Git storage
- auto tmp_dir = storage_config->CreateTypedTmpDir("fetch-remote-git-tree");
+ auto tmp_dir =
+ native_storage_config->CreateTypedTmpDir("fetch-remote-git-tree");
if (not tmp_dir) {
(*logger)(fmt::format("Failed to create tmp directory for copying "
"git-tree {} from remote CAS",
@@ -110,10 +126,13 @@ void MoveCASTreeToGitAndProcess(
ts,
{std::move(c_info)},
[&serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
tmp_dir, // keep tmp_dir alive
tree_id,
digest,
+ local_api,
remote_api,
ignore_special,
setter,
@@ -126,9 +145,12 @@ void MoveCASTreeToGitAndProcess(
// upload tree from Git cache to remote CAS and tell serve to set up
// the root from the remote CAS tree; set root as absent on success
UploadToServeAndSetRoot(serve,
- *storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
tree_id,
digest,
+ local_api,
*remote_api,
ignore_special,
setter,
@@ -152,7 +174,9 @@ auto CreateTreeIdGitMap(
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
bool fetch_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
std::size_t jobs) -> TreeIdGitMap {
@@ -161,7 +185,9 @@ auto CreateTreeIdGitMap(
import_to_git_map,
fetch_absent,
serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
local_api,
remote_api](auto ts,
auto setter,
@@ -205,12 +231,17 @@ auto CreateTreeIdGitMap(
if (remote_api->IsAvailable({digest})) {
// tell serve to set up the root from the remote CAS tree;
// upload can be skipped
- if (EnsureAbsentRootOnServe(*serve,
- key.tree_info.tree_hash.Hash(),
- /*repo_path=*/"",
- /*remote_api=*/nullptr,
- logger,
- /*no_sync_is_fatal=*/true)) {
+ if (EnsureAbsentRootOnServe(
+ *serve,
+ key.tree_info.tree_hash.Hash(),
+ /*repo_path=*/"",
+ native_storage_config,
+ /*compat_storage_config=*/nullptr,
+ /*compat_storage=*/nullptr,
+ /*local_api=*/nullptr,
+ /*remote_api=*/nullptr,
+ logger,
+ /*no_sync_is_fatal=*/true)) {
// set workspace root as absent
auto root = nlohmann::json::array(
{key.ignore_special
@@ -233,18 +264,20 @@ auto CreateTreeIdGitMap(
GitOpKey op_key = {
.params =
{
- storage_config->GitRoot(), // target_path
- "", // git_hash
- std::nullopt, // message
- std::nullopt, // source_path
- true // init_bare
+ native_storage_config->GitRoot(), // target_path
+ "", // git_hash
+ std::nullopt, // message
+ std::nullopt, // source_path
+ true // init_bare
},
.op_type = GitOpType::ENSURE_INIT};
critical_git_op_map->ConsumeAfterKeysReady(
ts,
{std::move(op_key)},
[serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
digest,
import_to_git_map,
local_api,
@@ -266,8 +299,9 @@ auto CreateTreeIdGitMap(
op_result.git_cas); // link fake repo to odb
if (not git_repo) {
(*logger)(
- fmt::format("Could not open repository {}",
- storage_config->GitRoot().string()),
+ fmt::format(
+ "Could not open repository {}",
+ native_storage_config->GitRoot().string()),
/*fatal=*/true);
return;
}
@@ -294,9 +328,12 @@ auto CreateTreeIdGitMap(
// tree, then set root as absent
UploadToServeAndSetRoot(
*serve,
- *storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
key.tree_info.tree_hash.Hash(),
digest,
+ local_api,
*remote_api,
key.ignore_special,
setter,
@@ -310,7 +347,9 @@ auto CreateTreeIdGitMap(
// continue processing it by UploadToServeAndSetRoot
MoveCASTreeToGitAndProcess(
*serve,
- storage_config,
+ native_storage_config,
+ compat_storage_config,
+ compat_storage,
key.tree_info.tree_hash.Hash(),
digest,
import_to_git_map,
@@ -332,7 +371,7 @@ auto CreateTreeIdGitMap(
key.tree_info.tree_hash.Hash()),
/*fatal=*/true);
},
- [logger, target_path = storage_config->GitRoot()](
+ [logger, target_path = native_storage_config->GitRoot()](
auto const& msg, bool fatal) {
(*logger)(
fmt::format("While running critical Git op "
@@ -363,7 +402,7 @@ auto CreateTreeIdGitMap(
git_tree_fetch_map->ConsumeAfterKeysReady(
ts,
{key.tree_info},
- [storage_config, key, setter](auto const& values) {
+ [native_storage_config, key, setter](auto const& values) {
// tree is now in Git cache;
// get cache hit info
auto is_cache_hit = *values[0];
@@ -374,7 +413,7 @@ auto CreateTreeIdGitMap(
? FileRoot::kGitTreeIgnoreSpecialMarker
: FileRoot::kGitTreeMarker,
key.tree_info.tree_hash.Hash(),
- storage_config->GitRoot().string()}),
+ native_storage_config->GitRoot().string()}),
is_cache_hit));
},
[logger, hash = key.tree_info.tree_hash.Hash()](auto const& msg,
diff --git a/src/other_tools/root_maps/tree_id_git_map.hpp b/src/other_tools/root_maps/tree_id_git_map.hpp
index faec4379..db7dfd68 100644
--- a/src/other_tools/root_maps/tree_id_git_map.hpp
+++ b/src/other_tools/root_maps/tree_id_git_map.hpp
@@ -26,6 +26,7 @@
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/serve_api/remote/serve_api.hpp"
#include "src/buildtool/storage/config.hpp"
+#include "src/buildtool/storage/storage.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
#include "src/other_tools/ops_maps/git_tree_fetch_map.hpp"
#include "src/other_tools/ops_maps/import_to_git_map.hpp"
@@ -71,7 +72,9 @@ using TreeIdGitMap =
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
bool fetch_absent,
ServeApi const* serve,
- gsl::not_null<StorageConfig const*> const& storage_config,
+ gsl::not_null<StorageConfig const*> const& native_storage_config,
+ StorageConfig const* compat_storage_config,
+ Storage const* compat_storage,
gsl::not_null<IExecutionApi const*> const& local_api,
IExecutionApi const* remote_api,
std::size_t jobs) -> TreeIdGitMap;