diff options
22 files changed, 239 insertions, 106 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index b355641b..176acea5 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -43,6 +43,8 @@ , ["src/buildtool/auth", "auth"] , ["src/buildtool/common", "config"] , ["src/buildtool/common/remote", "remote_common"] + , ["src/buildtool/storage", "config"] + , ["src/buildtool/storage", "storage"] ] , "private-deps": [ ["src/buildtool/execution_api/bazel_msg", "bazel_msg"] diff --git a/src/buildtool/execution_api/common/api_bundle.cpp b/src/buildtool/execution_api/common/api_bundle.cpp index 41637eb4..933ac1f3 100644 --- a/src/buildtool/execution_api/common/api_bundle.cpp +++ b/src/buildtool/execution_api/common/api_bundle.cpp @@ -18,10 +18,14 @@ #include "src/buildtool/execution_api/local/local_api.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" -ApiBundle::ApiBundle(RepositoryConfig const* repo_config, +ApiBundle::ApiBundle(gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, + RepositoryConfig const* repo_config, gsl::not_null<Auth const*> const& authentication, std::optional<ServerAddress> const& remote_address) - : local{std::make_shared<LocalApi>(repo_config)}, // needed by remote + : local{std::make_shared<LocalApi>(storage_config, + storage, + repo_config)}, // needed by remote auth{*authentication}, // needed by remote remote{CreateRemote(remote_address)} {} diff --git a/src/buildtool/execution_api/common/api_bundle.hpp b/src/buildtool/execution_api/common/api_bundle.hpp index 0148b6af..3c084c0d 100644 --- a/src/buildtool/execution_api/common/api_bundle.hpp +++ b/src/buildtool/execution_api/common/api_bundle.hpp @@ -23,14 +23,19 @@ #include "src/buildtool/common/remote/remote_common.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" /// \brief Utility structure for instantiation of local and remote apis at the /// same time. If the remote api cannot be instantiated, it falls back to /// exactly the same instance that local api is (&*remote == & *local). struct ApiBundle final { - explicit ApiBundle(RepositoryConfig const* repo_config, - gsl::not_null<Auth const*> const& authentication, - std::optional<ServerAddress> const& remote_address); + explicit ApiBundle( + gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, + RepositoryConfig const* repo_config, + gsl::not_null<Auth const*> const& authentication, + std::optional<ServerAddress> const& remote_address); [[nodiscard]] auto CreateRemote(std::optional<ServerAddress> const& address) const -> gsl::not_null<IExecutionApi::Ptr>; diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index f29e0c3a..64f8bb6e 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -33,6 +33,7 @@ , ["src/buildtool/common", "common"] , ["src/buildtool/common", "config"] , ["src/buildtool/storage", "storage"] + , ["src/buildtool/storage", "config"] , ["src/buildtool/execution_api/common", "common"] , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index 86192146..060390cb 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -31,7 +31,6 @@ #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" -#include "src/buildtool/storage/config.hpp" #include "src/buildtool/system/system_command.hpp" namespace { @@ -58,9 +57,9 @@ class BuildCleanupAnchor { }; [[nodiscard]] auto CreateDigestFromLocalOwnedTree( - gsl::not_null<Storage const*> const& storage, + Storage const& storage, std::filesystem::path const& dir_path) -> std::optional<bazel_re::Digest> { - auto const& cas = storage->CAS(); + auto const& cas = storage.CAS(); auto store_blob = [&cas](std::filesystem::path const& path, auto is_exec) -> std::optional<bazel_re::Digest> { return cas.StoreBlob</*kOwner=*/true>(path, is_exec); @@ -97,14 +96,14 @@ auto LocalAction::Execute(Logger const* logger) noexcept } if (do_cache) { - if (auto result = storage_->ActionCache().CachedResult(action)) { + if (auto result = storage_.ActionCache().CachedResult(action)) { if (result->exit_code() == 0 and ActionResultContainsExpectedOutputs( *result, output_files_, output_dirs_)) { return IExecutionResponse::Ptr{ new LocalResponse{action.hash(), {std::move(*result), /*is_cached=*/true}, - storage_}}; + &storage_}}; } } } @@ -116,10 +115,10 @@ auto LocalAction::Execute(Logger const* logger) noexcept auto action_id = CreateActionDigest(root_digest_, false).hash(); output->is_cached = true; return IExecutionResponse::Ptr{new LocalResponse{ - std::move(action_id), std::move(*output), storage_}}; + std::move(action_id), std::move(*output), &storage_}}; } - return IExecutionResponse::Ptr{ - new LocalResponse{action.hash(), std::move(*output), storage_}}; + return IExecutionResponse::Ptr{new LocalResponse{ + action.hash(), std::move(*output), &storage_}}; } } @@ -129,7 +128,7 @@ auto LocalAction::Execute(Logger const* logger) noexcept auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept -> std::optional<Output> { auto exec_path = - CreateUniquePath(StorageConfig::Instance().ExecutionRoot() / + CreateUniquePath(storage_config_.ExecutionRoot() / NativeSupport::Unprefix(action_id.hash())); if (not exec_path) { @@ -169,8 +168,8 @@ auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept if (CollectAndStoreOutputs(&result.action, build_root)) { if (cache_flag_ == CacheFlag::CacheOutput) { - if (not storage_->ActionCache().StoreResult(action_id, - result.action)) { + if (not storage_.ActionCache().StoreResult(action_id, + result.action)) { logger_.Emit(LogLevel::Warning, "failed to store action results"); } @@ -192,7 +191,7 @@ auto LocalAction::StageInput(std::filesystem::path const& target_path, } auto blob_path = - storage_->CAS().BlobPath(info.digest, IsExecutableObject(info.type)); + storage_.CAS().BlobPath(info.digest, IsExecutableObject(info.type)); if (not blob_path) { logger_.Emit(LogLevel::Error, @@ -222,7 +221,7 @@ auto LocalAction::StageInputs( if (FileSystemManager::IsRelativePath(exec_path)) { return false; } - auto reader = TreeReader<LocalCasReader>{storage_->CAS()}; + auto reader = TreeReader<LocalCasReader>{storage_.CAS()}; auto result = reader.RecursivelyReadTreeLeafs( root_digest_, exec_path, /*include_trees=*/true); if (not result) { @@ -291,7 +290,7 @@ auto LocalAction::CollectOutputFileOrSymlink( } if (IsSymlinkObject(*type)) { auto content = FileSystemManager::ReadSymlink(file_path); - if (content and storage_->CAS().StoreBlob(*content)) { + if (content and storage_.CAS().StoreBlob(*content)) { auto out_symlink = bazel_re::OutputSymlink{}; out_symlink.set_path(local_path); out_symlink.set_target(*content); @@ -300,8 +299,8 @@ auto LocalAction::CollectOutputFileOrSymlink( } else if (IsFileObject(*type)) { bool is_executable = IsExecutableObject(*type); - auto digest = storage_->CAS().StoreBlob</*kOwner=*/true>(file_path, - is_executable); + auto digest = + storage_.CAS().StoreBlob</*kOwner=*/true>(file_path, is_executable); if (digest) { auto out_file = bazel_re::OutputFile{}; out_file.set_path(local_path); @@ -330,7 +329,7 @@ auto LocalAction::CollectOutputDirOrSymlink( } if (IsSymlinkObject(*type)) { auto content = FileSystemManager::ReadSymlink(dir_path); - if (content and storage_->CAS().StoreBlob(*content)) { + if (content and storage_.CAS().StoreBlob(*content)) { auto out_symlink = bazel_re::OutputSymlink{}; out_symlink.set_path(local_path); out_symlink.set_target(*content); @@ -417,7 +416,7 @@ auto LocalAction::CollectAndStoreOutputs( auto LocalAction::DigestFromOwnedFile(std::filesystem::path const& file_path) const noexcept -> gsl::owner<bazel_re::Digest*> { - if (auto digest = storage_->CAS().StoreBlob</*kOwner=*/true>( + if (auto digest = storage_.CAS().StoreBlob</*kOwner=*/true>( file_path, /*is_executable=*/false)) { return new bazel_re::Digest{std::move(*digest)}; } diff --git a/src/buildtool/execution_api/local/local_action.hpp b/src/buildtool/execution_api/local/local_action.hpp index b88b3a97..13f1e4c0 100644 --- a/src/buildtool/execution_api/local/local_action.hpp +++ b/src/buildtool/execution_api/local/local_action.hpp @@ -27,6 +27,7 @@ #include "src/buildtool/execution_api/common/execution_action.hpp" #include "src/buildtool/execution_api/common/execution_response.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" class LocalApi; @@ -57,7 +58,8 @@ class LocalAction final : public IExecutionAction { private: Logger logger_{"LocalExecution"}; - gsl::not_null<Storage const*> storage_; + StorageConfig const& storage_config_; + Storage const& storage_; ArtifactDigest root_digest_{}; std::vector<std::string> cmdline_{}; std::vector<std::string> output_files_{}; @@ -68,6 +70,7 @@ class LocalAction final : public IExecutionAction { CacheFlag cache_flag_{CacheFlag::CacheOutput}; explicit LocalAction( + gsl::not_null<StorageConfig const*> storage_config, gsl::not_null<Storage const*> const& storage, ArtifactDigest root_digest, std::vector<std::string> command, @@ -75,7 +78,8 @@ class LocalAction final : public IExecutionAction { std::vector<std::string> output_dirs, std::map<std::string, std::string> env_vars, std::map<std::string, std::string> const& properties) noexcept - : storage_{storage}, + : storage_config_{*storage_config}, + storage_{*storage}, root_digest_{std::move(root_digest)}, cmdline_{std::move(command)}, output_files_{std::move(output_files)}, diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index f9eefb64..6f43dd44 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -48,13 +48,18 @@ #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" /// \brief API for local execution. class LocalApi final : public IExecutionApi { public: - explicit LocalApi(RepositoryConfig const* repo_config = nullptr) noexcept - : repo_config_{repo_config} {} + explicit LocalApi(gsl::not_null<StorageConfig const*> const& storage_config, + gsl::not_null<Storage const*> const& storage, + RepositoryConfig const* repo_config = nullptr) noexcept + : storage_config_{*storage_config}, + storage_{*storage}, + repo_config_{repo_config} {} [[nodiscard]] auto CreateAction( ArtifactDigest const& root_digest, @@ -64,7 +69,8 @@ class LocalApi final : public IExecutionApi { std::map<std::string, std::string> const& env_vars, std::map<std::string, std::string> const& properties) const noexcept -> IExecutionAction::Ptr final { - return IExecutionAction::Ptr{new LocalAction{storage_, + return IExecutionAction::Ptr{new LocalAction{&storage_config_, + &storage_, root_digest, command, output_files, @@ -89,7 +95,7 @@ class LocalApi final : public IExecutionApi { auto const& info = artifacts_info[i]; if (IsTreeObject(info.type)) { // read object infos from sub tree and call retrieve recursively - auto reader = TreeReader<LocalCasReader>{storage_->CAS()}; + auto reader = TreeReader<LocalCasReader>{storage_.CAS()}; auto const result = reader.RecursivelyReadTreeLeafs( info.digest, output_paths[i]); if (not result) { @@ -109,7 +115,7 @@ class LocalApi final : public IExecutionApi { } } else { - auto const blob_path = storage_->CAS().BlobPath( + auto const blob_path = storage_.CAS().BlobPath( info.digest, IsExecutableObject(info.type)); if (not blob_path) { if (Compatibility::IsCompatible()) { @@ -143,7 +149,7 @@ class LocalApi final : public IExecutionApi { std::vector<Artifact::ObjectInfo> const& artifacts_info, std::vector<int> const& fds, bool raw_tree) const noexcept -> bool final { - auto dumper = StreamDumper<LocalCasReader>{storage_->CAS()}; + auto dumper = StreamDumper<LocalCasReader>{storage_.CAS()}; return CommonRetrieveToFds( artifacts_info, fds, @@ -196,7 +202,7 @@ class LocalApi final : public IExecutionApi { auto const& info = missing_artifacts_info->back_map[dgst]; // Recursively process trees. if (IsTreeObject(info.type)) { - auto reader = TreeReader<LocalCasReader>{storage_->CAS()}; + auto reader = TreeReader<LocalCasReader>{storage_.CAS()}; auto const& result = reader.ReadDirectTreeEntries( info.digest, std::filesystem::path{}); if (not result or not RetrieveToCas(result->infos, api)) { @@ -207,9 +213,9 @@ class LocalApi final : public IExecutionApi { // Determine artifact path. auto const& path = IsTreeObject(info.type) - ? storage_->CAS().TreePath(info.digest) - : storage_->CAS().BlobPath(info.digest, - IsExecutableObject(info.type)); + ? storage_.CAS().TreePath(info.digest) + : storage_.CAS().BlobPath(info.digest, + IsExecutableObject(info.type)); if (not path) { return false; } @@ -221,7 +227,7 @@ class LocalApi final : public IExecutionApi { } // Regenerate digest since object infos read by - // storage_->ReadTreeInfos() will contain 0 as size. + // storage_.ReadTreeInfos() will contain 0 as size. ArtifactDigest digest = IsTreeObject(info.type) ? ArtifactDigest::Create<ObjectType::Tree>(*content) @@ -251,10 +257,10 @@ class LocalApi final : public IExecutionApi { -> std::optional<std::string> override { std::optional<std::filesystem::path> location{}; if (IsTreeObject(artifact_info.type)) { - location = storage_->CAS().TreePath(artifact_info.digest); + location = storage_.CAS().TreePath(artifact_info.digest); } else { - location = storage_->CAS().BlobPath( + location = storage_.CAS().BlobPath( artifact_info.digest, IsExecutableObject(artifact_info.type)); } std::optional<std::string> content = std::nullopt; @@ -274,8 +280,8 @@ class LocalApi final : public IExecutionApi { auto const is_tree = NativeSupport::IsTree( static_cast<bazel_re::Digest>(blob.digest).hash()); auto cas_digest = - is_tree ? storage_->CAS().StoreTree(*blob.data) - : storage_->CAS().StoreBlob(*blob.data, blob.is_exec); + is_tree ? storage_.CAS().StoreTree(*blob.data) + : storage_.CAS().StoreBlob(*blob.data, blob.is_exec); if (not cas_digest or not std::equal_to<bazel_re::Digest>{}( *cas_digest, blob.digest)) { return false; @@ -298,7 +304,7 @@ class LocalApi final : public IExecutionApi { return CommonUploadTreeCompatible( *this, *build_root, - [&cas = storage_->CAS()]( + [&cas = storage_.CAS()]( std::vector<bazel_re::Digest> const& digests, std::vector<std::string>* targets) { targets->reserve(digests.size()); @@ -317,8 +323,8 @@ class LocalApi final : public IExecutionApi { -> bool final { return static_cast<bool>( NativeSupport::IsTree(static_cast<bazel_re::Digest>(digest).hash()) - ? storage_->CAS().TreePath(digest) - : storage_->CAS().BlobPath(digest, false)); + ? storage_.CAS().TreePath(digest) + : storage_.CAS().BlobPath(digest, false)); } [[nodiscard]] auto IsAvailable(std::vector<ArtifactDigest> const& digests) @@ -327,8 +333,8 @@ class LocalApi final : public IExecutionApi { for (auto const& digest : digests) { auto const& path = NativeSupport::IsTree( static_cast<bazel_re::Digest>(digest).hash()) - ? storage_->CAS().TreePath(digest) - : storage_->CAS().BlobPath(digest, false); + ? storage_.CAS().TreePath(digest) + : storage_.CAS().BlobPath(digest, false); if (not path) { result.push_back(digest); } @@ -340,7 +346,7 @@ class LocalApi final : public IExecutionApi { const noexcept -> std::optional<std::vector<ArtifactDigest>> final { Logger::Log(LogLevel::Debug, "SplitBlob({})", blob_digest.hash()); auto split_result = CASUtils::SplitBlobFastCDC( - static_cast<bazel_re::Digest>(blob_digest), *storage_); + static_cast<bazel_re::Digest>(blob_digest), storage_); if (not split_result) { Logger::Log(LogLevel::Error, split_result.error().error_message()); return std::nullopt; @@ -390,7 +396,7 @@ class LocalApi final : public IExecutionApi { return static_cast<bazel_re::Digest>(artifact_digest); }); auto splice_result = CASUtils::SpliceBlob( - static_cast<bazel_re::Digest>(blob_digest), digests, *storage_); + static_cast<bazel_re::Digest>(blob_digest), digests, storage_); if (not splice_result) { Logger::Log(LogLevel::Error, splice_result.error().error_message()); return std::nullopt; @@ -403,8 +409,9 @@ class LocalApi final : public IExecutionApi { } private: + StorageConfig const& storage_config_; + Storage const& storage_; RepositoryConfig const* const repo_config_ = nullptr; - gsl::not_null<Storage const*> storage_ = &Storage::Instance(); }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP diff --git a/src/buildtool/execution_api/local/local_response.hpp b/src/buildtool/execution_api/local/local_response.hpp index bcb98a2e..30d47ca7 100644 --- a/src/buildtool/execution_api/local/local_response.hpp +++ b/src/buildtool/execution_api/local/local_response.hpp @@ -42,8 +42,8 @@ class LocalResponse final : public IExecutionResponse { return (output_.action.stdout_digest().size_bytes() != 0); } auto StdErr() noexcept -> std::string final { - if (auto path = storage_->CAS().BlobPath(output_.action.stderr_digest(), - /*is_executable=*/false)) { + if (auto path = storage_.CAS().BlobPath(output_.action.stderr_digest(), + /*is_executable=*/false)) { if (auto content = FileSystemManager::ReadFile(*path)) { return std::move(*content); } @@ -52,8 +52,8 @@ class LocalResponse final : public IExecutionResponse { return {}; } auto StdOut() noexcept -> std::string final { - if (auto path = storage_->CAS().BlobPath(output_.action.stdout_digest(), - /*is_executable=*/false)) { + if (auto path = storage_.CAS().BlobPath(output_.action.stdout_digest(), + /*is_executable=*/false)) { if (auto content = FileSystemManager::ReadFile(*path)) { return std::move(*content); } @@ -87,7 +87,7 @@ class LocalResponse final : public IExecutionResponse { private: std::string action_id_{}; LocalAction::Output output_{}; - gsl::not_null<Storage const*> storage_; + Storage const& storage_; ArtifactInfos artifacts_{}; DirSymlinks dir_symlinks_{}; bool populated_{false}; @@ -98,7 +98,7 @@ class LocalResponse final : public IExecutionResponse { gsl::not_null<Storage const*> const& storage) noexcept : action_id_{std::move(action_id)}, output_{std::move(output)}, - storage_{storage} {} + storage_{*storage} {} [[nodiscard]] auto Populate() noexcept -> bool { if (populated_) { diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index 530244f2..878cfc50 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -794,7 +794,9 @@ auto main(int argc, char* argv[]) -> int { if (arguments.cmd == SubCommand::kExecute) { SetupExecutionServiceConfig(arguments.service); - ApiBundle const exec_apis{/*repo_config=*/nullptr, + ApiBundle const exec_apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, &*auth_config, RemoteExecutionConfig::RemoteAddress()}; if (not ServerImpl::Instance().Run(StorageConfig::Instance(), @@ -813,6 +815,8 @@ auto main(int argc, char* argv[]) -> int { arguments.service.pid_file); if (serve_server) { ApiBundle const serve_apis{ + &StorageConfig::Instance(), + &Storage::Instance(), /*repo_config=*/nullptr, &*auth_config, RemoteExecutionConfig::RemoteAddress()}; @@ -871,7 +875,9 @@ auto main(int argc, char* argv[]) -> int { if (not SetupRetryConfig(arguments.retry)) { std::exit(kExitFailure); } - ApiBundle const main_apis{&repo_config, + ApiBundle const main_apis{&StorageConfig::Instance(), + &Storage::Instance(), + &repo_config, &*auth_config, RemoteExecutionConfig::RemoteAddress()}; GraphTraverser const traverser{ diff --git a/src/buildtool/serve_api/serve_service/target.cpp b/src/buildtool/serve_api/serve_service/target.cpp index 6afea7e0..727751f4 100644 --- a/src/buildtool/serve_api/serve_service/target.cpp +++ b/src/buildtool/serve_api/serve_service/target.cpp @@ -497,7 +497,11 @@ auto TargetService::ServeTarget( // Use a new ApiBundle that knows about local repository config for // traversing. - ApiBundle const local_apis{&repository_config, &apis_.auth, address}; + ApiBundle const local_apis{&storage_config_, + &storage_, + &repository_config, + &apis_.auth, + address}; GraphTraverser const traverser{ std::move(traverser_args), &repository_config, diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index 791c2dc5..4954ecb7 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -126,6 +126,7 @@ , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] ] } @@ -183,6 +184,7 @@ , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/remote", "serve_api"] , ["src/buildtool/storage", "storage"] + , ["src/buildtool/storage", "config"] ] } , "launch": diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index 0e2d3e60..edd38652 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -28,6 +28,7 @@ #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" @@ -404,7 +405,9 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, return kExitConfigError; } - ApiBundle const apis{/*repo_config=*/nullptr, + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, &*auth_config, RemoteExecutionConfig::RemoteAddress()}; diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index c96af8b1..5c24f6c8 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -31,6 +31,7 @@ #include "src/buildtool/multithreading/task_system.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/fs_utils.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" @@ -123,7 +124,9 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, return std::nullopt; } - ApiBundle const apis{/*repo_config=*/nullptr, + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, &*auth_config, RemoteExecutionConfig::RemoteAddress()}; diff --git a/test/buildtool/build_engine/target_map/TARGETS b/test/buildtool/build_engine/target_map/TARGETS index c73a0fa2..9111efc7 100644 --- a/test/buildtool/build_engine/target_map/TARGETS +++ b/test/buildtool/build_engine/target_map/TARGETS @@ -33,6 +33,7 @@ , ["@", "src", "src/buildtool/common", "common"] , ["@", "src", "src/buildtool/file_system", "file_root"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] + , ["@", "src", "src/buildtool/storage", "config"] , ["@", "src", "src/buildtool/storage", "storage"] , ["@", "src", "src/buildtool/main", "analyse_context"] , ["@", "src", "src/buildtool/common", "config"] diff --git a/test/buildtool/build_engine/target_map/target_map.test.cpp b/test/buildtool/build_engine/target_map/target_map.test.cpp index e392902b..a3e1d350 100644 --- a/test/buildtool/build_engine/target_map/target_map.test.cpp +++ b/test/buildtool/build_engine/target_map/target_map.test.cpp @@ -36,6 +36,7 @@ #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" +#include "src/buildtool/storage/config.hpp" #include "src/buildtool/storage/storage.hpp" #include "test/utils/hermeticity/local.hpp" #include "test/utils/serve_service/test_serve_config.hpp" @@ -101,8 +102,11 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "simple targets", "[target_map]") { REQUIRE(serve_config); Auth auth{}; - ApiBundle const apis{ - /*repo_config=*/nullptr, &auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, + &auth, + RemoteExecutionConfig::RemoteAddress()}; auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); AnalyseContext ctx{.repo_config = &repo_config, .target_cache = &Storage::Instance().TargetCache(), @@ -546,8 +550,11 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, REQUIRE(serve_config); Auth auth{}; - ApiBundle const apis{ - /*repo_config=*/nullptr, &auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, + &auth, + RemoteExecutionConfig::RemoteAddress()}; auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); AnalyseContext ctx{.repo_config = &repo_config, .target_cache = &Storage::Instance().TargetCache(), @@ -636,8 +643,11 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, REQUIRE(serve_config); Auth auth{}; - ApiBundle const apis{ - /*repo_config=*/nullptr, &auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, + &auth, + RemoteExecutionConfig::RemoteAddress()}; auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); AnalyseContext ctx{.repo_config = &repo_config, .target_cache = &Storage::Instance().TargetCache(), @@ -736,8 +746,11 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "built-in rules", "[target_map]") { REQUIRE(serve_config); Auth auth{}; - ApiBundle const apis{ - /*repo_config=*/nullptr, &auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, + &auth, + RemoteExecutionConfig::RemoteAddress()}; auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); AnalyseContext ctx{.repo_config = &repo_config, .target_cache = &Storage::Instance().TargetCache(), @@ -946,8 +959,11 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "target reference", "[target_map]") { REQUIRE(serve_config); Auth auth{}; - ApiBundle const apis{ - /*repo_config=*/nullptr, &auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, + &auth, + RemoteExecutionConfig::RemoteAddress()}; auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); AnalyseContext ctx{.repo_config = &repo_config, .target_cache = &Storage::Instance().TargetCache(), @@ -1089,8 +1105,11 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "trees", "[target_map]") { REQUIRE(serve_config); Auth auth{}; - ApiBundle const apis{ - /*repo_config=*/nullptr, &auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, + &auth, + RemoteExecutionConfig::RemoteAddress()}; auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); AnalyseContext ctx{.repo_config = &repo_config, .target_cache = &Storage::Instance().TargetCache(), @@ -1198,8 +1217,11 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, REQUIRE(serve_config); Auth auth{}; - ApiBundle const apis{ - /*repo_config=*/nullptr, &auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, + &auth, + RemoteExecutionConfig::RemoteAddress()}; auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); AnalyseContext ctx{.repo_config = &repo_config, .target_cache = &Storage::Instance().TargetCache(), @@ -1364,8 +1386,11 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "wrong arguments", "[target_map]") { REQUIRE(serve_config); Auth auth{}; - ApiBundle const apis{ - /*repo_config=*/nullptr, &auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + /*repo_config=*/nullptr, + &auth, + RemoteExecutionConfig::RemoteAddress()}; auto serve = ServeApi::Create(*serve_config, &Storage::Instance(), &apis); AnalyseContext ctx{.repo_config = &repo_config, .target_cache = &Storage::Instance().TargetCache(), diff --git a/test/buildtool/execution_api/local/TARGETS b/test/buildtool/execution_api/local/TARGETS index 4b9c7b81..282166a2 100644 --- a/test/buildtool/execution_api/local/TARGETS +++ b/test/buildtool/execution_api/local/TARGETS @@ -13,6 +13,8 @@ , ["@", "src", "src/buildtool/logging", "log_level"] , ["@", "src", "src/buildtool/logging", "logging"] , ["utils", "local_hermeticity"] + , ["@", "src", "src/buildtool/storage", "storage"] + , ["@", "src", "src/buildtool/storage", "config"] ] , "stage": ["test", "buildtool", "execution_api", "local"] } @@ -26,6 +28,8 @@ , ["@", "src", "src/buildtool/execution_api/local", "local"] , ["buildtool/execution_api/common", "api_test"] , ["utils", "local_hermeticity"] + , ["@", "src", "src/buildtool/storage", "storage"] + , ["@", "src", "src/buildtool/storage", "config"] ] , "stage": ["test", "buildtool", "execution_api", "local"] } diff --git a/test/buildtool/execution_api/local/local_api.test.cpp b/test/buildtool/execution_api/local/local_api.test.cpp index f5680a10..fa296c9b 100644 --- a/test/buildtool/execution_api/local/local_api.test.cpp +++ b/test/buildtool/execution_api/local/local_api.test.cpp @@ -17,12 +17,17 @@ #include "catch2/catch_test_macros.hpp" #include "src/buildtool/execution_api/local/local_api.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "test/buildtool/execution_api/common/api_test.hpp" #include "test/utils/hermeticity/local.hpp" namespace { -auto const kApiFactory = []() { return IExecutionApi::Ptr{new LocalApi()}; }; +auto const kApiFactory = []() { + return IExecutionApi::Ptr{ + new LocalApi(&StorageConfig::Instance(), &Storage::Instance())}; +}; } // namespace diff --git a/test/buildtool/execution_api/local/local_execution.test.cpp b/test/buildtool/execution_api/local/local_execution.test.cpp index d76a5a18..6f9ccfb1 100644 --- a/test/buildtool/execution_api/local/local_execution.test.cpp +++ b/test/buildtool/execution_api/local/local_execution.test.cpp @@ -26,6 +26,8 @@ #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "test/utils/hermeticity/local.hpp" namespace { @@ -60,7 +62,8 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "LocalExecution: No input, no output", "[execution_api]") { RepositoryConfig repo_config{}; - auto api = LocalApi(&repo_config); + auto api = LocalApi( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); std::string test_content("test"); std::vector<std::string> const cmdline = {"echo", "-n", test_content}; @@ -106,7 +109,8 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "LocalExecution: No input, no output, env variables used", "[execution_api]") { RepositoryConfig repo_config{}; - auto api = LocalApi(&repo_config); + auto api = LocalApi( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); std::string test_content("test from env var"); std::vector<std::string> const cmdline = { @@ -156,7 +160,8 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "LocalExecution: No input, create output", "[execution_api]") { RepositoryConfig repo_config{}; - auto api = LocalApi(&repo_config); + auto api = LocalApi( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); std::string test_content("test"); auto test_digest = ArtifactDigest::Create<ObjectType::File>(test_content); @@ -212,7 +217,8 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "LocalExecution: One input copied to output", "[execution_api]") { RepositoryConfig repo_config{}; - auto api = LocalApi(&repo_config); + auto api = LocalApi( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); std::string test_content("test"); auto test_digest = ArtifactDigest::Create<ObjectType::File>(test_content); @@ -282,7 +288,8 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "LocalExecution: Cache failed action's result", "[execution_api]") { RepositoryConfig repo_config{}; - auto api = LocalApi(&repo_config); + auto api = LocalApi( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); auto flag = GetTestDir() / "flag"; std::vector<std::string> const cmdline = { diff --git a/test/buildtool/execution_engine/executor/TARGETS b/test/buildtool/execution_engine/executor/TARGETS index df0f9df6..c1b50f35 100644 --- a/test/buildtool/execution_engine/executor/TARGETS +++ b/test/buildtool/execution_engine/executor/TARGETS @@ -54,6 +54,8 @@ , ["utils", "local_hermeticity"] , ["utils", "test_auth_config"] , ["@", "catch2", "", "catch2"] + , ["@", "src", "src/buildtool/storage", "storage"] + , ["@", "src", "src/buildtool/storage", "config"] ] , "stage": ["test", "buildtool", "execution_engine", "executor"] } diff --git a/test/buildtool/execution_engine/executor/executor_api_local.test.cpp b/test/buildtool/execution_engine/executor/executor_api_local.test.cpp index 0572dc7b..ea6baa60 100755..100644 --- a/test/buildtool/execution_engine/executor/executor_api_local.test.cpp +++ b/test/buildtool/execution_engine/executor/executor_api_local.test.cpp @@ -21,6 +21,8 @@ #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_engine/executor/executor.hpp" #include "src/buildtool/progress_reporting/progress.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "test/buildtool/execution_engine/executor/executor_api.test.hpp" #include "test/utils/hermeticity/local.hpp" #include "test/utils/remote_execution/test_auth_config.hpp" @@ -29,8 +31,10 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, "Executor<LocalApi>: Upload blob", "[executor]") { RepositoryConfig repo_config{}; - TestBlobUpload(&repo_config, - [&] { return std::make_unique<LocalApi>(&repo_config); }); + TestBlobUpload(&repo_config, [&] { + return std::make_unique<LocalApi>( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); + }); } TEST_CASE_METHOD(HermeticLocalTestFixture, @@ -45,7 +49,10 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, &repo_config, &stats, &progress, - [&] { return std::make_unique<LocalApi>(&repo_config); }, + [&] { + return std::make_unique<LocalApi>( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); + }, &*auth_config); } @@ -61,7 +68,10 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, &repo_config, &stats, &progress, - [&] { return std::make_unique<LocalApi>(&repo_config); }, + [&] { + return std::make_unique<LocalApi>( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); + }, &*auth_config); } @@ -77,7 +87,10 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, &repo_config, &stats, &progress, - [&] { return std::make_unique<LocalApi>(&repo_config); }, + [&] { + return std::make_unique<LocalApi>( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); + }, &*auth_config); } @@ -93,6 +106,9 @@ TEST_CASE_METHOD(HermeticLocalTestFixture, &repo_config, &stats, &progress, - [&] { return std::make_unique<LocalApi>(&repo_config); }, + [&] { + return std::make_unique<LocalApi>( + &StorageConfig::Instance(), &Storage::Instance(), &repo_config); + }, &*auth_config); } diff --git a/test/buildtool/graph_traverser/TARGETS b/test/buildtool/graph_traverser/TARGETS index 1d11b04a..d04e4365 100644 --- a/test/buildtool/graph_traverser/TARGETS +++ b/test/buildtool/graph_traverser/TARGETS @@ -15,6 +15,8 @@ , ["@", "src", "src/buildtool/logging", "logging"] , ["@", "src", "src/buildtool/progress_reporting", "progress"] , ["@", "src", "src/buildtool/execution_api/common", "api_bundle"] + , ["@", "src", "src/buildtool/storage", "storage"] + , ["@", "src", "src/buildtool/storage", "config"] ] , "stage": ["test", "buildtool", "graph_traverser"] } diff --git a/test/buildtool/graph_traverser/graph_traverser.test.hpp b/test/buildtool/graph_traverser/graph_traverser.test.hpp index f1be7eec..2999c0e0 100644 --- a/test/buildtool/graph_traverser/graph_traverser.test.hpp +++ b/test/buildtool/graph_traverser/graph_traverser.test.hpp @@ -38,6 +38,8 @@ #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/progress.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/json.hpp" // NOLINTNEXTLINE(google-build-namespaces) @@ -160,8 +162,11 @@ inline void SetLauncher() { auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; GraphTraverser const gt{clargs.gtargs, p.GetRepoConfig(), RemoteExecutionConfig::PlatformProperties(), @@ -188,8 +193,11 @@ inline void SetLauncher() { SECTION("Executable is retrieved as executable") { auto const clargs_exec = p.CmdLineArgs("_entry_points_get_executable"); - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; GraphTraverser const gt_get_exec{ clargs_exec.gtargs, p.GetRepoConfig(), @@ -226,8 +234,11 @@ inline void SetLauncher() { auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; GraphTraverser const gt{clargs.gtargs, p.GetRepoConfig(), RemoteExecutionConfig::PlatformProperties(), @@ -258,8 +269,11 @@ inline void SetLauncher() { auto const clargs = p.CmdLineArgs(); Statistics stats{}; Progress progress{}; - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; GraphTraverser const gt{clargs.gtargs, p.GetRepoConfig(), RemoteExecutionConfig::PlatformProperties(), @@ -311,7 +325,9 @@ inline void SetLauncher() { full_hello_world.CmdLineArgs("_entry_points_upload_source"); Statistics stats{}; Progress progress{}; - ApiBundle const apis{full_hello_world.GetRepoConfig(), + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + full_hello_world.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; GraphTraverser const gt_upload{clargs_update_cpp.gtargs, @@ -369,8 +385,11 @@ static void TestBlobsUploadedAndUsed(gsl::not_null<Auth const*> const& auth, SetLauncher(); Statistics stats{}; Progress progress{}; - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; GraphTraverser gt{clargs.gtargs, p.GetRepoConfig(), RemoteExecutionConfig::PlatformProperties(), @@ -409,8 +428,11 @@ static void TestEnvironmentVariablesSetAndUsed( SetLauncher(); Statistics stats{}; Progress progress{}; - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; GraphTraverser gt{clargs.gtargs, p.GetRepoConfig(), RemoteExecutionConfig::PlatformProperties(), @@ -448,8 +470,11 @@ static void TestTreesUsed(gsl::not_null<Auth const*> const& auth, SetLauncher(); Statistics stats{}; Progress progress{}; - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; GraphTraverser gt{clargs.gtargs, p.GetRepoConfig(), RemoteExecutionConfig::PlatformProperties(), @@ -487,8 +512,11 @@ static void TestNestedTreesUsed(gsl::not_null<Auth const*> const& auth, SetLauncher(); Statistics stats{}; Progress progress{}; - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; GraphTraverser gt{clargs.gtargs, p.GetRepoConfig(), RemoteExecutionConfig::PlatformProperties(), @@ -524,8 +552,11 @@ static void TestFlakyHelloWorldDetected(gsl::not_null<Auth const*> const& auth, Statistics stats{}; Progress progress{}; - ApiBundle const apis{ - p.GetRepoConfig(), auth, RemoteExecutionConfig::RemoteAddress()}; + ApiBundle const apis{&StorageConfig::Instance(), + &Storage::Instance(), + p.GetRepoConfig(), + auth, + RemoteExecutionConfig::RemoteAddress()}; { SetLauncher(); |