diff options
Diffstat (limited to 'src')
4 files changed, 154 insertions, 0 deletions
diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp index 661ecbf3..68d4b4cb 100644 --- a/src/buildtool/execution_api/common/execution_api.hpp +++ b/src/buildtool/execution_api/common/execution_api.hpp @@ -61,6 +61,13 @@ class IExecutionApi { std::vector<int> const& fds, bool raw_tree) noexcept -> bool = 0; + /// \brief Synchronization of artifacts between two CASes. Retrieves + /// artifacts from one CAS and writes to another CAS. Tree artifacts are + /// resolved and its containing file artifacts are recursively retrieved. + [[nodiscard]] virtual auto RetrieveToCas( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + gsl::not_null<IExecutionApi*> const& api) noexcept -> bool = 0; + /// \brief Upload blobs to CAS. Uploads only the blobs that are not yet /// available in CAS, unless `skip_find_missing` is specified. /// \param blobs Container of blobs to upload. diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 6e1c0074..38b98601 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -110,6 +110,79 @@ class LocalApi final : public IExecutionApi { return true; } + // NOLINTNEXTLINE(misc-no-recursion) + [[nodiscard]] auto RetrieveToCas( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + gsl::not_null<IExecutionApi*> const& api) noexcept -> bool final { + + // Determine missing artifacts in other CAS. + std::vector<ArtifactDigest> digests; + digests.reserve(artifacts_info.size()); + std::unordered_map<ArtifactDigest, Artifact::ObjectInfo> info_map; + for (auto const& info : artifacts_info) { + digests.push_back(info.digest); + info_map[info.digest] = info; + } + auto const& missing_digests = api->IsAvailable(digests); + std::vector<Artifact::ObjectInfo> missing_artifacts_info; + missing_artifacts_info.reserve(missing_digests.size()); + for (auto const& digest : missing_digests) { + missing_artifacts_info.push_back(info_map[digest]); + } + + // Collect blobs of missing artifacts from local CAS. Trees are + // processed recursively before any blob is uploaded. + BlobContainer container{}; + for (auto const& info : missing_artifacts_info) { + // Recursively process trees. + if (IsTreeObject(info.type)) { + auto const& infos = storage_->ReadTreeInfos( + info.digest, std::filesystem::path{}); + if (not infos or not RetrieveToCas(infos->second, api)) { + return false; + } + } + + // Determine artifact path. + auto const& path = + IsTreeObject(info.type) + ? storage_->TreePath(info.digest) + : storage_->BlobPath(info.digest, + IsExecutableObject(info.type)); + if (not path) { + return false; + } + + // Read artifact content. + auto const& content = FileSystemManager::ReadFile(*path); + if (not content) { + return false; + } + + // Regenerate digest since object infos read by + // storage_->ReadTreeInfos() will contain 0 as size. + ArtifactDigest digest; + if (IsTreeObject(info.type)) { + digest = ArtifactDigest::Create<ObjectType::Tree>(*content); + } + else { + digest = ArtifactDigest::Create<ObjectType::File>(*content); + } + + // Collect blob. + try { + container.Emplace(BazelBlob{digest, *content}); + } catch (std::exception const& ex) { + Logger::Log( + LogLevel::Error, "failed to emplace blob: ", ex.what()); + return false; + } + } + + // Upload blobs to remote CAS. + return api->Upload(container, /*skip_find_missing=*/true); + } + [[nodiscard]] auto Upload(BlobContainer const& blobs, bool /*skip_find_missing*/) noexcept -> bool final { diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index b07bcede..a8846325 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -148,6 +148,76 @@ auto BazelApi::CreateAction( return true; } +// NOLINTNEXTLINE(misc-no-recursion) +[[nodiscard]] auto BazelApi::RetrieveToCas( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + gsl::not_null<IExecutionApi*> const& api) noexcept -> bool { + + // Determine missing artifacts in other CAS. + std::vector<ArtifactDigest> digests; + digests.reserve(artifacts_info.size()); + std::unordered_map<ArtifactDigest, Artifact::ObjectInfo> info_map; + for (auto const& info : artifacts_info) { + digests.push_back(info.digest); + info_map[info.digest] = info; + } + auto const& missing_digests = api->IsAvailable(digests); + std::vector<Artifact::ObjectInfo> missing_artifacts_info; + missing_artifacts_info.reserve(missing_digests.size()); + for (auto const& digest : missing_digests) { + missing_artifacts_info.push_back(info_map[digest]); + } + + // Recursively process trees. + std::vector<bazel_re::Digest> blob_digests{}; + for (auto const& info : missing_artifacts_info) { + if (IsTreeObject(info.type)) { + auto const infos = + network_->ReadTreeInfos(info.digest, std::filesystem::path{}); + if (not infos or not RetrieveToCas(infos->second, api)) { + return false; + } + } + + // Object infos created by network_->ReadTreeInfos() will contain 0 as + // size, but this is handled by the remote execution engine, so no need + // to regenerate the digest. + blob_digests.push_back(info.digest); + } + + // Fetch blobs from this CAS. + auto size = blob_digests.size(); + auto reader = network_->ReadBlobs(std::move(blob_digests)); + auto blobs = reader.Next(); + std::size_t count{}; + BlobContainer container{}; + while (not blobs.empty()) { + if (count + blobs.size() > size) { + Logger::Log(LogLevel::Error, "received more blobs than requested."); + return false; + } + for (auto& blob : blobs) { + try { + container.Emplace(std::move(blob)); + } catch (std::exception const& ex) { + Logger::Log( + LogLevel::Error, "failed to emplace blob: ", ex.what()); + return false; + } + } + count += blobs.size(); + blobs = reader.Next(); + } + + if (count != size) { + Logger::Log(LogLevel::Error, "could not retrieve all requested blobs."); + return false; + } + + // Upload blobs to other CAS. + return api->Upload(container, /*skip_find_missing=*/true); +} + [[nodiscard]] auto BazelApi::Upload(BlobContainer const& blobs, bool skip_find_missing) noexcept -> bool { return network_->UploadBlobs(blobs, skip_find_missing); diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp index d9619d82..3fddca1f 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp @@ -48,6 +48,10 @@ class BazelApi final : public IExecutionApi { std::vector<int> const& fds, bool raw_tree) noexcept -> bool final; + [[nodiscard]] auto RetrieveToCas( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + gsl::not_null<IExecutionApi*> const& api) noexcept -> bool final; + [[nodiscard]] auto Upload(BlobContainer const& blobs, bool skip_find_missing) noexcept -> bool final; |