diff options
| -rw-r--r-- | src/buildtool/execution_api/common/TARGETS | 20 | ||||
| -rw-r--r-- | src/buildtool/execution_api/common/common_api.cpp | 178 | ||||
| -rw-r--r-- | src/buildtool/execution_api/common/common_api.hpp | 99 | ||||
| -rw-r--r-- | src/buildtool/execution_api/git/TARGETS | 1 | ||||
| -rw-r--r-- | src/buildtool/execution_api/git/git_api.hpp | 26 | ||||
| -rw-r--r-- | src/buildtool/execution_api/local/TARGETS | 1 | ||||
| -rw-r--r-- | src/buildtool/execution_api/local/local_api.hpp | 186 | ||||
| -rw-r--r-- | src/buildtool/execution_api/remote/TARGETS | 1 | ||||
| -rw-r--r-- | src/buildtool/execution_api/remote/bazel/bazel_api.cpp | 216 | ||||
| -rw-r--r-- | src/buildtool/execution_api/remote/bazel/bazel_api.hpp | 3 |
10 files changed, 410 insertions, 321 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index d2fcfc6d..210cbb3a 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -48,4 +48,24 @@ , "deps": [["@", "grpc", "", "grpc++"]] , "stage": ["src", "buildtool", "execution_api", "common"] } +, "common_api": + { "type": ["@", "rules", "CC", "library"] + , "name": ["common_api"] + , "hdrs": ["common_api.hpp"] + , "srcs": ["common_api.cpp"] + , "deps": + [ "common" + , ["@", "gsl", "", "gsl"] + , ["src/buildtool/common", "common"] + , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] + , ["src/buildtool/execution_api/bazel_msg", "blob_tree"] + , ["src/buildtool/execution_api/bazel_msg", "directory_tree"] + ] + , "stage": ["src", "buildtool", "execution_api", "common"] + , "private-deps": + [ ["@", "fmt", "", "fmt"] + , ["src/buildtool/logging", "logging"] + , ["src/buildtool/logging", "log_level"] + ] + } } diff --git a/src/buildtool/execution_api/common/common_api.cpp b/src/buildtool/execution_api/common/common_api.cpp new file mode 100644 index 00000000..d1b3ad03 --- /dev/null +++ b/src/buildtool/execution_api/common/common_api.cpp @@ -0,0 +1,178 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/common/common_api.hpp" + +#include <cstddef> +#include <exception> + +#include "fmt/core.h" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" + +auto CommonRetrieveToFds( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + std::vector<int> const& fds, + std::function<bool(Artifact::ObjectInfo const&, + gsl::not_null<FILE*> const&)> const& dump_to_stream, + std::optional<std::function<bool(Artifact::ObjectInfo const&, int)>> const& + fallback) noexcept -> bool { + if (artifacts_info.size() != fds.size()) { + Logger::Log(LogLevel::Error, + "different number of digests and file descriptors."); + return false; + } + + for (std::size_t i{}; i < artifacts_info.size(); ++i) { + auto fd = fds[i]; + auto const& info = artifacts_info[i]; + + if (gsl::owner<FILE*> out = fdopen(dup(fd), "wb")) { // NOLINT + bool success{false}; + try { + success = dump_to_stream(info, out); + } catch (std::exception const& ex) { + std::fclose(out); // close file + Logger::Log(LogLevel::Error, + "dumping {} to stream failed with:\n{}", + info.ToString(), + ex.what()); + return false; + } + std::fclose(out); + if (not success) { + Logger::Log( + LogLevel::Debug, + "dumping {} {} from CAS to file descriptor {} failed.", + IsTreeObject(info.type) ? "tree" : "blob", + info.ToString(), + fd); + // locally we might be able to fallback to Git in native mode + try { + if (fallback and not(*fallback)(info, fd)) { + return false; + } + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "fallback dumping {} to file descriptor {} " + "failed with:\n{}", + info.ToString(), + fd, + ex.what()); + return false; + } + } + } + else { + Logger::Log( + LogLevel::Error, "dumping to file descriptor {} failed.", fd); + return false; + } + } + return true; +} + +/// NOLINTNEXTLINE(misc-no-recursion) +auto CommonUploadBlobTree(BlobTreePtr const& blob_tree, + gsl::not_null<IExecutionApi*> const& api) noexcept + -> bool { + // Create digest list from blobs for batch availability check. + auto missing_blobs_info = GetMissingArtifactsInfo<BlobTreePtr>( + api, blob_tree->begin(), blob_tree->end(), [](BlobTreePtr const& node) { + return ArtifactDigest{node->Blob().digest}; + }); + if (not missing_blobs_info) { + Logger::Log(LogLevel::Error, + "Failed to retrieve the missing tree blobs for upload"); + return false; + } + + // Process missing blobs. + BlobContainer container; + for (auto const& digest : missing_blobs_info->digests) { + if (auto it = missing_blobs_info->back_map.find(digest); + it != missing_blobs_info->back_map.end()) { + auto const& node = it->second; + // Process trees. + if (node->IsTree()) { + if (not CommonUploadBlobTree(node, api)) { + return false; + } + } + // Store blob. + try { + container.Emplace(node->Blob()); + } catch (...) { + return false; + } + } + } + + return api->Upload(container, /*skip_find_missing=*/true); +} + +auto CommonUploadTreeCompatible( + gsl::not_null<IExecutionApi*> const& api, + DirectoryTreePtr const& build_root, + BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) noexcept + -> std::optional<ArtifactDigest> { + BlobContainer blobs{}; + auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree( + build_root, resolve_links, [&blobs](BazelBlob&& blob) { + blobs.Emplace(std::move(blob)); + }); + if (not digest) { + Logger::Log(LogLevel::Debug, "failed to create digest for build root."); + return std::nullopt; + } + Logger::Log(LogLevel::Trace, [&digest]() { + std::ostringstream oss{}; + oss << "upload root directory" << std::endl; + oss << fmt::format(" - root digest: {}", digest->hash()) << std::endl; + return oss.str(); + }); + if (not api->Upload(blobs, /*skip_find_missing=*/false)) { + Logger::Log(LogLevel::Debug, "failed to upload blobs for build root."); + return std::nullopt; + } + return ArtifactDigest{*digest}; +} + +auto CommonUploadTreeNative(gsl::not_null<IExecutionApi*> const& api, + DirectoryTreePtr const& build_root) noexcept + -> std::optional<ArtifactDigest> { + auto blob_tree = BlobTree::FromDirectoryTree(build_root); + if (not blob_tree) { + Logger::Log(LogLevel::Debug, + "failed to create blob tree for build root."); + return std::nullopt; + } + auto tree_blob = (*blob_tree)->Blob(); + // Upload blob tree if tree is not available at the remote side (content + // first). + if (not api->IsAvailable(ArtifactDigest{tree_blob.digest})) { + if (not CommonUploadBlobTree(*blob_tree, api)) { + Logger::Log(LogLevel::Debug, + "failed to upload blob tree for build root."); + return std::nullopt; + } + if (not api->Upload(BlobContainer{{tree_blob}}, + /*skip_find_missing=*/true)) { + Logger::Log(LogLevel::Debug, + "failed to upload tree blob for build root."); + return std::nullopt; + } + } + return ArtifactDigest{tree_blob.digest}; +} diff --git a/src/buildtool/execution_api/common/common_api.hpp b/src/buildtool/execution_api/common/common_api.hpp new file mode 100644 index 00000000..312f97db --- /dev/null +++ b/src/buildtool/execution_api/common/common_api.hpp @@ -0,0 +1,99 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP + +#include <cstdio> +#include <exception> +#include <functional> +#include <optional> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "gsl/gsl" +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" +#include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" +#include "src/buildtool/execution_api/common/execution_api.hpp" + +/// \brief Stores a list of missing artifact digests, as well as a back-mapping +/// to some given original type. +template <typename T> +struct MissingArtifactsInfo { + std::vector<ArtifactDigest> digests; + std::unordered_map<ArtifactDigest, T> back_map; +}; + +/// \brief Common logic for RetrieveToFds. +/// \param dump_to_stream Dumps the artifact to the respective open stream. +/// \param fallback Processes the respective file descriptor further in case the +/// regular dump fails. +[[nodiscard]] auto CommonRetrieveToFds( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + std::vector<int> const& fds, + std::function<bool(Artifact::ObjectInfo const&, + gsl::not_null<FILE*> const&)> const& dump_to_stream, + std::optional<std::function<bool(Artifact::ObjectInfo const&, int)>> const& + fallback) noexcept -> bool; + +/// \brief Get the missing artifacts from a given input list, needed, e.g., to +/// be uploaded. +/// \returns A struct storing the missing artifacts and a back-mapping to the +/// original given type, or nullopt in case of exceptions. +template <typename T> +[[nodiscard]] auto GetMissingArtifactsInfo( + gsl::not_null<IExecutionApi*> const& api, + typename std::vector<T>::const_iterator const& begin, + typename std::vector<T>::const_iterator const& end, + typename std::function<ArtifactDigest(T const&)> const& converter) noexcept + -> std::optional<MissingArtifactsInfo<T>> { + std::vector<ArtifactDigest> digests; + digests.reserve(end - begin); + MissingArtifactsInfo<T> res{}; + for (auto it = begin; it != end; ++it) { + try { + auto dgst = converter(*it); // can't enforce it to be noexcept + digests.emplace_back(dgst); + res.back_map.emplace(std::move(dgst), *it); + } catch (...) { + return std::nullopt; + } + } + res.digests = api->IsAvailable(digests); + return res; +} + +/// \brief Upload missing blobs from a given BlobTree. +[[nodiscard]] auto CommonUploadBlobTree( + BlobTreePtr const& blob_tree, + gsl::not_null<IExecutionApi*> const& api) noexcept -> bool; + +/// \brief Runs the compatible branch of local/bazel UploadTree API. +[[nodiscard]] auto CommonUploadTreeCompatible( + gsl::not_null<IExecutionApi*> const& api, + DirectoryTreePtr const& build_root, + BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) noexcept + -> std::optional<ArtifactDigest>; + +/// \brief Runs the native branch of local/bazel UploadTree API. +[[nodiscard]] auto CommonUploadTreeNative( + gsl::not_null<IExecutionApi*> const& api, + DirectoryTreePtr const& build_root) noexcept + -> std::optional<ArtifactDigest>; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP diff --git a/src/buildtool/execution_api/git/TARGETS b/src/buildtool/execution_api/git/TARGETS index c77a9741..f0895c4b 100644 --- a/src/buildtool/execution_api/git/TARGETS +++ b/src/buildtool/execution_api/git/TARGETS @@ -7,6 +7,7 @@ , ["src/buildtool/common", "config"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] ] diff --git a/src/buildtool/execution_api/git/git_api.hpp b/src/buildtool/execution_api/git/git_api.hpp index 56cd78ab..3bbbfd1f 100644 --- a/src/buildtool/execution_api/git/git_api.hpp +++ b/src/buildtool/execution_api/git/git_api.hpp @@ -25,6 +25,7 @@ #include "gsl/gsl" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -179,24 +180,23 @@ class GitApi final : public IExecutionApi { } // Determine missing artifacts in other CAS. - std::vector<ArtifactDigest> digests; - digests.reserve(artifacts_info.size()); - std::unordered_map<ArtifactDigest, Artifact::ObjectInfo> info_map; - for (auto const& info : artifacts_info) { - digests.emplace_back(info.digest); - info_map[info.digest] = info; - } - auto const& missing_digests = api->IsAvailable(digests); - std::vector<Artifact::ObjectInfo> missing_artifacts_info; - missing_artifacts_info.reserve(missing_digests.size()); - for (auto const& digest : missing_digests) { - missing_artifacts_info.emplace_back(info_map[digest]); + auto missing_artifacts_info = + GetMissingArtifactsInfo<Artifact::ObjectInfo>( + api, + artifacts_info.begin(), + artifacts_info.end(), + [](Artifact::ObjectInfo const& info) { return info.digest; }); + if (not missing_artifacts_info) { + Logger::Log(LogLevel::Error, + "GitApi: Failed to retrieve the missing artifacts"); + return false; } // Collect blobs of missing artifacts from local CAS. Trees are // processed recursively before any blob is uploaded. BlobContainer container{}; - for (auto const& info : missing_artifacts_info) { + for (auto const& dgst : missing_artifacts_info->digests) { + auto const& info = missing_artifacts_info->back_map[dgst]; std::optional<std::string> content; // Recursively process trees. if (IsTreeObject(info.type)) { diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index 6467512f..b1b7c08c 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -28,6 +28,7 @@ , ["src/buildtool/common", "config"] , ["src/buildtool/storage", "storage"] , ["src/buildtool/execution_api/common", "common"] + , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] , ["src/buildtool/execution_api/bazel_msg", "blob_tree"] , ["src/buildtool/execution_api/git", "git"] diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index f69a5be2..53cee482 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -35,6 +35,7 @@ #include "src/buildtool/compatibility/native_support.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp" #include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" +#include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" #include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" @@ -137,47 +138,28 @@ class LocalApi final : public IExecutionApi { std::vector<Artifact::ObjectInfo> const& artifacts_info, std::vector<int> const& fds, bool raw_tree) noexcept -> bool final { - if (artifacts_info.size() != fds.size()) { - Logger::Log(LogLevel::Error, - "different number of digests and file descriptors."); - return false; - } - - for (std::size_t i{}; i < artifacts_info.size(); ++i) { - auto fd = fds[i]; - auto const& info = artifacts_info[i]; - - if (gsl::owner<FILE*> out = fdopen(dup(fd), "wb")) { // NOLINT - auto const success = - storage_->CAS().DumpToStream(info, out, raw_tree); - std::fclose(out); - if (not success) { - Logger::Log( - LogLevel::Debug, - "dumping {} {} from CAS to file descriptor {} failed.", - IsTreeObject(info.type) ? "tree" : "blob", - info.ToString(), - fd); - if (Compatibility::IsCompatible()) { - // infos not available, and in compatible mode cannot - // fall back to git - return false; - } - if (repo_config_ and - not GitApi(repo_config_.value()) - .RetrieveToFds({info}, {fd}, raw_tree)) { - return false; - } + return CommonRetrieveToFds( + artifacts_info, + fds, + [&cas = storage_->CAS(), &raw_tree]( + Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& out) { + return cas.DumpToStream(info, out, raw_tree); + }, + [&repo_config = repo_config_, &raw_tree]( + Artifact::ObjectInfo const& info, int fd) { + if (Compatibility::IsCompatible()) { + // infos not available, and in compatible mode cannot + // fall back to git + return false; } - } - else { - Logger::Log(LogLevel::Error, - "dumping to file descriptor {} failed.", - fd); - return false; - } - } - return true; + if (repo_config and + not GitApi(repo_config.value()) + .RetrieveToFds({info}, {fd}, raw_tree)) { + return false; + } + return true; + }); } // NOLINTNEXTLINE(misc-no-recursion) @@ -191,24 +173,23 @@ class LocalApi final : public IExecutionApi { } // Determine missing artifacts in other CAS. - std::vector<ArtifactDigest> digests; - digests.reserve(artifacts_info.size()); - std::unordered_map<ArtifactDigest, Artifact::ObjectInfo> info_map; - for (auto const& info : artifacts_info) { - digests.push_back(info.digest); - info_map[info.digest] = info; - } - auto const& missing_digests = api->IsAvailable(digests); - std::vector<Artifact::ObjectInfo> missing_artifacts_info; - missing_artifacts_info.reserve(missing_digests.size()); - for (auto const& digest : missing_digests) { - missing_artifacts_info.push_back(info_map[digest]); + auto missing_artifacts_info = + GetMissingArtifactsInfo<Artifact::ObjectInfo>( + api, + artifacts_info.begin(), + artifacts_info.end(), + [](Artifact::ObjectInfo const& info) { return info.digest; }); + if (not missing_artifacts_info) { + Logger::Log(LogLevel::Error, + "LocalApi: Failed to retrieve the missing artifacts"); + return false; } // Collect blobs of missing artifacts from local CAS. Trees are // processed recursively before any blob is uploaded. BlobContainer container{}; - for (auto const& info : missing_artifacts_info) { + for (auto const& dgst : missing_artifacts_info->digests) { + auto const& info = missing_artifacts_info->back_map[dgst]; // Recursively process trees. if (IsTreeObject(info.type)) { auto const& infos = storage_->CAS().ReadDirectTreeEntries( @@ -325,50 +306,6 @@ class LocalApi final : public IExecutionApi { return false; } - /// NOLINTNEXTLINE(misc-no-recursion) - [[nodiscard]] auto UploadBlobTree(BlobTreePtr const& blob_tree) noexcept - -> bool { - - // Create digest list from blobs for batch availability check. - std::vector<ArtifactDigest> digests; - digests.reserve(blob_tree->size()); - std::unordered_map<ArtifactDigest, BlobTreePtr> tree_map; - for (auto const& node : *blob_tree) { - auto digest = ArtifactDigest{node->Blob().digest}; - digests.emplace_back(digest); - try { - tree_map.emplace(std::move(digest), node); - } catch (...) { - return false; - } - } - - // Find missing digests. - auto missing_digests = IsAvailable(digests); - - // Process missing blobs. - BlobContainer container; - for (auto const& digest : missing_digests) { - if (auto it = tree_map.find(digest); it != tree_map.end()) { - auto const& node = it->second; - // Process trees. - if (node->IsTree()) { - if (not UploadBlobTree(node)) { - return false; - } - } - // Store blob. - try { - container.Emplace(node->Blob()); - } catch (...) { - return false; - } - } - } - - return Upload(container, /*skip_find_missing=*/true); - } - [[nodiscard]] auto UploadTree( std::vector<DependencyGraph::NamedArtifactNodePtr> const& artifacts) noexcept -> std::optional<ArtifactDigest> final { @@ -380,63 +317,22 @@ class LocalApi final : public IExecutionApi { } if (Compatibility::IsCompatible()) { - BlobContainer blobs{}; - auto const& cas = storage_->CAS(); - auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree( + return CommonUploadTreeCompatible( + this, *build_root, - [&cas](std::vector<bazel_re::Digest> const& digests, - std::vector<std::string>* targets) { + [&cas = storage_->CAS()]( + std::vector<bazel_re::Digest> const& digests, + std::vector<std::string>* targets) { targets->reserve(digests.size()); for (auto const& digest : digests) { auto p = cas.BlobPath(digest, /*is_executable=*/false); auto content = FileSystemManager::ReadFile(*p); targets->emplace_back(*content); } - }, - [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); }); - if (not digest) { - Logger::Log(LogLevel::Debug, - "failed to create digest for build root."); - return std::nullopt; - } - Logger::Log(LogLevel::Trace, [&digest]() { - std::ostringstream oss{}; - oss << "upload root directory" << std::endl; - oss << fmt::format(" - root digest: {}", digest->hash()) - << std::endl; - return oss.str(); - }); - if (not Upload(blobs, /*skip_find_missing=*/false)) { - Logger::Log(LogLevel::Debug, - "failed to upload blobs for build root."); - return std::nullopt; - } - return ArtifactDigest{*digest}; + }); } - auto blob_tree = BlobTree::FromDirectoryTree(*build_root); - if (not blob_tree) { - Logger::Log(LogLevel::Debug, - "failed to create blob tree for build root."); - return std::nullopt; - } - auto tree_blob = (*blob_tree)->Blob(); - // Upload blob tree if tree is not available at the remote side (content - // first). - if (not IsAvailable(ArtifactDigest{tree_blob.digest})) { - if (not UploadBlobTree(*blob_tree)) { - Logger::Log(LogLevel::Debug, - "failed to upload blob tree for build root."); - return std::nullopt; - } - if (not Upload(BlobContainer{{tree_blob}}, - /*skip_find_missing=*/true)) { - Logger::Log(LogLevel::Debug, - "failed to upload tree blob for build root."); - return std::nullopt; - } - } - return ArtifactDigest{tree_blob.digest}; + return CommonUploadTreeNative(this, *build_root); } [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index ef6c9d00..88310cbb 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -74,6 +74,7 @@ , ["src/buildtool/multithreading", "task_system"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/execution_api/common", "common_api"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/storage", "fs_utils"] ] diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index 4ae8b3d8..0e3a1ca2 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -30,6 +30,7 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_action.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" @@ -276,35 +277,15 @@ auto BazelApi::CreateAction( std::vector<Artifact::ObjectInfo> const& artifacts_info, std::vector<int> const& fds, bool raw_tree) noexcept -> bool { - if (artifacts_info.size() != fds.size()) { - Logger::Log(LogLevel::Warning, - "different number of digests and file descriptors."); - return false; - } - - for (std::size_t i{}; i < artifacts_info.size(); ++i) { - auto fd = fds[i]; - auto const& info = artifacts_info[i]; - - if (gsl::owner<FILE*> out = fdopen(fd, "wb")) { // NOLINT - auto const success = network_->DumpToStream(info, out, raw_tree); - std::fclose(out); - if (not success) { - Logger::Log(LogLevel::Warning, - "dumping {} {} to file descriptor {} failed.", - IsTreeObject(info.type) ? "tree" : "blob", - info.ToString(), - fd); - return false; - } - } - else { - Logger::Log( - LogLevel::Warning, "opening file descriptor {} failed.", fd); - return false; - } - } - return true; + return CommonRetrieveToFds( + artifacts_info, + fds, + [&network = network_, &raw_tree](Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& out) { + return network->DumpToStream(info, out, raw_tree); + }, + std::nullopt // remote can't fallback to Git + ); } // NOLINTNEXTLINE(misc-no-recursion) @@ -318,23 +299,21 @@ auto BazelApi::CreateAction( } // Determine missing artifacts in other CAS. - std::vector<ArtifactDigest> digests; - digests.reserve(artifacts_info.size()); - std::unordered_map<ArtifactDigest, Artifact::ObjectInfo> info_map; - for (auto const& info : artifacts_info) { - digests.push_back(info.digest); - info_map[info.digest] = info; - } - auto const& missing_digests = api->IsAvailable(digests); - std::vector<Artifact::ObjectInfo> missing_artifacts_info; - missing_artifacts_info.reserve(missing_digests.size()); - for (auto const& digest : missing_digests) { - missing_artifacts_info.push_back(info_map[digest]); + auto missing_artifacts_info = GetMissingArtifactsInfo<Artifact::ObjectInfo>( + api, + artifacts_info.begin(), + artifacts_info.end(), + [](Artifact::ObjectInfo const& info) { return info.digest; }); + if (not missing_artifacts_info) { + Logger::Log(LogLevel::Error, + "BazelApi: Failed to retrieve the missing artifacts"); + return false; } // Recursively process trees. std::vector<bazel_re::Digest> blob_digests{}; - for (auto const& info : missing_artifacts_info) { + for (auto const& dgst : missing_artifacts_info->digests) { + auto const& info = missing_artifacts_info->back_map[dgst]; if (IsTreeObject(info.type)) { auto const infos = network_->ReadDirectTreeEntries( info.digest, std::filesystem::path{}); @@ -349,7 +328,8 @@ auto BazelApi::CreateAction( blob_digests.push_back(info.digest); } - return ::RetrieveToCas(blob_digests, api, network_, info_map); + return ::RetrieveToCas( + blob_digests, api, network_, missing_artifacts_info->back_map); } /// NOLINTNEXTLINE(misc-no-recursion) @@ -365,25 +345,23 @@ auto BazelApi::CreateAction( } // Determine missing artifacts in other CAS. - std::vector<ArtifactDigest> digests; - digests.reserve(artifacts_info.size()); - std::unordered_map<ArtifactDigest, Artifact::ObjectInfo> info_map; - for (auto const& info : artifacts_info) { - digests.push_back(info.digest); - info_map[info.digest] = info; - } - auto const& missing_digests = api->IsAvailable(digests); - std::vector<Artifact::ObjectInfo> missing_artifacts_info; - missing_artifacts_info.reserve(missing_digests.size()); - for (auto const& digest : missing_digests) { - missing_artifacts_info.push_back(info_map[digest]); + auto missing_artifacts_info = GetMissingArtifactsInfo<Artifact::ObjectInfo>( + api, + artifacts_info.begin(), + artifacts_info.end(), + [](Artifact::ObjectInfo const& info) { return info.digest; }); + if (not missing_artifacts_info) { + Logger::Log(LogLevel::Error, + "BazelApi: Failed to retrieve the missing artifacts"); + return false; } // Recursively process trees. std::atomic_bool failure{false}; try { auto ts = TaskSystem{jobs}; - for (auto const& info : missing_artifacts_info) { + for (auto const& dgst : missing_artifacts_info->digests) { + auto const& info = missing_artifacts_info->back_map[dgst]; if (IsTreeObject(info.type)) { auto const infos = network_->ReadDirectTreeEntries( info.digest, std::filesystem::path{}); @@ -397,18 +375,22 @@ auto BazelApi::CreateAction( // Object infos created by network_->ReadTreeInfos() will contain 0 // as size, but this is handled by the remote execution engine, so // no need to regenerate the digest. - ts.QueueTask( - [this, &info, &api, &failure, &info_map, use_blob_splitting]() { - if (use_blob_splitting and network_->BlobSplitSupport() and - api->BlobSpliceSupport() - ? ::RetrieveToCasSplitted( - info, this, api, network_, info_map) - : ::RetrieveToCas( - {info.digest}, api, network_, info_map)) { - return; - } - failure = true; - }); + ts.QueueTask([this, + &info, + &api, + &failure, + &info_map = missing_artifacts_info->back_map, + use_blob_splitting]() { + if (use_blob_splitting and network_->BlobSplitSupport() and + api->BlobSpliceSupport() + ? ::RetrieveToCasSplitted( + info, this, api, network_, info_map) + : ::RetrieveToCas( + {info.digest}, api, network_, info_map)) { + return; + } + failure = true; + }); } } catch (std::exception const& ex) { Logger::Log(LogLevel::Warning, @@ -435,50 +417,6 @@ auto BazelApi::CreateAction( return network_->UploadBlobs(blobs, skip_find_missing); } -/// NOLINTNEXTLINE(misc-no-recursion) -[[nodiscard]] auto BazelApi::UploadBlobTree( - BlobTreePtr const& blob_tree) noexcept -> bool { - - // Create digest list from blobs for batch availability check. - std::vector<bazel_re::Digest> digests; - digests.reserve(blob_tree->size()); - std::unordered_map<bazel_re::Digest, BlobTreePtr> tree_map; - for (auto const& node : *blob_tree) { - auto digest = node->Blob().digest; - digests.emplace_back(digest); - try { - tree_map.emplace(std::move(digest), node); - } catch (...) { - return false; - } - } - - // Find missing digests. - auto missing_digests = network_->IsAvailable(digests); - - // Process missing blobs. - BlobContainer container; - for (auto const& digest : missing_digests) { - if (auto it = tree_map.find(digest); it != tree_map.end()) { - auto const& node = it->second; - // Process trees. - if (node->IsTree()) { - if (not UploadBlobTree(node)) { - return false; - } - } - // Store blob. - try { - container.Emplace(node->Blob()); - } catch (...) { - return false; - } - } - } - - return network_->UploadBlobs(container, /*skip_find_missing=*/true); -} - [[nodiscard]] auto BazelApi::UploadTree( std::vector<DependencyGraph::NamedArtifactNodePtr> const& artifacts) noexcept -> std::optional<ArtifactDigest> { @@ -490,12 +428,11 @@ auto BazelApi::CreateAction( } if (Compatibility::IsCompatible()) { - BlobContainer blobs{}; - auto const& network = network_; - auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree( + return CommonUploadTreeCompatible( + this, *build_root, - [&network](std::vector<bazel_re::Digest> const& digests, - std::vector<std::string>* targets) { + [&network = network_](std::vector<bazel_re::Digest> const& digests, + std::vector<std::string>* targets) { auto reader = network->ReadBlobs(digests); auto blobs = reader.Next(); targets->reserve(digests.size()); @@ -505,51 +442,10 @@ auto BazelApi::CreateAction( } blobs = reader.Next(); } - }, - [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); }); - if (not digest) { - Logger::Log(LogLevel::Debug, - "failed to create digest for build root."); - return std::nullopt; - } - Logger::Log(LogLevel::Trace, [&digest]() { - std::ostringstream oss{}; - oss << "upload root directory" << std::endl; - oss << fmt::format(" - root digest: {}", digest->hash()) - << std::endl; - return oss.str(); - }); - if (not Upload(blobs, /*skip_find_missing=*/false)) { - Logger::Log(LogLevel::Debug, - "failed to upload blobs for build root."); - return std::nullopt; - } - return ArtifactDigest{*digest}; + }); } - auto blob_tree = BlobTree::FromDirectoryTree(*build_root); - if (not blob_tree) { - Logger::Log(LogLevel::Debug, - "failed to create blob tree for build root."); - return std::nullopt; - } - auto tree_blob = (*blob_tree)->Blob(); - // Upload blob tree if tree is not available at the remote side (content - // first). - if (not network_->IsAvailable(tree_blob.digest)) { - if (not UploadBlobTree(*blob_tree)) { - Logger::Log(LogLevel::Debug, - "failed to upload blob tree for build root."); - return std::nullopt; - } - if (not Upload(BlobContainer{{tree_blob}}, - /*skip_find_missing=*/true)) { - Logger::Log(LogLevel::Debug, - "failed to upload tree blob for build root."); - return std::nullopt; - } - } - return ArtifactDigest{tree_blob.digest}; + return CommonUploadTreeNative(this, *build_root); } [[nodiscard]] auto BazelApi::IsAvailable( diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp index adb6bd33..3411b6b7 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp @@ -111,9 +111,6 @@ class BazelApi final : public IExecutionApi { private: std::shared_ptr<BazelNetwork> network_; - - [[nodiscard]] auto UploadBlobTree(BlobTreePtr const& blob_tree) noexcept - -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_API_HPP |