diff options
Diffstat (limited to 'src/buildtool/execution_api/common')
| -rw-r--r-- | src/buildtool/execution_api/common/TARGETS | 20 | ||||
| -rw-r--r-- | src/buildtool/execution_api/common/common_api.cpp | 178 | ||||
| -rw-r--r-- | src/buildtool/execution_api/common/common_api.hpp | 99 |
3 files changed, 297 insertions, 0 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index d2fcfc6d..210cbb3a 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -48,4 +48,24 @@ , "deps": [["@", "grpc", "", "grpc++"]] , "stage": ["src", "buildtool", "execution_api", "common"] } +, "common_api": + { "type": ["@", "rules", "CC", "library"] + , "name": ["common_api"] + , "hdrs": ["common_api.hpp"] + , "srcs": ["common_api.cpp"] + , "deps": + [ "common" + , ["@", "gsl", "", "gsl"] + , ["src/buildtool/common", "common"] + , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"] + , ["src/buildtool/execution_api/bazel_msg", "blob_tree"] + , ["src/buildtool/execution_api/bazel_msg", "directory_tree"] + ] + , "stage": ["src", "buildtool", "execution_api", "common"] + , "private-deps": + [ ["@", "fmt", "", "fmt"] + , ["src/buildtool/logging", "logging"] + , ["src/buildtool/logging", "log_level"] + ] + } } diff --git a/src/buildtool/execution_api/common/common_api.cpp b/src/buildtool/execution_api/common/common_api.cpp new file mode 100644 index 00000000..d1b3ad03 --- /dev/null +++ b/src/buildtool/execution_api/common/common_api.cpp @@ -0,0 +1,178 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/common/common_api.hpp" + +#include <cstddef> +#include <exception> + +#include "fmt/core.h" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" + +auto CommonRetrieveToFds( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + std::vector<int> const& fds, + std::function<bool(Artifact::ObjectInfo const&, + gsl::not_null<FILE*> const&)> const& dump_to_stream, + std::optional<std::function<bool(Artifact::ObjectInfo const&, int)>> const& + fallback) noexcept -> bool { + if (artifacts_info.size() != fds.size()) { + Logger::Log(LogLevel::Error, + "different number of digests and file descriptors."); + return false; + } + + for (std::size_t i{}; i < artifacts_info.size(); ++i) { + auto fd = fds[i]; + auto const& info = artifacts_info[i]; + + if (gsl::owner<FILE*> out = fdopen(dup(fd), "wb")) { // NOLINT + bool success{false}; + try { + success = dump_to_stream(info, out); + } catch (std::exception const& ex) { + std::fclose(out); // close file + Logger::Log(LogLevel::Error, + "dumping {} to stream failed with:\n{}", + info.ToString(), + ex.what()); + return false; + } + std::fclose(out); + if (not success) { + Logger::Log( + LogLevel::Debug, + "dumping {} {} from CAS to file descriptor {} failed.", + IsTreeObject(info.type) ? "tree" : "blob", + info.ToString(), + fd); + // locally we might be able to fallback to Git in native mode + try { + if (fallback and not(*fallback)(info, fd)) { + return false; + } + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "fallback dumping {} to file descriptor {} " + "failed with:\n{}", + info.ToString(), + fd, + ex.what()); + return false; + } + } + } + else { + Logger::Log( + LogLevel::Error, "dumping to file descriptor {} failed.", fd); + return false; + } + } + return true; +} + +/// NOLINTNEXTLINE(misc-no-recursion) +auto CommonUploadBlobTree(BlobTreePtr const& blob_tree, + gsl::not_null<IExecutionApi*> const& api) noexcept + -> bool { + // Create digest list from blobs for batch availability check. + auto missing_blobs_info = GetMissingArtifactsInfo<BlobTreePtr>( + api, blob_tree->begin(), blob_tree->end(), [](BlobTreePtr const& node) { + return ArtifactDigest{node->Blob().digest}; + }); + if (not missing_blobs_info) { + Logger::Log(LogLevel::Error, + "Failed to retrieve the missing tree blobs for upload"); + return false; + } + + // Process missing blobs. + BlobContainer container; + for (auto const& digest : missing_blobs_info->digests) { + if (auto it = missing_blobs_info->back_map.find(digest); + it != missing_blobs_info->back_map.end()) { + auto const& node = it->second; + // Process trees. + if (node->IsTree()) { + if (not CommonUploadBlobTree(node, api)) { + return false; + } + } + // Store blob. + try { + container.Emplace(node->Blob()); + } catch (...) { + return false; + } + } + } + + return api->Upload(container, /*skip_find_missing=*/true); +} + +auto CommonUploadTreeCompatible( + gsl::not_null<IExecutionApi*> const& api, + DirectoryTreePtr const& build_root, + BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) noexcept + -> std::optional<ArtifactDigest> { + BlobContainer blobs{}; + auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree( + build_root, resolve_links, [&blobs](BazelBlob&& blob) { + blobs.Emplace(std::move(blob)); + }); + if (not digest) { + Logger::Log(LogLevel::Debug, "failed to create digest for build root."); + return std::nullopt; + } + Logger::Log(LogLevel::Trace, [&digest]() { + std::ostringstream oss{}; + oss << "upload root directory" << std::endl; + oss << fmt::format(" - root digest: {}", digest->hash()) << std::endl; + return oss.str(); + }); + if (not api->Upload(blobs, /*skip_find_missing=*/false)) { + Logger::Log(LogLevel::Debug, "failed to upload blobs for build root."); + return std::nullopt; + } + return ArtifactDigest{*digest}; +} + +auto CommonUploadTreeNative(gsl::not_null<IExecutionApi*> const& api, + DirectoryTreePtr const& build_root) noexcept + -> std::optional<ArtifactDigest> { + auto blob_tree = BlobTree::FromDirectoryTree(build_root); + if (not blob_tree) { + Logger::Log(LogLevel::Debug, + "failed to create blob tree for build root."); + return std::nullopt; + } + auto tree_blob = (*blob_tree)->Blob(); + // Upload blob tree if tree is not available at the remote side (content + // first). + if (not api->IsAvailable(ArtifactDigest{tree_blob.digest})) { + if (not CommonUploadBlobTree(*blob_tree, api)) { + Logger::Log(LogLevel::Debug, + "failed to upload blob tree for build root."); + return std::nullopt; + } + if (not api->Upload(BlobContainer{{tree_blob}}, + /*skip_find_missing=*/true)) { + Logger::Log(LogLevel::Debug, + "failed to upload tree blob for build root."); + return std::nullopt; + } + } + return ArtifactDigest{tree_blob.digest}; +} diff --git a/src/buildtool/execution_api/common/common_api.hpp b/src/buildtool/execution_api/common/common_api.hpp new file mode 100644 index 00000000..312f97db --- /dev/null +++ b/src/buildtool/execution_api/common/common_api.hpp @@ -0,0 +1,99 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP + +#include <cstdio> +#include <exception> +#include <functional> +#include <optional> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "gsl/gsl" +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" +#include "src/buildtool/execution_api/bazel_msg/directory_tree.hpp" +#include "src/buildtool/execution_api/common/execution_api.hpp" + +/// \brief Stores a list of missing artifact digests, as well as a back-mapping +/// to some given original type. +template <typename T> +struct MissingArtifactsInfo { + std::vector<ArtifactDigest> digests; + std::unordered_map<ArtifactDigest, T> back_map; +}; + +/// \brief Common logic for RetrieveToFds. +/// \param dump_to_stream Dumps the artifact to the respective open stream. +/// \param fallback Processes the respective file descriptor further in case the +/// regular dump fails. +[[nodiscard]] auto CommonRetrieveToFds( + std::vector<Artifact::ObjectInfo> const& artifacts_info, + std::vector<int> const& fds, + std::function<bool(Artifact::ObjectInfo const&, + gsl::not_null<FILE*> const&)> const& dump_to_stream, + std::optional<std::function<bool(Artifact::ObjectInfo const&, int)>> const& + fallback) noexcept -> bool; + +/// \brief Get the missing artifacts from a given input list, needed, e.g., to +/// be uploaded. +/// \returns A struct storing the missing artifacts and a back-mapping to the +/// original given type, or nullopt in case of exceptions. +template <typename T> +[[nodiscard]] auto GetMissingArtifactsInfo( + gsl::not_null<IExecutionApi*> const& api, + typename std::vector<T>::const_iterator const& begin, + typename std::vector<T>::const_iterator const& end, + typename std::function<ArtifactDigest(T const&)> const& converter) noexcept + -> std::optional<MissingArtifactsInfo<T>> { + std::vector<ArtifactDigest> digests; + digests.reserve(end - begin); + MissingArtifactsInfo<T> res{}; + for (auto it = begin; it != end; ++it) { + try { + auto dgst = converter(*it); // can't enforce it to be noexcept + digests.emplace_back(dgst); + res.back_map.emplace(std::move(dgst), *it); + } catch (...) { + return std::nullopt; + } + } + res.digests = api->IsAvailable(digests); + return res; +} + +/// \brief Upload missing blobs from a given BlobTree. +[[nodiscard]] auto CommonUploadBlobTree( + BlobTreePtr const& blob_tree, + gsl::not_null<IExecutionApi*> const& api) noexcept -> bool; + +/// \brief Runs the compatible branch of local/bazel UploadTree API. +[[nodiscard]] auto CommonUploadTreeCompatible( + gsl::not_null<IExecutionApi*> const& api, + DirectoryTreePtr const& build_root, + BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) noexcept + -> std::optional<ArtifactDigest>; + +/// \brief Runs the native branch of local/bazel UploadTree API. +[[nodiscard]] auto CommonUploadTreeNative( + gsl::not_null<IExecutionApi*> const& api, + DirectoryTreePtr const& build_root) noexcept + -> std::optional<ArtifactDigest>; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_COMMON_API_HPP |