diff options
Diffstat (limited to 'src/buildtool/execution_api/remote')
5 files changed, 55 insertions, 128 deletions
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index 47d024f9..0f1fdd04 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -31,6 +31,7 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" +#include "src/buildtool/execution_api/common/stream_dumper.hpp" #include "src/buildtool/execution_api/common/tree_reader.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_action.hpp" @@ -284,12 +285,13 @@ auto BazelApi::CreateAction( std::vector<Artifact::ObjectInfo> const& artifacts_info, std::vector<int> const& fds, bool raw_tree) noexcept -> bool { + auto dumper = StreamDumper<BazelNetworkReader>{*network_}; return CommonRetrieveToFds( artifacts_info, fds, - [&network = network_, &raw_tree](Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& out) { - return network->DumpToStream(info, out, raw_tree); + [&dumper, &raw_tree](Artifact::ObjectInfo const& info, + gsl::not_null<FILE*> const& out) { + return dumper.DumpToStream(info, out, raw_tree); }, std::nullopt // remote can't fallback to Git ); diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index 485e65dc..4cc3a274 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -23,118 +23,6 @@ #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" -namespace { - -[[nodiscard]] auto ReadDirectory( - gsl::not_null<BazelNetwork const*> const& network, - bazel_re::Digest const& digest) noexcept - -> std::optional<bazel_re::Directory> { - auto blobs = network->ReadBlobs({digest}).Next(); - if (blobs.size() == 1) { - return BazelMsgFactory::MessageFromString<bazel_re::Directory>( - blobs.at(0).data); - } - Logger::Log(LogLevel::Debug, - "Directory {} not found in CAS", - NativeSupport::Unprefix(digest.hash())); - return std::nullopt; -} - -[[nodiscard]] auto ReadGitTree( - gsl::not_null<BazelNetwork const*> const& network, - bazel_re::Digest const& digest) noexcept - -> std::optional<GitRepo::tree_entries_t> { - auto blobs = network->ReadBlobs({digest}).Next(); - if (blobs.size() == 1) { - auto const& content = blobs.at(0).data; - auto check_symlinks = - [&network](std::vector<bazel_re::Digest> const& ids) { - auto size = ids.size(); - auto reader = network->ReadBlobs(ids); - auto blobs = reader.Next(); - std::size_t count{}; - while (not blobs.empty()) { - if (count + blobs.size() > size) { - Logger::Log(LogLevel::Debug, - "received more blobs than requested."); - return false; - } - for (auto const& blob : blobs) { - if (not PathIsNonUpwards(blob.data)) { - return false; - } - } - count += blobs.size(); - blobs = reader.Next(); - } - return true; - }; - return GitRepo::ReadTreeData( - content, - HashFunction::ComputeTreeHash(content).Bytes(), - check_symlinks, - /*is_hex_id=*/false); - } - Logger::Log(LogLevel::Debug, - "Tree {} not found in CAS", - NativeSupport::Unprefix(digest.hash())); - return std::nullopt; -} - -[[nodiscard]] auto TreeToStream( - gsl::not_null<BazelNetwork const*> const& network, - bazel_re::Digest const& tree_digest, - gsl::not_null<FILE*> const& stream, - bool raw_tree) noexcept -> bool { - if (raw_tree) { - auto blobs = network->ReadBlobs({tree_digest}).Next(); - if (blobs.size() != 1) { - Logger::Log(LogLevel::Debug, - "Object {} not found in CAS", - NativeSupport::Unprefix(tree_digest.hash())); - return false; - } - auto const& str = blobs.at(0).data; - std::fwrite(str.data(), 1, str.size(), stream); - return true; - } - if (Compatibility::IsCompatible()) { - if (auto dir = ReadDirectory(network, tree_digest)) { - if (auto data = BazelMsgFactory::DirectoryToString(*dir)) { - auto const& str = *data; - std::fwrite(str.data(), 1, str.size(), stream); - return true; - } - } - } - else { - if (auto entries = ReadGitTree(network, tree_digest)) { - if (auto data = BazelMsgFactory::GitTreeToString(*entries)) { - auto const& str = *data; - std::fwrite(str.data(), 1, str.size(), stream); - return true; - } - } - } - return false; -} - -[[nodiscard]] auto BlobToStream( - gsl::not_null<BazelNetwork const*> const& network, - bazel_re::Digest const& blob_digest, - gsl::not_null<FILE*> const& stream) noexcept -> bool { - auto reader = network->IncrementalReadSingleBlob(blob_digest); - auto data = reader.Next(); - while (data and not data->empty()) { - auto const& str = *data; - std::fwrite(str.data(), 1, str.size(), stream); - data = reader.Next(); - } - return data.has_value(); -} - -} // namespace - BazelNetwork::BazelNetwork(std::string instance_name, std::string const& host, Port port, @@ -314,11 +202,3 @@ auto BazelNetwork::QueryFullTree(bazel_re::Digest const& digest) const noexcept } return cas_->GetTree(instance_name_, digest, kMaxBatchTransferSize); } - -auto BazelNetwork::DumpToStream(Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& stream, - bool raw_tree) const noexcept -> bool { - return IsTreeObject(info.type) - ? TreeToStream(this, info.digest, stream, raw_tree) - : BlobToStream(this, info.digest, stream); -} diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp index 6ee44921..82a20c4d 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp @@ -110,10 +110,6 @@ class BazelNetwork { std::vector<std::string> const& output_files) const noexcept -> std::optional<bazel_re::ActionResult>; - [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, - gsl::not_null<FILE*> const& stream, - bool raw_tree) const noexcept -> bool; - /// \brief Query full tree from remote CAS. Note that this is currently not // supported by Buildbarn revision c3c06bbe2a. [[nodiscard]] auto QueryFullTree(bazel_re::Digest const& digest) diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp index ee0fbc78..a0e96237 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp @@ -92,6 +92,41 @@ auto BazelNetworkReader::ReadGitTree(ArtifactDigest const& digest) return std::nullopt; } +auto BazelNetworkReader::DumpRawTree(Artifact::ObjectInfo const& info, + DumpCallback const& dumper) const noexcept + -> bool { + auto blobs = network_.ReadBlobs({info.digest}).Next(); + if (blobs.size() != 1) { + Logger::Log( + LogLevel::Debug, "Object {} not found in CAS", info.digest.hash()); + return false; + } + + try { + return std::invoke(dumper, blobs.at(0).data); + } catch (...) { + return false; + } +} + +auto BazelNetworkReader::DumpBlob(Artifact::ObjectInfo const& info, + DumpCallback const& dumper) const noexcept + -> bool { + auto reader = network_.IncrementalReadSingleBlob(info.digest); + auto data = reader.Next(); + while (data and not data->empty()) { + try { + if (not std::invoke(dumper, *data)) { + return false; + } + } catch (...) { + return false; + } + data = reader.Next(); + } + return data.has_value(); +} + auto BazelNetworkReader::MakeAuxiliaryMap( std::vector<bazel_re::Directory>&& full_tree) noexcept -> std::optional<DirectoryMap> { diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp index 8d46c9f8..fb5f5184 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp @@ -15,9 +15,13 @@ #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP +#include <filesystem> +#include <functional> #include <optional> +#include <string> #include <unordered_map> +#include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp" @@ -25,6 +29,8 @@ class BazelNetworkReader final { public: + using DumpCallback = std::function<bool(std::string const&)>; + explicit BazelNetworkReader( BazelNetwork const& network, std::optional<ArtifactDigest> request_remote_tree = @@ -36,6 +42,14 @@ class BazelNetworkReader final { [[nodiscard]] auto ReadGitTree(ArtifactDigest const& digest) const noexcept -> std::optional<GitRepo::tree_entries_t>; + [[nodiscard]] auto DumpRawTree(Artifact::ObjectInfo const& info, + DumpCallback const& dumper) const noexcept + -> bool; + + [[nodiscard]] auto DumpBlob(Artifact::ObjectInfo const& info, + DumpCallback const& dumper) const noexcept + -> bool; + private: using DirectoryMap = std::unordered_map<bazel_re::Digest, bazel_re::Directory>; @@ -48,4 +62,4 @@ class BazelNetworkReader final { -> std::optional<DirectoryMap>; }; -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP
\ No newline at end of file +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP |