diff options
Diffstat (limited to 'src')
5 files changed, 86 insertions, 53 deletions
diff --git a/src/buildtool/execution_api/common/tree_reader_utils.cpp b/src/buildtool/execution_api/common/tree_reader_utils.cpp index 151594d0..250d8245 100644 --- a/src/buildtool/execution_api/common/tree_reader_utils.cpp +++ b/src/buildtool/execution_api/common/tree_reader_utils.cpp @@ -27,24 +27,35 @@ #include "src/utils/cpp/hex_string.hpp" namespace { -[[nodiscard]] auto CreateObjectInfo(bazel_re::DirectoryNode const& node) - -> Artifact::ObjectInfo { - return Artifact::ObjectInfo{.digest = ArtifactDigest{node.digest()}, +[[nodiscard]] auto CreateObjectInfo(HashFunction hash_function, + bazel_re::DirectoryNode const& node) + -> std::optional<Artifact::ObjectInfo> { + auto digest = ArtifactDigestFactory::FromBazel(hash_function.GetType(), + node.digest()); + if (not digest) { + return std::nullopt; + } + return Artifact::ObjectInfo{.digest = *std::move(digest), .type = ObjectType::Tree}; } -[[nodiscard]] auto CreateObjectInfo(bazel_re::FileNode const& node) - -> Artifact::ObjectInfo { - return Artifact::ObjectInfo{.digest = ArtifactDigest{node.digest()}, +[[nodiscard]] auto CreateObjectInfo(HashFunction hash_function, + bazel_re::FileNode const& node) + -> std::optional<Artifact::ObjectInfo> { + auto digest = ArtifactDigestFactory::FromBazel(hash_function.GetType(), + node.digest()); + if (not digest) { + return std::nullopt; + } + return Artifact::ObjectInfo{.digest = *std::move(digest), .type = node.is_executable() ? ObjectType::Executable : ObjectType::File}; } -[[nodiscard]] auto CreateObjectInfo(bazel_re::SymlinkNode const& node, - HashFunction hash_function) +[[nodiscard]] auto CreateObjectInfo(HashFunction hash_function, + bazel_re::SymlinkNode const& node) -> Artifact::ObjectInfo { - return Artifact::ObjectInfo{ .digest = ArtifactDigestFactory::HashDataAs<ObjectType::File>( hash_function, node.target()), @@ -84,22 +95,24 @@ template <typename TTree> auto TreeReaderUtils::ReadObjectInfos(bazel_re::Directory const& dir, InfoStoreFunc const& store_info) noexcept -> bool { + // SHA256 is used since bazel types are processed here. + HashFunction const hash_function{HashFunction::Type::PlainSHA256}; try { for (auto const& f : dir.files()) { - if (not store_info(f.name(), CreateObjectInfo(f))) { + auto const info = CreateObjectInfo(hash_function, f); + if (not info or not store_info(f.name(), *info)) { return false; } } - // SHA256 is used since bazel types are processed here. - HashFunction const hash_function{HashFunction::Type::PlainSHA256}; for (auto const& l : dir.symlinks()) { - if (not store_info(l.name(), CreateObjectInfo(l, hash_function))) { + if (not store_info(l.name(), CreateObjectInfo(hash_function, l))) { return false; } } for (auto const& d : dir.directories()) { - if (not store_info(d.name(), CreateObjectInfo(d))) { + auto const info = CreateObjectInfo(hash_function, d); + if (not store_info(d.name(), *info)) { return false; } } diff --git a/src/buildtool/execution_api/local/local_cas_reader.cpp b/src/buildtool/execution_api/local/local_cas_reader.cpp index 3e5ebe7b..2da3bc00 100644 --- a/src/buildtool/execution_api/local/local_cas_reader.cpp +++ b/src/buildtool/execution_api/local/local_cas_reader.cpp @@ -23,6 +23,7 @@ #include <vector> #include "gsl/gsl" +#include "src/buildtool/common/artifact_digest_factory.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" @@ -52,6 +53,7 @@ auto LocalCasReader::ReadDirectory(ArtifactDigest const& digest) const noexcept auto LocalCasReader::MakeTree(ArtifactDigest const& root) const noexcept -> std::optional<bazel_re::Tree> { + auto const hash_type = cas_.GetHashFunction().GetType(); try { std::unordered_map<ArtifactDigest, bazel_re::Directory> directories; @@ -70,7 +72,12 @@ auto LocalCasReader::MakeTree(ArtifactDigest const& root) const noexcept return std::nullopt; } for (auto const& node : read_dir->directories()) { - to_check.push(ArtifactDigest{node.digest()}); + auto digest = + ArtifactDigestFactory::FromBazel(hash_type, node.digest()); + if (not digest) { + return std::nullopt; + } + to_check.push(*std::move(digest)); } directories.insert_or_assign(std::move(current), *std::move(read_dir)); diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index c3cedd7a..41079f33 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -40,6 +40,7 @@ , ["src/buildtool/common/remote", "retry_config"] , ["src/buildtool/file_system", "git_repo"] , ["src/buildtool/crypto", "hash_function"] + , ["src/buildtool/crypto", "hash_info"] ] , "proto": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp index be28d58e..02f189d0 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp @@ -17,6 +17,7 @@ #include <algorithm> #include "src/buildtool/common/artifact_digest_factory.hpp" +#include "src/buildtool/common/bazel_digest_factory.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/message_limits.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" @@ -44,7 +45,7 @@ BazelNetworkReader::BazelNetworkReader( // supported by Buildbarn revision c3c06bbe2a. auto full_tree = cas_.GetTree(instance_name_, - static_cast<bazel_re::Digest>(*request_remote_tree), + ArtifactDigestFactory::ToBazel(*request_remote_tree), kMaxBatchTransferSize); auxiliary_map_ = MakeAuxiliaryMap(std::move(full_tree)); } @@ -126,7 +127,7 @@ auto BazelNetworkReader::DumpBlob(Artifact::ObjectInfo const& info, DumpCallback const& dumper) const noexcept -> bool { auto reader = cas_.IncrementalReadSingleBlob( - instance_name_, static_cast<bazel_re::Digest>(info.digest)); + instance_name_, ArtifactDigestFactory::ToBazel(info.digest)); auto data = reader.Next(); while (data and not data->empty()) { try { @@ -163,16 +164,22 @@ auto BazelNetworkReader::MakeAuxiliaryMap( auto BazelNetworkReader::ReadSingleBlob(bazel_re::Digest const& digest) const noexcept -> std::optional<ArtifactBlob> { auto blob = cas_.ReadSingleBlob(instance_name_, digest); - if (blob and Validate(*blob)) { - return ArtifactBlob{ - ArtifactDigest{blob->digest}, blob->data, blob->is_exec}; + if (not blob) { + return std::nullopt; } - return std::nullopt; + auto hash_info = Validate(*blob); + if (not hash_info) { + return std::nullopt; + } + return ArtifactBlob{ + ArtifactDigest{*std::move(hash_info), blob->data->size()}, + blob->data, + blob->is_exec}; } auto BazelNetworkReader::ReadSingleBlob(ArtifactDigest const& digest) const noexcept -> std::optional<ArtifactBlob> { - return ReadSingleBlob(static_cast<bazel_re::Digest>(digest)); + return ReadSingleBlob(ArtifactDigestFactory::ToBazel(digest)); } auto BazelNetworkReader::ReadIncrementally( @@ -184,7 +191,7 @@ auto BazelNetworkReader::ReadIncrementally( digests.end(), std::back_inserter(bazel_digests), [](ArtifactDigest const& d) { - return static_cast<bazel_re::Digest>(d); + return ArtifactDigestFactory::ToBazel(d); }); return ReadIncrementally(std::move(bazel_digests)); } @@ -197,44 +204,47 @@ auto BazelNetworkReader::ReadIncrementally( auto BazelNetworkReader::BatchReadBlobs( std::vector<bazel_re::Digest> const& blobs) const noexcept -> std::vector<ArtifactBlob> { - std::vector<BazelBlob> result = + std::vector<BazelBlob> const result = cas_.BatchReadBlobs(instance_name_, blobs.begin(), blobs.end()); - auto it = std::remove_if( - result.begin(), result.end(), [this](BazelBlob const& blob) { - return not Validate(blob); - }); - result.erase(it, result.end()); - std::vector<ArtifactBlob> artifacts; artifacts.reserve(result.size()); - std::transform(result.begin(), - result.end(), - std::back_inserter(artifacts), - [](BazelBlob const& blob) { - return ArtifactBlob{ArtifactDigest{blob.digest}, - blob.data, - blob.is_exec}; - }); + for (auto const& blob : result) { + if (auto hash_info = Validate(blob)) { + artifacts.emplace_back( + ArtifactDigest{*std::move(hash_info), blob.data->size()}, + blob.data, + blob.is_exec); + } + } return artifacts; } auto BazelNetworkReader::Validate(BazelBlob const& blob) const noexcept - -> bool { - ArtifactDigest const rehashed_digest = - NativeSupport::IsTree(blob.digest.hash()) - ? ArtifactDigestFactory::HashDataAs<ObjectType::Tree>( - hash_function_, *blob.data) - : ArtifactDigestFactory::HashDataAs<ObjectType::File>( - hash_function_, *blob.data); - if (rehashed_digest == ArtifactDigest{blob.digest}) { - return true; + -> std::optional<HashInfo> { + // validate digest + auto requested_hash_info = + BazelDigestFactory::ToHashInfo(hash_function_.GetType(), blob.digest); + if (not requested_hash_info) { + Logger::Log(LogLevel::Warning, + "BazelNetworkReader: {}", + std::move(requested_hash_info).error()); + return std::nullopt; + } + + // rehash data + auto const rehashed_info = HashInfo::HashData( + hash_function_, *blob.data, requested_hash_info->IsTree()); + + // ensure rehashed data produce the same hash + if (*requested_hash_info != rehashed_info) { + Logger::Log(LogLevel::Warning, + "Requested {}, but received {}", + requested_hash_info->Hash(), + rehashed_info.Hash()); + return std::nullopt; } - Logger::Log(LogLevel::Warning, - "Requested {}, but received {}", - ArtifactDigest{blob.digest}.hash(), - rehashed_digest.hash()); - return false; + return rehashed_info; } namespace { diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp index f009d968..2acd10c7 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp @@ -29,6 +29,7 @@ #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/crypto/hash_info.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp" #include "src/buildtool/execution_api/common/artifact_blob_container.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" @@ -92,7 +93,8 @@ class BazelNetworkReader final { std::vector<bazel_re::Digest> const& blobs) const noexcept -> std::vector<ArtifactBlob>; - [[nodiscard]] auto Validate(BazelBlob const& blob) const noexcept -> bool; + [[nodiscard]] auto Validate(BazelBlob const& blob) const noexcept + -> std::optional<HashInfo>; }; class BazelNetworkReader::IncrementalReader final { |