diff options
Diffstat (limited to 'src/buildtool/execution_api')
13 files changed, 488 insertions, 193 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index 210cbb3a..214e87be 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -6,6 +6,7 @@ , "execution_api.hpp" , "execution_action.hpp" , "execution_response.hpp" + , "tree_reader.hpp" ] , "srcs": ["execution_api.cpp"] , "deps": @@ -20,6 +21,7 @@ , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "gsl"] , ["src/utils/cpp", "hex_string"] + , ["src/buildtool/file_system", "git_repo"] ] , "stage": ["src", "buildtool", "execution_api", "common"] } diff --git a/src/buildtool/execution_api/common/tree_reader.hpp b/src/buildtool/execution_api/common/tree_reader.hpp new file mode 100644 index 00000000..0a6057fe --- /dev/null +++ b/src/buildtool/execution_api/common/tree_reader.hpp @@ -0,0 +1,160 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_HPP + +#include <filesystem> +#include <functional> +#include <optional> +#include <utility> +#include <vector> + +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/file_system/object_type.hpp" + +struct ReadTreeResult final { + std::vector<std::filesystem::path> paths; + std::vector<Artifact::ObjectInfo> infos; +}; + +template <typename TImpl> +class TreeReader final { + public: + template <typename... Args> + explicit TreeReader(Args&&... args) noexcept + : impl_(std::forward<Args>(args)...) {} + + /// \brief Reads the flat content of a tree and returns object infos of all + /// its direct entries (trees and blobs). + /// \param tree_digest Digest of the tree. + /// \param parent Local parent path. + /// \returns A struct containing filesystem paths and object infos. + [[nodiscard]] auto ReadDirectTreeEntries( + ArtifactDigest const& digest, + std::filesystem::path const& parent) const noexcept + -> std::optional<ReadTreeResult> { + ReadTreeResult result; + + BazelMsgFactory::InfoStoreFunc store_info = + [&result, &parent](std::filesystem::path const& path, + Artifact::ObjectInfo const& info) { + result.paths.emplace_back(parent / path); + result.infos.emplace_back(info); + return true; + }; + + if (Compatibility::IsCompatible()) { + auto tree = impl_.ReadDirectory(digest); + if (tree and not BazelMsgFactory::ReadObjectInfosFromDirectory( + *tree, store_info)) { + return std::nullopt; + } + } + else { + auto tree = impl_.ReadGitTree(digest); + if (tree and not BazelMsgFactory::ReadObjectInfosFromGitTree( + *tree, store_info)) { + return std::nullopt; + } + } + return result; + } + + /// \brief Traverses a tree recursively and retrieves object infos of all + /// found blobs (leafs). Tree objects are by default not added to the result + /// list, but converted to a path name. + /// \param tree_digest Digest of the tree. + /// \param parent Local parent path. + /// \param include_trees Include leaf tree objects (empty trees). + /// \returns A struct containing filesystem paths and object infos. + [[nodiscard]] auto RecursivelyReadTreeLeafs( + ArtifactDigest const& digest, + std::filesystem::path const& parent, + bool include_trees = false) const noexcept + -> std::optional<ReadTreeResult> { + ReadTreeResult result; + + auto store = [&result](std::filesystem::path const& path, + Artifact::ObjectInfo const& info) { + result.paths.emplace_back(path); + result.infos.emplace_back(info); + return true; + }; + + try { + if (ReadObjectInfosRecursively( + store, parent, digest, include_trees)) { + return result; + } + } catch (...) { + // fallthrough + } + return std::nullopt; + } + + private: + TImpl impl_; + + [[nodiscard]] static inline auto IsDirectoryEmpty( + bazel_re::Directory const& dir) noexcept -> bool { + return dir.files().empty() and dir.directories().empty() and + dir.symlinks().empty(); + } + + [[nodiscard]] auto ReadObjectInfosRecursively( + BazelMsgFactory::InfoStoreFunc const& store, + std::filesystem::path const& parent, + ArtifactDigest const& digest, + bool const include_trees) const -> bool { + BazelMsgFactory::InfoStoreFunc internal_store = + [this, &store, &parent, include_trees]( + std::filesystem::path const& path, + Artifact::ObjectInfo const& info) -> bool { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively( + store, parent / path, info.digest, include_trees) + : store(parent / path, info); + }; + + if (Compatibility::IsCompatible()) { + if (auto tree = impl_.ReadDirectory(digest)) { + if (include_trees and IsDirectoryEmpty(*tree)) { + if (not store(parent, {digest, ObjectType::Tree})) { + return false; + } + } + return BazelMsgFactory::ReadObjectInfosFromDirectory( + *tree, internal_store); + } + } + else { + if (auto tree = impl_.ReadGitTree(digest)) { + if (include_trees and tree->empty()) { + if (not store(parent, {digest, ObjectType::Tree})) { + return false; + } + } + return BazelMsgFactory::ReadObjectInfosFromGitTree( + *tree, internal_store); + } + } + return false; + } +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_TREE_READER_HPP diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS index b1b7c08c..b192b402 100644 --- a/src/buildtool/execution_api/local/TARGETS +++ b/src/buildtool/execution_api/local/TARGETS @@ -19,8 +19,13 @@ , "local": { "type": ["@", "rules", "CC", "library"] , "name": ["local"] - , "hdrs": ["local_api.hpp", "local_action.hpp", "local_response.hpp"] - , "srcs": ["local_action.cpp"] + , "hdrs": + [ "local_api.hpp" + , "local_action.hpp" + , "local_response.hpp" + , "local_cas_reader.hpp" + ] + , "srcs": ["local_action.cpp", "local_cas_reader.cpp"] , "deps": [ ["@", "fmt", "", "fmt"] , ["@", "gsl", "", "gsl"] @@ -38,6 +43,7 @@ , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/execution_api/execution_service", "cas_utils"] + , ["src/buildtool/file_system", "git_repo"] ] , "stage": ["src", "buildtool", "execution_api", "local"] , "private-deps": @@ -48,6 +54,8 @@ , ["src/buildtool/common", "bazel_types"] , ["src/buildtool/file_system", "file_system_manager"] , ["src/buildtool/execution_api/utils", "outputscheck"] + , ["src/buildtool/crypto", "hash_function"] + , ["src/utils/cpp", "path"] ] } } diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp index 52b513f0..cb3e3563 100644 --- a/src/buildtool/execution_api/local/local_action.cpp +++ b/src/buildtool/execution_api/local/local_action.cpp @@ -21,7 +21,9 @@ #include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/compatibility/native_support.hpp" +#include "src/buildtool/execution_api/common/tree_reader.hpp" #include "src/buildtool/execution_api/local/config.hpp" +#include "src/buildtool/execution_api/local/local_cas_reader.hpp" #include "src/buildtool/execution_api/local/local_response.hpp" #include "src/buildtool/execution_api/utils/outputscheck.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" @@ -216,14 +218,14 @@ auto LocalAction::StageInputs( if (FileSystemManager::IsRelativePath(exec_path)) { return false; } - - auto infos = storage_->CAS().RecursivelyReadTreeLeafs( + auto reader = TreeReader<LocalCasReader>{storage_->CAS()}; + auto result = reader.RecursivelyReadTreeLeafs( root_digest_, exec_path, /*include_trees=*/true); - if (not infos) { + if (not result) { return false; } - for (std::size_t i{}; i < infos->first.size(); ++i) { - if (not StageInput(infos->first.at(i), infos->second.at(i))) { + for (std::size_t i{}; i < result->paths.size(); ++i) { + if (not StageInput(result->paths.at(i), result->infos.at(i))) { return false; } } diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp index 53cee482..c81b75c1 100644 --- a/src/buildtool/execution_api/local/local_api.hpp +++ b/src/buildtool/execution_api/local/local_api.hpp @@ -37,9 +37,11 @@ #include "src/buildtool/execution_api/bazel_msg/blob_tree.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" +#include "src/buildtool/execution_api/common/tree_reader.hpp" #include "src/buildtool/execution_api/execution_service/cas_utils.hpp" #include "src/buildtool/execution_api/git/git_api.hpp" #include "src/buildtool/execution_api/local/local_action.hpp" +#include "src/buildtool/execution_api/local/local_cas_reader.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -85,11 +87,12 @@ class LocalApi final : public IExecutionApi { auto const& info = artifacts_info[i]; if (IsTreeObject(info.type)) { // read object infos from sub tree and call retrieve recursively - auto const infos = storage_->CAS().RecursivelyReadTreeLeafs( + auto reader = TreeReader<LocalCasReader>{storage_->CAS()}; + auto const result = reader.RecursivelyReadTreeLeafs( info.digest, output_paths[i]); - if (not infos) { + if (not result) { if (Compatibility::IsCompatible()) { - // infos not available, and in compatible mode cannot + // result not available, and in compatible mode cannot // fall back to git return false; } @@ -99,7 +102,7 @@ class LocalApi final : public IExecutionApi { return false; } } - else if (not RetrieveToPaths(infos->second, infos->first)) { + else if (not RetrieveToPaths(result->infos, result->paths)) { return false; } } @@ -192,9 +195,10 @@ class LocalApi final : public IExecutionApi { auto const& info = missing_artifacts_info->back_map[dgst]; // Recursively process trees. if (IsTreeObject(info.type)) { - auto const& infos = storage_->CAS().ReadDirectTreeEntries( + auto reader = TreeReader<LocalCasReader>{storage_->CAS()}; + auto const& result = reader.ReadDirectTreeEntries( info.digest, std::filesystem::path{}); - if (not infos or not RetrieveToCas(infos->second, api)) { + if (not result or not RetrieveToCas(result->infos, api)) { return false; } } diff --git a/src/buildtool/execution_api/local/local_cas_reader.cpp b/src/buildtool/execution_api/local/local_cas_reader.cpp new file mode 100644 index 00000000..411bc5d7 --- /dev/null +++ b/src/buildtool/execution_api/local/local_cas_reader.cpp @@ -0,0 +1,66 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/local/local_cas_reader.hpp" + +#include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/utils/cpp/path.hpp" + +auto LocalCasReader::ReadDirectory(ArtifactDigest const& digest) const noexcept + -> std::optional<bazel_re::Directory> { + if (auto const path = cas_.TreePath(digest)) { + if (auto const content = FileSystemManager::ReadFile(*path)) { + return BazelMsgFactory::MessageFromString<bazel_re::Directory>( + *content); + } + } + Logger::Log( + LogLevel::Error, "Directory {} not found in CAS", digest.hash()); + return std::nullopt; +} + +auto LocalCasReader::ReadGitTree(ArtifactDigest const& digest) const noexcept + -> std::optional<GitRepo::tree_entries_t> { + if (auto const path = cas_.TreePath(digest)) { + if (auto const content = FileSystemManager::ReadFile(*path)) { + auto check_symlinks = + [this](std::vector<bazel_re::Digest> const& ids) { + for (auto const& id : ids) { + auto link_path = cas_.BlobPath(id, + /*is_executable=*/false); + if (not link_path) { + return false; + } + // in the local CAS we store as files + auto content = FileSystemManager::ReadFile(*link_path); + if (not content or not PathIsNonUpwards(*content)) { + return false; + } + } + return true; + }; + return GitRepo::ReadTreeData( + *content, + HashFunction::ComputeTreeHash(*content).Bytes(), + check_symlinks, + /*is_hex_id=*/false); + } + } + Logger::Log(LogLevel::Debug, "Tree {} not found in CAS", digest.hash()); + return std::nullopt; +} diff --git a/src/buildtool/execution_api/local/local_cas_reader.hpp b/src/buildtool/execution_api/local/local_cas_reader.hpp new file mode 100644 index 00000000..69baf3a9 --- /dev/null +++ b/src/buildtool/execution_api/local/local_cas_reader.hpp @@ -0,0 +1,39 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_READER_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_READER_HPP + +#include <optional> + +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/file_system/git_repo.hpp" +#include "src/buildtool/storage/local_cas.hpp" + +class LocalCasReader final { + public: + explicit LocalCasReader(LocalCAS<true> const& cas) noexcept : cas_(cas) {} + + [[nodiscard]] auto ReadDirectory(ArtifactDigest const& digest) + const noexcept -> std::optional<bazel_re::Directory>; + + [[nodiscard]] auto ReadGitTree(ArtifactDigest const& digest) const noexcept + -> std::optional<GitRepo::tree_entries_t>; + + private: + LocalCAS<true> const& cas_; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_READER_HPP
\ No newline at end of file diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS index 88310cbb..c919b7bf 100644 --- a/src/buildtool/execution_api/remote/TARGETS +++ b/src/buildtool/execution_api/remote/TARGETS @@ -9,6 +9,7 @@ , "bazel/bazel_ac_client.hpp" , "bazel/bazel_cas_client.hpp" , "bazel/bazel_execution_client.hpp" + , "bazel/bazel_network_reader.hpp" ] , "srcs": [ "bazel/bazel_action.cpp" @@ -17,6 +18,7 @@ , "bazel/bazel_ac_client.cpp" , "bazel/bazel_cas_client.cpp" , "bazel/bazel_execution_client.cpp" + , "bazel/bazel_network_reader.cpp" ] , "deps": [ "config" @@ -32,6 +34,7 @@ , ["src/utils/cpp", "gsl"] , ["src/buildtool/common/remote", "client_common"] , ["src/buildtool/common/remote", "port"] + , ["src/buildtool/file_system", "git_repo"] ] , "proto": [ ["@", "bazel_remote_apis", "", "remote_execution_proto"] @@ -50,6 +53,8 @@ , ["@", "grpc", "", "grpc++"] , ["src/buildtool/common/remote", "retry"] , ["src/buildtool/execution_api/common", "message_limits"] + , ["src/buildtool/crypto", "hash_function"] + , ["src/utils/cpp", "path"] ] } , "bazel": diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp index 0e3a1ca2..47d024f9 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp @@ -31,11 +31,13 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/common_api.hpp" +#include "src/buildtool/execution_api/common/tree_reader.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_action.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp" +#include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_response.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/object_type.hpp" @@ -223,10 +225,15 @@ auto BazelApi::CreateAction( else { if (IsTreeObject(info.type)) { // read object infos from sub tree and call retrieve recursively - auto const infos = network_->RecursivelyReadTreeLeafs( - info.digest, output_paths[i], alternative.has_value()); - if (not infos or - not RetrieveToPaths(infos->second, infos->first)) { + auto request_remote_tree = alternative.has_value() + ? std::make_optional(info.digest) + : std::nullopt; + auto reader = TreeReader<BazelNetworkReader>{ + *network_, std::move(request_remote_tree)}; + auto const result = reader.RecursivelyReadTreeLeafs( + info.digest, output_paths[i]); + if (not result or + not RetrieveToPaths(result->infos, result->paths)) { return false; } } @@ -315,9 +322,10 @@ auto BazelApi::CreateAction( for (auto const& dgst : missing_artifacts_info->digests) { auto const& info = missing_artifacts_info->back_map[dgst]; if (IsTreeObject(info.type)) { - auto const infos = network_->ReadDirectTreeEntries( + auto reader = TreeReader<BazelNetworkReader>{*network_}; + auto const result = reader.ReadDirectTreeEntries( info.digest, std::filesystem::path{}); - if (not infos or not RetrieveToCas(infos->second, api)) { + if (not result or not RetrieveToCas(result->infos, api)) { return false; } } @@ -363,11 +371,12 @@ auto BazelApi::CreateAction( for (auto const& dgst : missing_artifacts_info->digests) { auto const& info = missing_artifacts_info->back_map[dgst]; if (IsTreeObject(info.type)) { - auto const infos = network_->ReadDirectTreeEntries( + auto reader = TreeReader<BazelNetworkReader>{*network_}; + auto const result = reader.ReadDirectTreeEntries( info.digest, std::filesystem::path{}); - if (not infos or + if (not result or not ParallelRetrieveToCas( - infos->second, api, jobs, use_blob_splitting)) { + result->infos, api, jobs, use_blob_splitting)) { return false; } } diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index 325abd02..485e65dc 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -307,141 +307,12 @@ auto BazelNetwork::GetCachedActionResult( instance_name_, action, false, false, output_files); } -auto BazelNetwork::RecursivelyReadTreeLeafs( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent, - bool request_remote_tree) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>> { - std::optional<DirectoryMap> dir_map{std::nullopt}; - if (Compatibility::IsCompatible() and request_remote_tree) { - // Query full tree from remote CAS. Note that this is currently not - // supported by Buildbarn revision c3c06bbe2a. - auto dirs = - cas_->GetTree(instance_name_, tree_digest, kMaxBatchTransferSize); - - // Convert to Directory map - dir_map = DirectoryMap{}; - dir_map->reserve(dirs.size()); - for (auto& dir : dirs) { - try { - dir_map->emplace(ArtifactDigest::Create<ObjectType::File>( - dir.SerializeAsString()), - std::move(dir)); - } catch (...) { - return std::nullopt; - } - } - } - - std::vector<std::filesystem::path> paths{}; - std::vector<Artifact::ObjectInfo> infos{}; - - auto store_info = [&paths, &infos](auto path, auto info) { - paths.emplace_back(path); - infos.emplace_back(info); - return true; - }; - - if (ReadObjectInfosRecursively(dir_map, store_info, parent, tree_digest)) { - return std::make_pair(std::move(paths), std::move(infos)); - } - - return std::nullopt; -} - -auto BazelNetwork::ReadDirectTreeEntries( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>> { - std::vector<std::filesystem::path> paths{}; - std::vector<Artifact::ObjectInfo> infos{}; - - auto store_info = [&paths, &infos](auto path, auto info) { - paths.emplace_back(path); - infos.emplace_back(info); - return true; - }; - - if (Compatibility::IsCompatible()) { - // read from CAS - if (auto dir = ReadDirectory(this, tree_digest)) { - if (not BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, [&store_info, &parent](auto path, auto info) { - return store_info(parent / path, info); - })) { - return std::nullopt; - } - } - } - else { - if (auto entries = ReadGitTree(this, tree_digest)) { - if (not BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, [&store_info, &parent](auto path, auto info) { - return store_info(parent / path, info); - })) { - return std::nullopt; - } - } - } - - return std::make_pair(std::move(paths), std::move(infos)); -} - -// NOLINTNEXTLINE(misc-no-recursion) -auto BazelNetwork::ReadObjectInfosRecursively( - std::optional<DirectoryMap> const& dir_map, - BazelMsgFactory::InfoStoreFunc const& store_info, - std::filesystem::path const& parent, - bazel_re::Digest const& digest) const noexcept -> bool { - if (Compatibility::IsCompatible()) { - // read from in-memory Directory map - if (dir_map) { - if (dir_map->contains(digest)) { - return BazelMsgFactory::ReadObjectInfosFromDirectory( - dir_map->at(digest), - [this, &dir_map, &store_info, &parent](auto path, - auto info) { - return IsTreeObject(info.type) - ? ReadObjectInfosRecursively(dir_map, - store_info, - parent / path, - info.digest) - : store_info(parent / path, info); - }); - } - } - - // fallback read from CAS - if (auto dir = ReadDirectory(this, digest)) { - return BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, - [this, &dir_map, &store_info, &parent](auto path, auto info) { - return IsTreeObject(info.type) - ? ReadObjectInfosRecursively(dir_map, - store_info, - parent / path, - info.digest) - : store_info(parent / path, info); - }); - } - } - else { - if (auto entries = ReadGitTree(this, digest)) { - return BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, - [this, &dir_map, &store_info, &parent](auto path, auto info) { - return IsTreeObject(info.type) - ? ReadObjectInfosRecursively(dir_map, - store_info, - parent / path, - info.digest) - : store_info(parent / path, info); - }); - } +auto BazelNetwork::QueryFullTree(bazel_re::Digest const& digest) const noexcept + -> std::optional<std::vector<bazel_re::Directory>> { + if (not Compatibility::IsCompatible()) { + return std::nullopt; } - return false; + return cas_->GetTree(instance_name_, digest, kMaxBatchTransferSize); } auto BazelNetwork::DumpToStream(Artifact::ObjectInfo const& info, diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp index 7101c926..6ee44921 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp @@ -19,7 +19,6 @@ #include <memory> #include <optional> #include <string> -#include <unordered_map> #include <utility> #include <vector> @@ -111,41 +110,16 @@ class BazelNetwork { std::vector<std::string> const& output_files) const noexcept -> std::optional<bazel_re::ActionResult>; - /// \brief Traverses a tree recursively and retrieves object infos of all - /// found blobs. Tree objects are not added to the result list, but - /// converted to a path name. - /// \param tree_digest Digest of the tree. - /// \param parent Local parent path. - /// \param request_remote_tree Query full tree from remote CAS. - /// \returns Pair of vectors, first containing filesystem paths, second - /// containing object infos. - [[nodiscard]] auto RecursivelyReadTreeLeafs( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent, - bool request_remote_tree = false) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>>; - - /// \brief Reads the flat content of a tree and returns object infos of all - /// its direct entries (trees and blobs). - /// \param tree_digest Digest of the tree. - /// \param parent Local parent path. - /// \returns Pair of vectors, first containing filesystem paths, second - /// containing object infos. - [[nodiscard]] auto ReadDirectTreeEntries( - bazel_re::Digest const& tree_digest, - std::filesystem::path const& parent) const noexcept - -> std::optional<std::pair<std::vector<std::filesystem::path>, - std::vector<Artifact::ObjectInfo>>>; - [[nodiscard]] auto DumpToStream(Artifact::ObjectInfo const& info, gsl::not_null<FILE*> const& stream, bool raw_tree) const noexcept -> bool; - private: - using DirectoryMap = - std::unordered_map<bazel_re::Digest, bazel_re::Directory>; + /// \brief Query full tree from remote CAS. Note that this is currently not + // supported by Buildbarn revision c3c06bbe2a. + [[nodiscard]] auto QueryFullTree(bazel_re::Digest const& digest) + const noexcept -> std::optional<std::vector<bazel_re::Directory>>; + private: std::string const instance_name_{}; ExecutionConfiguration exec_config_{}; std::unique_ptr<BazelCasClient> cas_{}; @@ -155,12 +129,6 @@ class BazelNetwork { template <class T_Iter> [[nodiscard]] auto DoUploadBlobs(T_Iter const& first, T_Iter const& last) noexcept -> bool; - - [[nodiscard]] auto ReadObjectInfosRecursively( - std::optional<DirectoryMap> const& dir_map, - BazelMsgFactory::InfoStoreFunc const& store_info, - std::filesystem::path const& parent, - bazel_re::Digest const& digest) const noexcept -> bool; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_NETWORK_HPP diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp new file mode 100644 index 00000000..ee0fbc78 --- /dev/null +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.cpp @@ -0,0 +1,110 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp" + +#include <algorithm> + +#include "src/buildtool/crypto/hash_function.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp" +#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" +#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" +#include "src/utils/cpp/path.hpp" + +BazelNetworkReader::BazelNetworkReader( + BazelNetwork const& network, + std::optional<ArtifactDigest> request_remote_tree) noexcept + : network_(network) { + if (Compatibility::IsCompatible() and request_remote_tree) { + auto full_tree = network_.QueryFullTree(*request_remote_tree); + if (full_tree) { + auxiliary_map_ = MakeAuxiliaryMap(std::move(*full_tree)); + } + } +} + +auto BazelNetworkReader::ReadDirectory(ArtifactDigest const& digest) + const noexcept -> std::optional<bazel_re::Directory> { + try { + if (auxiliary_map_ and auxiliary_map_->contains(digest)) { + return auxiliary_map_->at(digest); + } + } catch (...) { + // fallthrough + } + + auto blobs = network_.ReadBlobs({digest}).Next(); + if (blobs.size() == 1) { + return BazelMsgFactory::MessageFromString<bazel_re::Directory>( + blobs.at(0).data); + } + Logger::Log( + LogLevel::Debug, "Directory {} not found in CAS", digest.hash()); + return std::nullopt; +} + +auto BazelNetworkReader::ReadGitTree(ArtifactDigest const& digest) + const noexcept -> std::optional<GitRepo::tree_entries_t> { + auto blobs = network_.ReadBlobs({digest}).Next(); + if (blobs.size() == 1) { + auto const& content = blobs.at(0).data; + auto check_symlinks = [this](std::vector<bazel_re::Digest> const& ids) { + auto size = ids.size(); + auto reader = network_.ReadBlobs(ids); + auto blobs = reader.Next(); + std::size_t count{}; + while (not blobs.empty()) { + if (count + blobs.size() > size) { + Logger::Log(LogLevel::Debug, + "received more blobs than requested."); + return false; + } + for (auto const& blob : blobs) { + if (not PathIsNonUpwards(blob.data)) { + return false; + } + } + count += blobs.size(); + blobs = reader.Next(); + } + return true; + }; + return GitRepo::ReadTreeData( + content, + HashFunction::ComputeTreeHash(content).Bytes(), + check_symlinks, + /*is_hex_id=*/false); + } + Logger::Log(LogLevel::Debug, "Tree {} not found in CAS", digest.hash()); + return std::nullopt; +} + +auto BazelNetworkReader::MakeAuxiliaryMap( + std::vector<bazel_re::Directory>&& full_tree) noexcept + -> std::optional<DirectoryMap> { + DirectoryMap result; + result.reserve(full_tree.size()); + for (auto& dir : full_tree) { + try { + result.emplace(ArtifactDigest::Create<ObjectType::File>( + dir.SerializeAsString()), + std::move(dir)); + } catch (...) { + return std::nullopt; + } + } + return result; +} diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp new file mode 100644 index 00000000..8d46c9f8 --- /dev/null +++ b/src/buildtool/execution_api/remote/bazel/bazel_network_reader.hpp @@ -0,0 +1,51 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP +#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP + +#include <optional> +#include <unordered_map> + +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp" +#include "src/buildtool/file_system/git_repo.hpp" + +class BazelNetworkReader final { + public: + explicit BazelNetworkReader( + BazelNetwork const& network, + std::optional<ArtifactDigest> request_remote_tree = + std::nullopt) noexcept; + + [[nodiscard]] auto ReadDirectory(ArtifactDigest const& digest) + const noexcept -> std::optional<bazel_re::Directory>; + + [[nodiscard]] auto ReadGitTree(ArtifactDigest const& digest) const noexcept + -> std::optional<GitRepo::tree_entries_t>; + + private: + using DirectoryMap = + std::unordered_map<bazel_re::Digest, bazel_re::Directory>; + + BazelNetwork const& network_; + std::optional<DirectoryMap> auxiliary_map_; + + [[nodiscard]] static auto MakeAuxiliaryMap( + std::vector<bazel_re::Directory>&& full_tree) noexcept + -> std::optional<DirectoryMap>; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_TREE_READER_HPP
\ No newline at end of file |