diff options
Diffstat (limited to 'src')
6 files changed, 43 insertions, 313 deletions
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS index e6043889..47ec0fc0 100644 --- a/src/buildtool/execution_api/common/TARGETS +++ b/src/buildtool/execution_api/common/TARGETS @@ -6,7 +6,6 @@ , "execution_api.hpp" , "execution_action.hpp" , "execution_response.hpp" - , "local_tree_map.hpp" ] , "deps": [ ["@", "gsl-lite", "", "gsl-lite"] diff --git a/src/buildtool/execution_api/common/local_tree_map.hpp b/src/buildtool/execution_api/common/local_tree_map.hpp deleted file mode 100644 index 915c7a71..00000000 --- a/src/buildtool/execution_api/common/local_tree_map.hpp +++ /dev/null @@ -1,146 +0,0 @@ -#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP -#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP - -#include <filesystem> -#include <shared_mutex> -#include <string> -#include <unordered_map> -#include <unordered_set> - -#include "gsl-lite/gsl-lite.hpp" -#include "src/buildtool/common/artifact.hpp" -#include "src/buildtool/logging/logger.hpp" - -/// \brief Maps `bazel_re::Digest` to `LocalTree`. -/// Digest may refer to `bazel_re::Directory` or Git tree object, depending on -/// mode being compatible or native, respectively. -class LocalTreeMap { - /// \brief Thread-safe pool of unique object infos. - class ObjectInfoPool { - public: - /// Get pointer to stored info, or a add new one and return its pointer. - [[nodiscard]] auto GetOrAdd(Artifact::ObjectInfo const& info) - -> Artifact::ObjectInfo const* { - { // get - std::shared_lock lock{mutex_}; - auto it = infos_.find(info); - if (it != infos_.end()) { - return &(*it); - } - } - { // or add - std::unique_lock lock{mutex_}; - return &(*infos_.emplace(info).first); - } - } - - private: - std::unordered_set<Artifact::ObjectInfo> infos_; - mutable std::shared_mutex mutex_; - }; - - public: - /// \brief Maps blob paths to object infos. - /// Stores a flat list of blobs, without any trees. Subtrees are represented - /// by joining the tree segments for the blob's path. - class LocalTree { - friend class LocalTreeMap; - - public: - /// \brief Add a new path and info pair to the tree. - /// Path must not be absolute, empty, or contain dot-segments. - /// Object MUST NOT be a tree object. - /// \param path The location to add the object info. - /// \param info The object info to add. - /// \returns true if successfully inserted or info existed before. - [[nodiscard]] auto AddInfo(std::filesystem::path const& path, - Artifact::ObjectInfo const& info) noexcept - -> bool { - gsl_Expects(not IsTreeObject(info.type)); - auto norm_path = path.lexically_normal(); - if (norm_path.is_absolute() or norm_path.empty() or - *norm_path.begin() == "..") { - Logger::Log(LogLevel::Error, - "cannot add malformed path to local tree: {}", - path.string()); - return false; - } - try { - if (entries_.contains(norm_path.string())) { - return true; - } - if (auto const* info_ptr = infos_->GetOrAdd(info)) { - entries_.emplace(norm_path.string(), info_ptr); - return true; - } - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "adding object info to tree failed with:\n{}", - ex.what()); - } - return false; - } - - [[nodiscard]] auto size() const noexcept { return entries_.size(); } - [[nodiscard]] auto begin() const noexcept { return entries_.begin(); } - [[nodiscard]] auto end() const noexcept { return entries_.end(); } - - private: - gsl::not_null<ObjectInfoPool*> infos_; - std::unordered_map<std::string, - gsl::not_null<Artifact::ObjectInfo const*>> - entries_{}; - - explicit LocalTree(gsl::not_null<ObjectInfoPool*> infos) noexcept - : infos_{std::move(infos)} {} - }; - - /// \brief Create a new `LocalTree` object. - [[nodiscard]] auto CreateTree() noexcept -> LocalTree { - return LocalTree{&infos_}; - } - - /// \brief Get pointer to existing `LocalTree` object. - /// \param root_digest The root digest of the tree to lookup. - /// \returns nullptr if no tree was found for given root digest. - [[nodiscard]] auto GetTree(bazel_re::Digest const& root_digest) - const noexcept -> LocalTree const* { - std::shared_lock lock{mutex_}; - auto it = trees_.find(root_digest); - return (it != trees_.end()) ? &(it->second) : nullptr; - } - - /// \brief Checks if entry for root digest exists. - [[nodiscard]] auto HasTree( - bazel_re::Digest const& root_digest) const noexcept -> bool { - return GetTree(root_digest) != nullptr; - } - - /// \brief Add new `LocalTree` for given root digest. Does not overwrite if - /// a tree for the given root digest already exists. - /// \param root_digest The root digest to add the new tree for. - /// \param tree The new tree to add. - /// \returns true if the tree was successfully added or existed before. - [[nodiscard]] auto AddTree(bazel_re::Digest const& root_digest, - LocalTree&& tree) noexcept -> bool { - if (not HasTree(root_digest)) { - try { - std::unique_lock lock{mutex_}; - trees_.emplace(root_digest, std::move(tree)); - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "adding local tree to tree map failed with:\n{}", - ex.what()); - return false; - } - } - return true; - } - - private: - ObjectInfoPool infos_; // pool to store each solid object info exactly once - std::unordered_map<bazel_re::Digest, LocalTree> trees_; - mutable std::shared_mutex mutex_; -}; - -#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP diff --git a/src/buildtool/execution_api/local/local_storage.cpp b/src/buildtool/execution_api/local/local_storage.cpp index 483f664f..b8e8401c 100644 --- a/src/buildtool/execution_api/local/local_storage.cpp +++ b/src/buildtool/execution_api/local/local_storage.cpp @@ -113,78 +113,27 @@ auto LocalStorage::ReadObjectInfosRecursively( BazelMsgFactory::InfoStoreFunc const& store_info, std::filesystem::path const& parent, bazel_re::Digest const& digest) const noexcept -> bool { - // read from in-memory tree map - auto const* tree = tree_map_.GetTree(digest); - if (tree != nullptr) { - return std::all_of( - tree->begin(), - tree->end(), - [&store_info, &parent](auto const& entry) { - try { - // LocalTree (from tree_map_) is flat, no recursion needed - auto const& [path, info] = entry; - return store_info(parent / path, *info); - } catch (...) { // satisfy clang-tidy, store_info() could - return false; - } - }); - } - Logger::Log( - LogLevel::Debug, "tree {} not found in tree map", digest.hash()); - - // fallback read from CAS and cache it in in-memory tree map + // read from CAS if (Compatibility::IsCompatible()) { if (auto dir = ReadDirectory(this, digest)) { - auto tree = tree_map_.CreateTree(); return BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, - [this, &store_info, &parent, &tree](auto path, - auto info) { - if (IsTreeObject(info.type)) { - // LocalTree (from tree_map_) is flat, so - // recursively traverse subtrees and add blobs. - auto tree_store_info = - [&store_info, &tree, &parent](auto path, - auto info) { - auto tree_path = - path.lexically_relative(parent); - return tree.AddInfo(tree_path, info) and - store_info(path, info); - }; - return ReadObjectInfosRecursively( - tree_store_info, parent / path, info.digest); - } - return tree.AddInfo(path, info) and - store_info(parent / path, info); - }) and - tree_map_.AddTree(digest, std::move(tree)); + *dir, [this, &store_info, &parent](auto path, auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively( + store_info, parent / path, info.digest) + : store_info(parent / path, info); + }); } } else { if (auto entries = ReadGitTree(this, digest)) { - auto tree = tree_map_.CreateTree(); return BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, - [this, &store_info, &parent, &tree](auto path, - auto info) { - if (IsTreeObject(info.type)) { - // LocalTree (from tree_map_) is flat, so - // recursively traverse subtrees and add blobs. - auto tree_store_info = - [&store_info, &tree, &parent](auto path, - auto info) { - auto tree_path = - path.lexically_relative(parent); - return tree.AddInfo(tree_path, info) and - store_info(path, info); - }; - return ReadObjectInfosRecursively( - tree_store_info, parent / path, info.digest); - } - return tree.AddInfo(path, info) and - store_info(parent / path, info); - }) and - tree_map_.AddTree(digest, std::move(tree)); + *entries, [this, &store_info, &parent](auto path, auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively( + store_info, parent / path, info.digest) + : store_info(parent / path, info); + }); } } return false; diff --git a/src/buildtool/execution_api/local/local_storage.hpp b/src/buildtool/execution_api/local/local_storage.hpp index f9594218..5ad7f142 100644 --- a/src/buildtool/execution_api/local/local_storage.hpp +++ b/src/buildtool/execution_api/local/local_storage.hpp @@ -6,7 +6,6 @@ #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp" #include "src/buildtool/execution_api/common/execution_common.hpp" -#include "src/buildtool/execution_api/common/local_tree_map.hpp" #include "src/buildtool/execution_api/local/local_ac.hpp" #include "src/buildtool/execution_api/local/local_cas.hpp" @@ -85,7 +84,6 @@ class LocalStorage { LocalCAS<ObjectType::Executable> cas_exec_{}; LocalCAS<ObjectType::Tree> cas_tree_{}; LocalAC ac_{&cas_file_}; - mutable LocalTreeMap tree_map_; /// \brief Try to sync blob between file CAS and executable CAS. /// \param digest Blob digest. diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp index fa09b16c..e4205d25 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp @@ -285,118 +285,50 @@ auto BazelNetwork::ReadObjectInfosRecursively( BazelMsgFactory::InfoStoreFunc const& store_info, std::filesystem::path const& parent, bazel_re::Digest const& digest) const noexcept -> bool { - // read from in-memory tree map - auto const* tree = tree_map_.GetTree(digest); - if (tree != nullptr) { - return std::all_of( - tree->begin(), - tree->end(), - [&store_info, &parent](auto const& entry) { - try { - // LocalTree (from tree_map_) is flat, no recursion needed - auto const& [path, info] = entry; - return store_info(parent / path, *info); - } catch (...) { // satisfy clang-tidy, store_info() could throw - return false; - } - }); - } - Logger::Log( - LogLevel::Debug, "tree {} not found in tree map", digest.hash()); - if (Compatibility::IsCompatible()) { - // read from in-memory Directory map and cache it in in-memory tree map + // read from in-memory Directory map if (dir_map) { if (dir_map->contains(digest)) { - auto tree = tree_map_.CreateTree(); return BazelMsgFactory::ReadObjectInfosFromDirectory( - dir_map->at(digest), - [this, &dir_map, &store_info, &parent, &tree]( - auto path, auto info) { - if (IsTreeObject(info.type)) { - // LocalTree (from tree_map_) is flat, so - // recursively traverse subtrees and add - // blobs. - auto tree_store_info = [&store_info, - &tree, - &parent](auto path, - auto info) { - auto tree_path = - path.lexically_relative(parent); - return tree.AddInfo(tree_path, info) and - store_info(path, info); - }; - return ReadObjectInfosRecursively( - dir_map, - tree_store_info, - parent / path, - info.digest); - } - return tree.AddInfo(path, info) and - store_info(parent / path, info); - }) and - tree_map_.AddTree(digest, std::move(tree)); + dir_map->at(digest), + [this, &dir_map, &store_info, &parent](auto path, + auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively(dir_map, + store_info, + parent / path, + info.digest) + : store_info(parent / path, info); + }); } } - // fallback read from CAS and cache it in in-memory tree map + // fallback read from CAS if (auto dir = ReadDirectory(this, digest)) { - auto tree = tree_map_.CreateTree(); return BazelMsgFactory::ReadObjectInfosFromDirectory( - *dir, - [this, &dir_map, &store_info, &parent, &tree]( - auto path, auto info) { - if (IsTreeObject(info.type)) { - // LocalTree (from tree_map_) is flat, so - // recursively traverse subtrees and add blobs. - auto tree_store_info = - [&store_info, &tree, &parent](auto path, - auto info) { - auto tree_path = - path.lexically_relative(parent); - return tree.AddInfo(tree_path, info) and - store_info(path, info); - }; - return ReadObjectInfosRecursively( - dir_map, - tree_store_info, - parent / path, - info.digest); - } - return tree.AddInfo(path, info) and - store_info(parent / path, info); - }) and - tree_map_.AddTree(digest, std::move(tree)); + *dir, + [this, &dir_map, &store_info, &parent](auto path, auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively(dir_map, + store_info, + parent / path, + info.digest) + : store_info(parent / path, info); + }); } } else { if (auto entries = ReadGitTree(this, digest)) { - auto tree = tree_map_.CreateTree(); return BazelMsgFactory::ReadObjectInfosFromGitTree( - *entries, - [this, &dir_map, &store_info, &parent, &tree]( - auto path, auto info) { - if (IsTreeObject(info.type)) { - // LocalTree (from tree_map_) is flat, so - // recursively traverse subtrees and add blobs. - auto tree_store_info = - [&store_info, &tree, &parent](auto path, - auto info) { - auto tree_path = - path.lexically_relative(parent); - return tree.AddInfo(tree_path, info) and - store_info(path, info); - }; - return ReadObjectInfosRecursively( - dir_map, - tree_store_info, - parent / path, - info.digest); - } - return tree.AddInfo(path, info) and - store_info(parent / path, info); - }) and - tree_map_.AddTree(digest, std::move(tree)); + *entries, + [this, &dir_map, &store_info, &parent](auto path, auto info) { + return IsTreeObject(info.type) + ? ReadObjectInfosRecursively(dir_map, + store_info, + parent / path, + info.digest) + : store_info(parent / path, info); + }); } } return false; diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp index 6e9d9c85..a323e00b 100644 --- a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp +++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp @@ -9,7 +9,6 @@ #include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp" #include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp" #include "src/buildtool/execution_api/common/execution_api.hpp" -#include "src/buildtool/execution_api/common/local_tree_map.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp" #include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp" @@ -104,7 +103,6 @@ class BazelNetwork { std::unique_ptr<BazelCasClient> cas_{}; std::unique_ptr<BazelAcClient> ac_{}; std::unique_ptr<BazelExecutionClient> exec_{}; - mutable LocalTreeMap tree_map_{}; template <class T_Iter> [[nodiscard]] auto DoUploadBlobs(T_Iter const& first, |