From 51f8b186803292f111011d04d5291deb374dc34c Mon Sep 17 00:00:00 2001 From: Oliver Reiche Date: Thu, 6 Oct 2022 12:18:25 +0200 Subject: LocalTreeMap: Prevent tree objects from being stored ... to align with the original idea of caching a flat list of blob objects, without the need to recursively traverse any trees. Consequently, we cannot create any map entry in places where we do not have all sub-tree entries at hand (e.g., LocalAPI, BazelAPI, BazelResponse). --- .../execution_api/local/local_storage.cpp | 96 ++++++++++++---------- 1 file changed, 52 insertions(+), 44 deletions(-) (limited to 'src/buildtool/execution_api/local/local_storage.cpp') diff --git a/src/buildtool/execution_api/local/local_storage.cpp b/src/buildtool/execution_api/local/local_storage.cpp index e6d9571c..483f664f 100644 --- a/src/buildtool/execution_api/local/local_storage.cpp +++ b/src/buildtool/execution_api/local/local_storage.cpp @@ -114,69 +114,77 @@ auto LocalStorage::ReadObjectInfosRecursively( std::filesystem::path const& parent, bazel_re::Digest const& digest) const noexcept -> bool { // read from in-memory tree map - if (tree_map_) { - auto const* tree = tree_map_->GetTree(digest); - if (tree != nullptr) { - return std::all_of( - tree->begin(), - tree->end(), - // NOLINTNEXTLINE(misc-no-recursion) - [this, &store_info, &parent](auto const& entry) { - try { - auto const& [path, info] = entry; - return IsTreeObject(info->type) - ? ReadObjectInfosRecursively(store_info, - parent / path, - info->digest) - : store_info(parent / path, *info); - } catch (...) { // satisfy clang-tidy, store_info() could - return false; - } - }); - } - Logger::Log( - LogLevel::Debug, "tree {} not found in tree map", digest.hash()); + auto const* tree = tree_map_.GetTree(digest); + if (tree != nullptr) { + return std::all_of( + tree->begin(), + tree->end(), + [&store_info, &parent](auto const& entry) { + try { + // LocalTree (from tree_map_) is flat, no recursion needed + auto const& [path, info] = entry; + return store_info(parent / path, *info); + } catch (...) { // satisfy clang-tidy, store_info() could + return false; + } + }); } + Logger::Log( + LogLevel::Debug, "tree {} not found in tree map", digest.hash()); // fallback read from CAS and cache it in in-memory tree map if (Compatibility::IsCompatible()) { if (auto dir = ReadDirectory(this, digest)) { - auto tree = tree_map_ ? std::make_optional(tree_map_->CreateTree()) - : std::nullopt; + auto tree = tree_map_.CreateTree(); return BazelMsgFactory::ReadObjectInfosFromDirectory( *dir, [this, &store_info, &parent, &tree](auto path, auto info) { - return (not tree or tree->AddInfo(path, info)) and - (IsTreeObject(info.type) - ? ReadObjectInfosRecursively( - store_info, - parent / path, - info.digest) - : store_info(parent / path, info)); + if (IsTreeObject(info.type)) { + // LocalTree (from tree_map_) is flat, so + // recursively traverse subtrees and add blobs. + auto tree_store_info = + [&store_info, &tree, &parent](auto path, + auto info) { + auto tree_path = + path.lexically_relative(parent); + return tree.AddInfo(tree_path, info) and + store_info(path, info); + }; + return ReadObjectInfosRecursively( + tree_store_info, parent / path, info.digest); + } + return tree.AddInfo(path, info) and + store_info(parent / path, info); }) and - (not tree_map_ or - tree_map_->AddTree(digest, std::move(*tree))); + tree_map_.AddTree(digest, std::move(tree)); } } else { if (auto entries = ReadGitTree(this, digest)) { - auto tree = tree_map_ ? std::make_optional(tree_map_->CreateTree()) - : std::nullopt; + auto tree = tree_map_.CreateTree(); return BazelMsgFactory::ReadObjectInfosFromGitTree( *entries, [this, &store_info, &parent, &tree](auto path, auto info) { - return (not tree or tree->AddInfo(path, info)) and - (IsTreeObject(info.type) - ? ReadObjectInfosRecursively( - store_info, - parent / path, - info.digest) - : store_info(parent / path, info)); + if (IsTreeObject(info.type)) { + // LocalTree (from tree_map_) is flat, so + // recursively traverse subtrees and add blobs. + auto tree_store_info = + [&store_info, &tree, &parent](auto path, + auto info) { + auto tree_path = + path.lexically_relative(parent); + return tree.AddInfo(tree_path, info) and + store_info(path, info); + }; + return ReadObjectInfosRecursively( + tree_store_info, parent / path, info.digest); + } + return tree.AddInfo(path, info) and + store_info(parent / path, info); }) and - (not tree_map_ or - tree_map_->AddTree(digest, std::move(*tree))); + tree_map_.AddTree(digest, std::move(tree)); } } return false; -- cgit v1.2.3