From 5ecbcbbcf71ca9fafaddf9a2353fe31a75fb33e4 Mon Sep 17 00:00:00 2001 From: Maksim Denisov Date: Thu, 29 Aug 2024 14:12:05 +0200 Subject: Replace bazel_re::Digest in Uplinker ...with ArtifactDigest. --- src/buildtool/storage/local_ac.hpp | 2 +- src/buildtool/storage/local_ac.tpp | 17 +++++----- src/buildtool/storage/local_cas.hpp | 12 +++---- src/buildtool/storage/local_cas.tpp | 63 ++++++++++++++++++------------------- src/buildtool/storage/uplinker.cpp | 9 +++--- src/buildtool/storage/uplinker.hpp | 14 +++------ 6 files changed, 55 insertions(+), 62 deletions(-) (limited to 'src') diff --git a/src/buildtool/storage/local_ac.hpp b/src/buildtool/storage/local_ac.hpp index 180f1c51..2e4f57a8 100644 --- a/src/buildtool/storage/local_ac.hpp +++ b/src/buildtool/storage/local_ac.hpp @@ -84,7 +84,7 @@ class LocalAC { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( LocalGenerationAC const& latest, - bazel_re::Digest const& action_id) const noexcept -> bool; + ArtifactDigest const& action_id) const noexcept -> bool; private: // The action cache stores the results of failed actions. For those to be diff --git a/src/buildtool/storage/local_ac.tpp b/src/buildtool/storage/local_ac.tpp index 3bf5ba31..2f67c798 100644 --- a/src/buildtool/storage/local_ac.tpp +++ b/src/buildtool/storage/local_ac.tpp @@ -55,16 +55,15 @@ template requires(kIsLocalGeneration) auto LocalAC::LocalUplinkEntry( LocalGenerationAC const& latest, - bazel_re::Digest const& action_id) const noexcept -> bool { + ArtifactDigest const& action_id) const noexcept -> bool { // Determine action cache key path in latest generation. - ArtifactDigest const a_digest{action_id}; if (FileSystemManager::IsFile( - latest.file_store_.GetPath(a_digest.hash()))) { + latest.file_store_.GetPath(action_id.hash()))) { return true; } // Read cache key - auto const cas_key = ReadActionKey(a_digest); + auto const cas_key = ReadActionKey(action_id); if (not cas_key) { return false; } @@ -77,8 +76,9 @@ auto LocalAC::LocalUplinkEntry( // Uplink result content for (auto const& file : result->output_files()) { + ArtifactDigest const a_digest{file.digest()}; if (not cas_.LocalUplinkBlob( - latest.cas_, file.digest(), file.is_executable())) { + latest.cas_, a_digest, file.is_executable())) { return false; } } @@ -99,7 +99,8 @@ auto LocalAC::LocalUplinkEntry( } } for (auto const& directory : result->output_directories()) { - if (not cas_.LocalUplinkTree(latest.cas_, directory.tree_digest())) { + ArtifactDigest const a_digest{directory.tree_digest()}; + if (not cas_.LocalUplinkTree(latest.cas_, a_digest)) { return false; } } @@ -111,9 +112,9 @@ auto LocalAC::LocalUplinkEntry( return false; } - auto const ac_entry_path = file_store_.GetPath(a_digest.hash()); + auto const ac_entry_path = file_store_.GetPath(action_id.hash()); // Uplink cache key - return latest.file_store_.AddFromFile(a_digest.hash(), + return latest.file_store_.AddFromFile(action_id.hash(), ac_entry_path, /*is_owner=*/true); } diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp index 6c653009..e7f5ce22 100644 --- a/src/buildtool/storage/local_cas.hpp +++ b/src/buildtool/storage/local_cas.hpp @@ -228,7 +228,7 @@ class LocalCAS { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBlob( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool is_executable, bool skip_sync = false, bool splice_result = false) const noexcept -> bool; @@ -251,7 +251,7 @@ class LocalCAS { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkTree( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool splice_result = false) const noexcept -> bool; /// \brief Uplink large entry from this generation to latest LocalCAS @@ -267,7 +267,7 @@ class LocalCAS { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkLargeObject( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest) const noexcept -> bool; + ArtifactDigest const& digest) const noexcept -> bool; private: ObjectCAS cas_file_; @@ -316,15 +316,15 @@ class LocalCAS { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkGitTree( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool splice_result = false) const noexcept -> bool; template requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBazelDirectory( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, - gsl::not_null*> const& seen, + ArtifactDigest const& digest, + gsl::not_null*> const& seen, bool splice_result = false) const noexcept -> bool; template diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp index cf1fd2d2..e2a77e87 100644 --- a/src/buildtool/storage/local_cas.tpp +++ b/src/buildtool/storage/local_cas.tpp @@ -19,6 +19,7 @@ #include // std::move #include "fmt/core.h" +#include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/local_cas.hpp" @@ -27,23 +28,22 @@ template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkBlob( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool is_executable, bool skip_sync, bool splice_result) const noexcept -> bool { - auto const a_digest = static_cast(digest); // Determine blob path in latest generation. - auto blob_path_latest = latest.BlobPathNoSync(a_digest, is_executable); + auto blob_path_latest = latest.BlobPathNoSync(digest, is_executable); if (blob_path_latest) { return true; } // Determine blob path of given generation. - auto blob_path = skip_sync ? BlobPathNoSync(a_digest, is_executable) - : BlobPath(a_digest, is_executable); + auto blob_path = skip_sync ? BlobPathNoSync(digest, is_executable) + : BlobPath(digest, is_executable); std::optional spliced; if (not blob_path) { - spliced = TrySplice(a_digest); + spliced = TrySplice(digest); blob_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not blob_path) { @@ -80,10 +80,10 @@ template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkTree( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool splice_result) const noexcept -> bool { if (Compatibility::IsCompatible()) { - std::unordered_set seen{}; + std::unordered_set seen{}; return LocalUplinkBazelDirectory(latest, digest, &seen, splice_result); } return LocalUplinkGitTree(latest, digest, splice_result); @@ -94,20 +94,19 @@ template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkGitTree( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool splice_result) const noexcept -> bool { // Determine tree path in latest generation. - auto const a_digest = static_cast(digest); - auto tree_path_latest = latest.cas_tree_.BlobPath(a_digest); + auto tree_path_latest = latest.cas_tree_.BlobPath(digest); if (tree_path_latest) { return true; } // Determine tree path of given generation. - auto tree_path = cas_tree_.BlobPath(a_digest); + auto tree_path = cas_tree_.BlobPath(digest); std::optional spliced; if (not tree_path) { - spliced = TrySplice(a_digest); + spliced = TrySplice(digest); tree_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not tree_path) { @@ -116,7 +115,6 @@ auto LocalCAS::LocalUplinkGitTree( // Determine tree entries. auto content = FileSystemManager::ReadFile(*tree_path); - auto id = NativeSupport::Unprefix(digest.hash()); auto check_symlinks = [this](std::vector const& ids) -> bool { for (auto const& id : ids) { @@ -139,7 +137,7 @@ auto LocalCAS::LocalUplinkGitTree( return true; }; auto tree_entries = GitRepo::ReadTreeData(*content, - id, + digest.hash(), check_symlinks, /*is_hex_id=*/true); if (not tree_entries) { @@ -151,17 +149,17 @@ auto LocalCAS::LocalUplinkGitTree( // Process only first entry from 'entry_vector' since all // entries represent the same blob, just with different // names. - auto entry = entry_vector.front(); - auto hash = ToHexString(raw_id); - auto digest = ArtifactDigest{hash, 0, IsTreeObject(entry.type)}; - if (entry.type == ObjectType::Tree) { + auto const entry_type = entry_vector.front().type; + auto const digest = + ArtifactDigest{ToHexString(raw_id), 0, IsTreeObject(entry_type)}; + if (digest.IsTree()) { if (not LocalUplinkGitTree(latest, digest)) { return false; } } else { if (not LocalUplinkBlob( - latest, digest, IsExecutableObject(entry.type))) { + latest, digest, IsExecutableObject(entry_type))) { return false; } } @@ -190,20 +188,19 @@ template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkBazelDirectory( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, - gsl::not_null*> const& seen, + ArtifactDigest const& digest, + gsl::not_null*> const& seen, bool splice_result) const noexcept -> bool { // Skip already uplinked directories if (seen->contains(digest)) { return true; } - auto const a_digest = static_cast(digest); // Determine bazel directory path of given generation. - auto dir_path = cas_tree_.BlobPath(a_digest); + auto dir_path = cas_tree_.BlobPath(digest); std::optional spliced; if (not dir_path) { - spliced = TrySplice(a_digest); + spliced = TrySplice(digest); dir_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not dir_path) { @@ -219,19 +216,20 @@ auto LocalCAS::LocalUplinkBazelDirectory( // Uplink bazel directory entries. for (auto const& file : dir.files()) { - if (not LocalUplinkBlob(latest, file.digest(), file.is_executable())) { + ArtifactDigest const a_digest{file.digest()}; + if (not LocalUplinkBlob(latest, a_digest, file.is_executable())) { return false; } } for (auto const& directory : dir.directories()) { - if (not LocalUplinkBazelDirectory(latest, directory.digest(), seen)) { + ArtifactDigest const a_digest{directory.digest()}; + if (not LocalUplinkBazelDirectory(latest, a_digest, seen)) { return false; } } // Determine bazel directory path in latest generation. - auto dir_path_latest = latest.cas_tree_.BlobPath(a_digest); - + auto const dir_path_latest = latest.cas_tree_.BlobPath(digest); if (spliced) { // Uplink the large entry afterwards: // The result of uplinking of a large object must not affect the @@ -263,15 +261,14 @@ template requires(kIsLocalGeneration) auto LocalCAS::LocalUplinkLargeObject( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest) const noexcept -> bool { - auto const a_digest = static_cast(digest); + ArtifactDigest const& digest) const noexcept -> bool { if constexpr (IsTreeObject(kType)) { return cas_tree_large_.LocalUplink( - latest, latest.cas_tree_large_, a_digest); + latest, latest.cas_tree_large_, digest); } else { return cas_file_large_.LocalUplink( - latest, latest.cas_file_large_, a_digest); + latest, latest.cas_file_large_, digest); } } diff --git a/src/buildtool/storage/uplinker.cpp b/src/buildtool/storage/uplinker.cpp index fee01af4..8b3b8c06 100644 --- a/src/buildtool/storage/uplinker.cpp +++ b/src/buildtool/storage/uplinker.cpp @@ -19,7 +19,6 @@ #include #include -#include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/local_ac.hpp" #include "src/buildtool/storage/local_cas.hpp" @@ -46,7 +45,7 @@ GlobalUplinker::GlobalUplinker( : storage_config_{*storage_config}, generations_{CreateGenerations(&storage_config_)} {} -auto GlobalUplinker::UplinkBlob(bazel_re::Digest const& digest, +auto GlobalUplinker::UplinkBlob(ArtifactDigest const& digest, bool is_executable) const noexcept -> bool { // Try to find blob in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); @@ -62,7 +61,7 @@ auto GlobalUplinker::UplinkBlob(bazel_re::Digest const& digest, }); } -auto GlobalUplinker::UplinkTree(bazel_re::Digest const& digest) const noexcept +auto GlobalUplinker::UplinkTree(ArtifactDigest const& digest) const noexcept -> bool { // Try to find blob in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); @@ -75,7 +74,7 @@ auto GlobalUplinker::UplinkTree(bazel_re::Digest const& digest) const noexcept } auto GlobalUplinker::UplinkLargeBlob( - bazel_re::Digest const& digest) const noexcept -> bool { + ArtifactDigest const& digest) const noexcept -> bool { // Try to find large entry in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); return std::any_of( @@ -88,7 +87,7 @@ auto GlobalUplinker::UplinkLargeBlob( } auto GlobalUplinker::UplinkActionCacheEntry( - bazel_re::Digest const& action_id) const noexcept -> bool { + ArtifactDigest const& action_id) const noexcept -> bool { // Try to find action-cache entry in all generations. auto const& latest = generations_[Generation::kYoungest].ActionCache(); return std::any_of(generations_.begin(), diff --git a/src/buildtool/storage/uplinker.hpp b/src/buildtool/storage/uplinker.hpp index c713155c..353caac6 100644 --- a/src/buildtool/storage/uplinker.hpp +++ b/src/buildtool/storage/uplinker.hpp @@ -21,17 +21,13 @@ #include #include "gsl/gsl" +#include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/storage/config.hpp" template class LocalStorage; class TargetCacheKey; -namespace build::bazel::remote::execution::v2 { -class Digest; -} -namespace bazel_re = build::bazel::remote::execution::v2; - /// \brief Global uplinker implementation. /// Responsible for uplinking objects across all generations to latest /// generation. @@ -45,7 +41,7 @@ class GlobalUplinker final { /// \param digest Digest of the blob to uplink. /// \param is_executable Indicate that blob is an executable. /// \returns true if blob was found and successfully uplinked. - [[nodiscard]] auto UplinkBlob(bazel_re::Digest const& digest, + [[nodiscard]] auto UplinkBlob(ArtifactDigest const& digest, bool is_executable) const noexcept -> bool; /// \brief Uplink tree across LocalCASes from all generations to latest. @@ -53,7 +49,7 @@ class GlobalUplinker final { /// by this tree will be uplinked before (including sub-trees). /// \param digest Digest of the tree to uplink. /// \returns true if tree was found and successfully uplinked (deep). - [[nodiscard]] auto UplinkTree(bazel_re::Digest const& digest) const noexcept + [[nodiscard]] auto UplinkTree(ArtifactDigest const& digest) const noexcept -> bool; /// \brief Uplink large blob entry across LocalCASes from all generations to @@ -61,14 +57,14 @@ class GlobalUplinker final { /// \param digest Digest of the large blob entry to uplink. /// \returns true if large entry was found and successfully uplinked. [[nodiscard]] auto UplinkLargeBlob( - bazel_re::Digest const& digest) const noexcept -> bool; + ArtifactDigest const& digest) const noexcept -> bool; /// \brief Uplink entry from action cache across all generations to latest. /// Note that the entry will be uplinked including all referenced items. /// \param action_id Id of the action to uplink entry for. /// \returns true if cache entry was found and successfully uplinked. [[nodiscard]] auto UplinkActionCacheEntry( - bazel_re::Digest const& action_id) const noexcept -> bool; + ArtifactDigest const& action_id) const noexcept -> bool; /// \brief Uplink entry from target cache across all generations to latest. /// Note that the entry will be uplinked including all referenced items. -- cgit v1.2.3