diff options
author | Maksim Denisov <denisov.maksim@huawei.com> | 2024-08-29 14:12:05 +0200 |
---|---|---|
committer | Maksim Denisov <denisov.maksim@huawei.com> | 2024-08-30 17:17:09 +0200 |
commit | 5ecbcbbcf71ca9fafaddf9a2353fe31a75fb33e4 (patch) | |
tree | e5d1efdf4e0f6952e89d45d2ec7df6b612829bb3 /src | |
parent | ace40adf1ed751072bc858d40c08f8f56340b592 (diff) | |
download | justbuild-5ecbcbbcf71ca9fafaddf9a2353fe31a75fb33e4.tar.gz |
Replace bazel_re::Digest in Uplinker
...with ArtifactDigest.
Diffstat (limited to 'src')
-rw-r--r-- | src/buildtool/storage/local_ac.hpp | 2 | ||||
-rw-r--r-- | src/buildtool/storage/local_ac.tpp | 17 | ||||
-rw-r--r-- | src/buildtool/storage/local_cas.hpp | 12 | ||||
-rw-r--r-- | src/buildtool/storage/local_cas.tpp | 63 | ||||
-rw-r--r-- | src/buildtool/storage/uplinker.cpp | 9 | ||||
-rw-r--r-- | src/buildtool/storage/uplinker.hpp | 14 |
6 files changed, 55 insertions, 62 deletions
diff --git a/src/buildtool/storage/local_ac.hpp b/src/buildtool/storage/local_ac.hpp index 180f1c51..2e4f57a8 100644 --- a/src/buildtool/storage/local_ac.hpp +++ b/src/buildtool/storage/local_ac.hpp @@ -84,7 +84,7 @@ class LocalAC { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkEntry( LocalGenerationAC const& latest, - bazel_re::Digest const& action_id) const noexcept -> bool; + ArtifactDigest const& action_id) const noexcept -> bool; private: // The action cache stores the results of failed actions. For those to be diff --git a/src/buildtool/storage/local_ac.tpp b/src/buildtool/storage/local_ac.tpp index 3bf5ba31..2f67c798 100644 --- a/src/buildtool/storage/local_ac.tpp +++ b/src/buildtool/storage/local_ac.tpp @@ -55,16 +55,15 @@ template <bool kIsLocalGeneration> requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry( LocalGenerationAC const& latest, - bazel_re::Digest const& action_id) const noexcept -> bool { + ArtifactDigest const& action_id) const noexcept -> bool { // Determine action cache key path in latest generation. - ArtifactDigest const a_digest{action_id}; if (FileSystemManager::IsFile( - latest.file_store_.GetPath(a_digest.hash()))) { + latest.file_store_.GetPath(action_id.hash()))) { return true; } // Read cache key - auto const cas_key = ReadActionKey(a_digest); + auto const cas_key = ReadActionKey(action_id); if (not cas_key) { return false; } @@ -77,8 +76,9 @@ auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry( // Uplink result content for (auto const& file : result->output_files()) { + ArtifactDigest const a_digest{file.digest()}; if (not cas_.LocalUplinkBlob( - latest.cas_, file.digest(), file.is_executable())) { + latest.cas_, a_digest, file.is_executable())) { return false; } } @@ -99,7 +99,8 @@ auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry( } } for (auto const& directory : result->output_directories()) { - if (not cas_.LocalUplinkTree(latest.cas_, directory.tree_digest())) { + ArtifactDigest const a_digest{directory.tree_digest()}; + if (not cas_.LocalUplinkTree(latest.cas_, a_digest)) { return false; } } @@ -111,9 +112,9 @@ auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry( return false; } - auto const ac_entry_path = file_store_.GetPath(a_digest.hash()); + auto const ac_entry_path = file_store_.GetPath(action_id.hash()); // Uplink cache key - return latest.file_store_.AddFromFile(a_digest.hash(), + return latest.file_store_.AddFromFile(action_id.hash(), ac_entry_path, /*is_owner=*/true); } diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp index 6c653009..e7f5ce22 100644 --- a/src/buildtool/storage/local_cas.hpp +++ b/src/buildtool/storage/local_cas.hpp @@ -228,7 +228,7 @@ class LocalCAS { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBlob( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool is_executable, bool skip_sync = false, bool splice_result = false) const noexcept -> bool; @@ -251,7 +251,7 @@ class LocalCAS { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkTree( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool splice_result = false) const noexcept -> bool; /// \brief Uplink large entry from this generation to latest LocalCAS @@ -267,7 +267,7 @@ class LocalCAS { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkLargeObject( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest) const noexcept -> bool; + ArtifactDigest const& digest) const noexcept -> bool; private: ObjectCAS<ObjectType::File> cas_file_; @@ -316,15 +316,15 @@ class LocalCAS { requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkGitTree( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool splice_result = false) const noexcept -> bool; template <bool kIsLocalGeneration = not kDoGlobalUplink> requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkBazelDirectory( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, - gsl::not_null<std::unordered_set<bazel_re::Digest>*> const& seen, + ArtifactDigest const& digest, + gsl::not_null<std::unordered_set<ArtifactDigest>*> const& seen, bool splice_result = false) const noexcept -> bool; template <ObjectType kType, bool kIsLocalGeneration = not kDoGlobalUplink> diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp index cf1fd2d2..e2a77e87 100644 --- a/src/buildtool/storage/local_cas.tpp +++ b/src/buildtool/storage/local_cas.tpp @@ -19,6 +19,7 @@ #include <utility> // std::move #include "fmt/core.h" +#include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/storage/local_cas.hpp" @@ -27,23 +28,22 @@ template <bool kIsLocalGeneration> requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkBlob( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool is_executable, bool skip_sync, bool splice_result) const noexcept -> bool { - auto const a_digest = static_cast<ArtifactDigest>(digest); // Determine blob path in latest generation. - auto blob_path_latest = latest.BlobPathNoSync(a_digest, is_executable); + auto blob_path_latest = latest.BlobPathNoSync(digest, is_executable); if (blob_path_latest) { return true; } // Determine blob path of given generation. - auto blob_path = skip_sync ? BlobPathNoSync(a_digest, is_executable) - : BlobPath(a_digest, is_executable); + auto blob_path = skip_sync ? BlobPathNoSync(digest, is_executable) + : BlobPath(digest, is_executable); std::optional<LargeObject> spliced; if (not blob_path) { - spliced = TrySplice<ObjectType::File>(a_digest); + spliced = TrySplice<ObjectType::File>(digest); blob_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not blob_path) { @@ -80,10 +80,10 @@ template <bool kIsLocalGeneration> requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkTree( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool splice_result) const noexcept -> bool { if (Compatibility::IsCompatible()) { - std::unordered_set<bazel_re::Digest> seen{}; + std::unordered_set<ArtifactDigest> seen{}; return LocalUplinkBazelDirectory(latest, digest, &seen, splice_result); } return LocalUplinkGitTree(latest, digest, splice_result); @@ -94,20 +94,19 @@ template <bool kIsLocalGeneration> requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, + ArtifactDigest const& digest, bool splice_result) const noexcept -> bool { // Determine tree path in latest generation. - auto const a_digest = static_cast<ArtifactDigest>(digest); - auto tree_path_latest = latest.cas_tree_.BlobPath(a_digest); + auto tree_path_latest = latest.cas_tree_.BlobPath(digest); if (tree_path_latest) { return true; } // Determine tree path of given generation. - auto tree_path = cas_tree_.BlobPath(a_digest); + auto tree_path = cas_tree_.BlobPath(digest); std::optional<LargeObject> spliced; if (not tree_path) { - spliced = TrySplice<ObjectType::Tree>(a_digest); + spliced = TrySplice<ObjectType::Tree>(digest); tree_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not tree_path) { @@ -116,7 +115,6 @@ auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree( // Determine tree entries. auto content = FileSystemManager::ReadFile(*tree_path); - auto id = NativeSupport::Unprefix(digest.hash()); auto check_symlinks = [this](std::vector<ArtifactDigest> const& ids) -> bool { for (auto const& id : ids) { @@ -139,7 +137,7 @@ auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree( return true; }; auto tree_entries = GitRepo::ReadTreeData(*content, - id, + digest.hash(), check_symlinks, /*is_hex_id=*/true); if (not tree_entries) { @@ -151,17 +149,17 @@ auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree( // Process only first entry from 'entry_vector' since all // entries represent the same blob, just with different // names. - auto entry = entry_vector.front(); - auto hash = ToHexString(raw_id); - auto digest = ArtifactDigest{hash, 0, IsTreeObject(entry.type)}; - if (entry.type == ObjectType::Tree) { + auto const entry_type = entry_vector.front().type; + auto const digest = + ArtifactDigest{ToHexString(raw_id), 0, IsTreeObject(entry_type)}; + if (digest.IsTree()) { if (not LocalUplinkGitTree(latest, digest)) { return false; } } else { if (not LocalUplinkBlob( - latest, digest, IsExecutableObject(entry.type))) { + latest, digest, IsExecutableObject(entry_type))) { return false; } } @@ -190,20 +188,19 @@ template <bool kIsLocalGeneration> requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkBazelDirectory( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest, - gsl::not_null<std::unordered_set<bazel_re::Digest>*> const& seen, + ArtifactDigest const& digest, + gsl::not_null<std::unordered_set<ArtifactDigest>*> const& seen, bool splice_result) const noexcept -> bool { // Skip already uplinked directories if (seen->contains(digest)) { return true; } - auto const a_digest = static_cast<ArtifactDigest>(digest); // Determine bazel directory path of given generation. - auto dir_path = cas_tree_.BlobPath(a_digest); + auto dir_path = cas_tree_.BlobPath(digest); std::optional<LargeObject> spliced; if (not dir_path) { - spliced = TrySplice<ObjectType::Tree>(a_digest); + spliced = TrySplice<ObjectType::Tree>(digest); dir_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not dir_path) { @@ -219,19 +216,20 @@ auto LocalCAS<kDoGlobalUplink>::LocalUplinkBazelDirectory( // Uplink bazel directory entries. for (auto const& file : dir.files()) { - if (not LocalUplinkBlob(latest, file.digest(), file.is_executable())) { + ArtifactDigest const a_digest{file.digest()}; + if (not LocalUplinkBlob(latest, a_digest, file.is_executable())) { return false; } } for (auto const& directory : dir.directories()) { - if (not LocalUplinkBazelDirectory(latest, directory.digest(), seen)) { + ArtifactDigest const a_digest{directory.digest()}; + if (not LocalUplinkBazelDirectory(latest, a_digest, seen)) { return false; } } // Determine bazel directory path in latest generation. - auto dir_path_latest = latest.cas_tree_.BlobPath(a_digest); - + auto const dir_path_latest = latest.cas_tree_.BlobPath(digest); if (spliced) { // Uplink the large entry afterwards: // The result of uplinking of a large object must not affect the @@ -263,15 +261,14 @@ template <ObjectType kType, bool kIsLocalGeneration> requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkLargeObject( LocalGenerationCAS const& latest, - bazel_re::Digest const& digest) const noexcept -> bool { - auto const a_digest = static_cast<ArtifactDigest>(digest); + ArtifactDigest const& digest) const noexcept -> bool { if constexpr (IsTreeObject(kType)) { return cas_tree_large_.LocalUplink( - latest, latest.cas_tree_large_, a_digest); + latest, latest.cas_tree_large_, digest); } else { return cas_file_large_.LocalUplink( - latest, latest.cas_file_large_, a_digest); + latest, latest.cas_file_large_, digest); } } diff --git a/src/buildtool/storage/uplinker.cpp b/src/buildtool/storage/uplinker.cpp index fee01af4..8b3b8c06 100644 --- a/src/buildtool/storage/uplinker.cpp +++ b/src/buildtool/storage/uplinker.cpp @@ -19,7 +19,6 @@ #include <algorithm> #include <cstddef> -#include "src/buildtool/common/bazel_types.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/local_ac.hpp" #include "src/buildtool/storage/local_cas.hpp" @@ -46,7 +45,7 @@ GlobalUplinker::GlobalUplinker( : storage_config_{*storage_config}, generations_{CreateGenerations(&storage_config_)} {} -auto GlobalUplinker::UplinkBlob(bazel_re::Digest const& digest, +auto GlobalUplinker::UplinkBlob(ArtifactDigest const& digest, bool is_executable) const noexcept -> bool { // Try to find blob in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); @@ -62,7 +61,7 @@ auto GlobalUplinker::UplinkBlob(bazel_re::Digest const& digest, }); } -auto GlobalUplinker::UplinkTree(bazel_re::Digest const& digest) const noexcept +auto GlobalUplinker::UplinkTree(ArtifactDigest const& digest) const noexcept -> bool { // Try to find blob in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); @@ -75,7 +74,7 @@ auto GlobalUplinker::UplinkTree(bazel_re::Digest const& digest) const noexcept } auto GlobalUplinker::UplinkLargeBlob( - bazel_re::Digest const& digest) const noexcept -> bool { + ArtifactDigest const& digest) const noexcept -> bool { // Try to find large entry in all generations. auto const& latest = generations_[Generation::kYoungest].CAS(); return std::any_of( @@ -88,7 +87,7 @@ auto GlobalUplinker::UplinkLargeBlob( } auto GlobalUplinker::UplinkActionCacheEntry( - bazel_re::Digest const& action_id) const noexcept -> bool { + ArtifactDigest const& action_id) const noexcept -> bool { // Try to find action-cache entry in all generations. auto const& latest = generations_[Generation::kYoungest].ActionCache(); return std::any_of(generations_.begin(), diff --git a/src/buildtool/storage/uplinker.hpp b/src/buildtool/storage/uplinker.hpp index c713155c..353caac6 100644 --- a/src/buildtool/storage/uplinker.hpp +++ b/src/buildtool/storage/uplinker.hpp @@ -21,17 +21,13 @@ #include <vector> #include "gsl/gsl" +#include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/storage/config.hpp" template <bool> class LocalStorage; class TargetCacheKey; -namespace build::bazel::remote::execution::v2 { -class Digest; -} -namespace bazel_re = build::bazel::remote::execution::v2; - /// \brief Global uplinker implementation. /// Responsible for uplinking objects across all generations to latest /// generation. @@ -45,7 +41,7 @@ class GlobalUplinker final { /// \param digest Digest of the blob to uplink. /// \param is_executable Indicate that blob is an executable. /// \returns true if blob was found and successfully uplinked. - [[nodiscard]] auto UplinkBlob(bazel_re::Digest const& digest, + [[nodiscard]] auto UplinkBlob(ArtifactDigest const& digest, bool is_executable) const noexcept -> bool; /// \brief Uplink tree across LocalCASes from all generations to latest. @@ -53,7 +49,7 @@ class GlobalUplinker final { /// by this tree will be uplinked before (including sub-trees). /// \param digest Digest of the tree to uplink. /// \returns true if tree was found and successfully uplinked (deep). - [[nodiscard]] auto UplinkTree(bazel_re::Digest const& digest) const noexcept + [[nodiscard]] auto UplinkTree(ArtifactDigest const& digest) const noexcept -> bool; /// \brief Uplink large blob entry across LocalCASes from all generations to @@ -61,14 +57,14 @@ class GlobalUplinker final { /// \param digest Digest of the large blob entry to uplink. /// \returns true if large entry was found and successfully uplinked. [[nodiscard]] auto UplinkLargeBlob( - bazel_re::Digest const& digest) const noexcept -> bool; + ArtifactDigest const& digest) const noexcept -> bool; /// \brief Uplink entry from action cache across all generations to latest. /// Note that the entry will be uplinked including all referenced items. /// \param action_id Id of the action to uplink entry for. /// \returns true if cache entry was found and successfully uplinked. [[nodiscard]] auto UplinkActionCacheEntry( - bazel_re::Digest const& action_id) const noexcept -> bool; + ArtifactDigest const& action_id) const noexcept -> bool; /// \brief Uplink entry from target cache across all generations to latest. /// Note that the entry will be uplinked including all referenced items. |