From d9ff1b6073330c304fecdf77a16702a0df5faeac Mon Sep 17 00:00:00 2001 From: Maksim Denisov Date: Tue, 23 Jul 2024 15:07:04 +0200 Subject: Replace bazel_re::Digest in LargeObjectCAS ...with ArtifactDigest. --- src/buildtool/storage/large_object_cas.hpp | 26 +++++------ src/buildtool/storage/large_object_cas.tpp | 53 +++++++++------------- src/buildtool/storage/local_cas.hpp | 71 ++++++++++++++++++++++++------ src/buildtool/storage/local_cas.tpp | 32 +++++++------- 4 files changed, 109 insertions(+), 73 deletions(-) (limited to 'src') diff --git a/src/buildtool/storage/large_object_cas.hpp b/src/buildtool/storage/large_object_cas.hpp index 971b943d..8a6986ee 100644 --- a/src/buildtool/storage/large_object_cas.hpp +++ b/src/buildtool/storage/large_object_cas.hpp @@ -21,7 +21,7 @@ #include #include -#include "src/buildtool/common/bazel_types.hpp" +#include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/file_system/file_storage.hpp" #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/storage/config.hpp" @@ -123,16 +123,16 @@ class LargeObjectCAS final { /// \brief Get the path to a large entry in the storage. /// \param digest The digest of a large object. /// \returns Path to the large entry if in the storage. - [[nodiscard]] auto GetEntryPath(bazel_re::Digest const& digest) - const noexcept -> std::optional; + [[nodiscard]] auto GetEntryPath(ArtifactDigest const& digest) const noexcept + -> std::optional; /// \brief Split an object from the main CAS into chunks. If the object had /// been split before, it would not get split again. /// \param digest The digest of the object to be split. /// \return A set of chunks the resulting object is composed of /// or an error on failure. - [[nodiscard]] auto Split(bazel_re::Digest const& digest) const noexcept - -> expected, LargeObjectError>; + [[nodiscard]] auto Split(ArtifactDigest const& digest) const noexcept + -> expected, LargeObjectError>; /// \brief Splice an object based on the reconstruction rules from the /// storage. This method doesn't check whether the result of splicing is @@ -140,7 +140,7 @@ class LargeObjectCAS final { /// \param digest The digest of the object to be spliced. /// \return A temporary directory that contains a single file /// "result" on success or an error on failure. - [[nodiscard]] auto TrySplice(bazel_re::Digest const& digest) const noexcept + [[nodiscard]] auto TrySplice(ArtifactDigest const& digest) const noexcept -> expected; /// \brief Splice an object from parts. This method doesn't check whether @@ -149,8 +149,8 @@ class LargeObjectCAS final { /// \param parts Parts to be concatenated. /// \return A temporary directory that contains a single file /// "result" on success or an error on failure. - [[nodiscard]] auto Splice(bazel_re::Digest const& digest, - std::vector const& parts) + [[nodiscard]] auto Splice(ArtifactDigest const& digest, + std::vector const& parts) const noexcept -> expected; /// \brief Uplink large entry from this generation to latest LocalCAS @@ -167,7 +167,7 @@ class LargeObjectCAS final { [[nodiscard]] auto LocalUplink( LocalCAS const& latest, LargeObjectCAS const& latest_large, - bazel_re::Digest const& digest) const noexcept -> bool; + ArtifactDigest const& digest) const noexcept -> bool; private: // By default, overwrite existing entries. Unless this is a generation @@ -185,16 +185,16 @@ class LargeObjectCAS final { /// \param digest The digest of a large object. /// \returns Parts the large object is composed of, if present in /// the storage. - [[nodiscard]] auto ReadEntry(bazel_re::Digest const& digest) const noexcept - -> std::optional>; + [[nodiscard]] auto ReadEntry(ArtifactDigest const& digest) const noexcept + -> std::optional>; /// \brief Create a new entry description and add it to the storage. /// \param digest The digest of the result. /// \param parts Parts the resulting object is composed of. /// \returns True if the entry exists afterwards. [[nodiscard]] auto WriteEntry( - bazel_re::Digest const& digest, - std::vector const& parts) const noexcept -> bool; + ArtifactDigest const& digest, + std::vector const& parts) const noexcept -> bool; }; #include "src/buildtool/storage/large_object_cas.tpp" diff --git a/src/buildtool/storage/large_object_cas.tpp b/src/buildtool/storage/large_object_cas.tpp index 5fbfee07..18c15ce6 100644 --- a/src/buildtool/storage/large_object_cas.tpp +++ b/src/buildtool/storage/large_object_cas.tpp @@ -23,7 +23,6 @@ #include "fmt/core.h" #include "nlohmann/json.hpp" #include "src/buildtool/compatibility/compatibility.hpp" -#include "src/buildtool/compatibility/native_support.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/storage/file_chunker.hpp" #include "src/buildtool/storage/large_object_cas.hpp" @@ -36,10 +35,9 @@ inline constexpr std::size_t kSizeIndex = 1; template auto LargeObjectCAS::GetEntryPath( - bazel_re::Digest const& digest) const noexcept + ArtifactDigest const& digest) const noexcept -> std::optional { - const std::string hash = NativeSupport::Unprefix(digest.hash()); - const std::filesystem::path file_path = file_store_.GetPath(hash); + const std::filesystem::path file_path = file_store_.GetPath(digest.hash()); if (FileSystemManager::IsFile(file_path)) { return file_path; } @@ -60,14 +58,14 @@ auto LargeObjectCAS::GetEntryPath( template auto LargeObjectCAS::ReadEntry( - bazel_re::Digest const& digest) const noexcept - -> std::optional> { + ArtifactDigest const& digest) const noexcept + -> std::optional> { auto const file_path = GetEntryPath(digest); if (not file_path) { return std::nullopt; } - std::vector parts; + std::vector parts; try { std::ifstream stream(*file_path); nlohmann::json j = nlohmann::json::parse(stream); @@ -77,8 +75,7 @@ auto LargeObjectCAS::ReadEntry( auto hash = j_part.at(kHashIndex).template get(); auto size = j_part.at(kSizeIndex).template get(); - parts.emplace_back( - ArtifactDigest{std::move(hash), size, /*is_tree=*/false}); + parts.emplace_back(std::move(hash), size, /*is_tree=*/false); } } catch (...) { return std::nullopt; @@ -88,8 +85,8 @@ auto LargeObjectCAS::ReadEntry( template auto LargeObjectCAS::WriteEntry( - bazel_re::Digest const& digest, - std::vector const& parts) const noexcept -> bool { + ArtifactDigest const& digest, + std::vector const& parts) const noexcept -> bool { if (GetEntryPath(digest)) { return true; } @@ -106,23 +103,18 @@ auto LargeObjectCAS::WriteEntry( try { for (auto const& part : parts) { auto& j_part = j.emplace_back(); - - ArtifactDigest const a_digest(part); - j_part[kHashIndex] = a_digest.hash(); - j_part[kSizeIndex] = a_digest.size(); + j_part[kHashIndex] = part.hash(); + j_part[kSizeIndex] = part.size(); } } catch (...) { return false; } - - const auto hash = NativeSupport::Unprefix(digest.hash()); - return file_store_.AddFromBytes(hash, j.dump()); + return file_store_.AddFromBytes(digest.hash(), j.dump()); } template -auto LargeObjectCAS::Split( - bazel_re::Digest const& digest) const noexcept - -> expected, LargeObjectError> { +auto LargeObjectCAS::Split(ArtifactDigest const& digest) + const noexcept -> expected, LargeObjectError> { if (auto large_entry = ReadEntry(digest)) { return std::move(*large_entry); } @@ -154,7 +146,7 @@ auto LargeObjectCAS::Split( fmt::format("could not split {}", digest.hash())}}; } - std::vector parts; + std::vector parts; try { while (auto chunk = chunker.NextChunk()) { auto part = local_cas_.StoreBlob(*chunk, /*is_executable=*/false); @@ -162,7 +154,7 @@ auto LargeObjectCAS::Split( return unexpected{LargeObjectError{ LargeObjectErrorCode::Internal, "could not store a part."}}; } - parts.push_back(std::move(*part)); + parts.emplace_back(*std::move(part)); } } catch (...) { return unexpected{LargeObjectError{LargeObjectErrorCode::Internal, @@ -180,7 +172,7 @@ auto LargeObjectCAS::Split( template auto LargeObjectCAS::TrySplice( - bazel_re::Digest const& digest) const noexcept + ArtifactDigest const& digest) const noexcept -> expected { auto parts = ReadEntry(digest); if (not parts) { @@ -193,8 +185,8 @@ auto LargeObjectCAS::TrySplice( template auto LargeObjectCAS::Splice( - bazel_re::Digest const& digest, - std::vector const& parts) const noexcept + ArtifactDigest const& digest, + std::vector const& parts) const noexcept -> expected { // Create temporary space for splicing: LargeObject large_object(storage_config_); @@ -236,7 +228,7 @@ auto LargeObjectCAS::Splice( stream.close(); } catch (...) { return unexpected{LargeObjectError{LargeObjectErrorCode::Internal, - "an unknown error occured"}}; + "an unknown error occurred"}}; } return large_object; } @@ -247,7 +239,7 @@ template auto LargeObjectCAS::LocalUplink( LocalCAS const& latest, LargeObjectCAS const& latest_large, - bazel_re::Digest const& digest) const noexcept -> bool { + ArtifactDigest const& digest) const noexcept -> bool { // Check the large entry in the youngest generation: if (latest_large.GetEntryPath(digest)) { return true; @@ -274,9 +266,8 @@ auto LargeObjectCAS::LocalUplink( if (not path) { return false; } - - const auto hash = NativeSupport::Unprefix(digest.hash()); - return latest_large.file_store_.AddFromFile(hash, *path, /*is_owner=*/true); + return latest_large.file_store_.AddFromFile( + digest.hash(), *path, /*is_owner=*/true); } #endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_TPP diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp index 3097780a..4fc11d4a 100644 --- a/src/buildtool/storage/local_cas.hpp +++ b/src/buildtool/storage/local_cas.hpp @@ -160,7 +160,17 @@ class LocalCAS { /// error code on failure. [[nodiscard]] auto SplitBlob(bazel_re::Digest const& digest) const noexcept -> expected, LargeObjectError> { - return cas_file_large_.Split(digest); + auto const a_digest = static_cast(digest); + auto split_result = cas_file_large_.Split(a_digest); + if (not split_result) { + return unexpected{std::move(split_result).error()}; + } + std::vector result; + result.reserve(split_result->size()); + std::copy(split_result->begin(), + split_result->end(), + std::back_inserter(result)); + return result; } /// \brief Splice a blob from parts. @@ -173,8 +183,22 @@ class LocalCAS { std::vector const& parts, bool is_executable) const noexcept -> expected { - return is_executable ? Splice(digest, parts) - : Splice(digest, parts); + auto const a_digest = static_cast(digest); + std::vector a_parts; + a_parts.reserve(parts.size()); + std::transform( + parts.begin(), + parts.end(), + std::back_inserter(a_parts), + [](auto const& digest) { return ArtifactDigest{digest}; }); + auto splice_result = + is_executable ? Splice(a_digest, a_parts) + : Splice(a_digest, a_parts); + + if (not splice_result) { + return unexpected{std::move(splice_result).error()}; + } + return static_cast(*splice_result); } /// \brief Obtain tree path from digest. @@ -192,7 +216,17 @@ class LocalCAS { /// error code on failure. [[nodiscard]] auto SplitTree(bazel_re::Digest const& digest) const noexcept -> expected, LargeObjectError> { - return cas_tree_large_.Split(digest); + auto const a_digest = static_cast(digest); + auto split_result = cas_tree_large_.Split(a_digest); + if (not split_result) { + return unexpected{std::move(split_result).error()}; + } + std::vector result; + result.reserve(split_result->size()); + std::copy(split_result->begin(), + split_result->end(), + std::back_inserter(result)); + return result; } /// \brief Splice a tree from parts. @@ -203,7 +237,19 @@ class LocalCAS { [[nodiscard]] auto SpliceTree(bazel_re::Digest const& digest, std::vector const& parts) const noexcept -> expected { - return Splice(digest, parts); + auto const a_digest = static_cast(digest); + std::vector a_parts; + a_parts.reserve(parts.size()); + std::transform( + parts.begin(), + parts.end(), + std::back_inserter(a_parts), + [](auto const& digest) { return ArtifactDigest{digest}; }); + auto splice_result = Splice(a_digest, a_parts); + if (not splice_result) { + return unexpected{std::move(splice_result).error()}; + } + return static_cast(*splice_result); } /// \brief Check whether all parts of the tree are in the storage. @@ -331,13 +377,13 @@ class LocalCAS { template requires(kIsLocalGeneration) - [[nodiscard]] auto TrySplice(bazel_re::Digest const& digest) const noexcept + [[nodiscard]] auto TrySplice(ArtifactDigest const& digest) const noexcept -> std::optional; template - [[nodiscard]] auto Splice(bazel_re::Digest const& digest, - std::vector const& parts) - const noexcept -> expected; + [[nodiscard]] auto Splice(ArtifactDigest const& digest, + std::vector const& parts) + const noexcept -> expected; }; #ifndef BOOTSTRAP_BUILD_TOOL @@ -353,10 +399,9 @@ auto LocalCAS::CheckTreeInvariant( template template -auto LocalCAS::Splice( - bazel_re::Digest const& digest, - std::vector const& parts) const noexcept - -> expected { +auto LocalCAS::Splice(ArtifactDigest const& digest, + std::vector const& parts) + const noexcept -> expected { return unexpected{ LargeObjectError{LargeObjectErrorCode::Internal, "not allowed"}}; } diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp index fb828617..7f44b0c2 100644 --- a/src/buildtool/storage/local_cas.tpp +++ b/src/buildtool/storage/local_cas.tpp @@ -59,7 +59,8 @@ auto LocalCAS::LocalUplinkBlob( : BlobPath(digest, is_executable); std::optional spliced; if (not blob_path) { - spliced = TrySplice(digest); + spliced = + TrySplice(static_cast(digest)); blob_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not blob_path) { @@ -123,7 +124,7 @@ auto LocalCAS::LocalUplinkGitTree( auto tree_path = cas_tree_.BlobPath(a_digest); std::optional spliced; if (not tree_path) { - spliced = TrySplice(digest); + spliced = TrySplice(a_digest); tree_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not tree_path) { @@ -219,7 +220,7 @@ auto LocalCAS::LocalUplinkBazelDirectory( auto dir_path = cas_tree_.BlobPath(a_digest); std::optional spliced; if (not dir_path) { - spliced = TrySplice(digest); + spliced = TrySplice(a_digest); dir_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt; } if (not dir_path) { @@ -280,21 +281,22 @@ template auto LocalCAS::LocalUplinkLargeObject( LocalGenerationCAS const& latest, bazel_re::Digest const& digest) const noexcept -> bool { + auto const a_digest = static_cast(digest); if constexpr (IsTreeObject(kType)) { return cas_tree_large_.LocalUplink( - latest, latest.cas_tree_large_, digest); + latest, latest.cas_tree_large_, a_digest); } else { return cas_file_large_.LocalUplink( - latest, latest.cas_file_large_, digest); + latest, latest.cas_file_large_, a_digest); } } template template requires(kIsLocalGeneration) -auto LocalCAS::TrySplice(bazel_re::Digest const& digest) - const noexcept -> std::optional { +auto LocalCAS::TrySplice( + ArtifactDigest const& digest) const noexcept -> std::optional { auto spliced = IsTreeObject(kType) ? cas_tree_large_.TrySplice(digest) : cas_file_large_.TrySplice(digest); return spliced and spliced->IsValid() ? std::optional{std::move(*spliced)} @@ -326,10 +328,9 @@ auto LocalCAS::CheckTreeInvariant( // Ensure all entries are in the storage: for (const auto& entry : *entries) { for (auto const& item : entry.second) { - bazel_re::Digest const digest = - ArtifactDigest(ToHexString(entry.first), - /*size_unknown=*/0ULL, - IsTreeObject(item.type)); + auto const digest = ArtifactDigest(ToHexString(entry.first), + /*size_unknown=*/0ULL, + IsTreeObject(item.type)); // To avoid splicing during search, large CASes are inspected first. bool const entry_exists = @@ -352,10 +353,9 @@ auto LocalCAS::CheckTreeInvariant( template template -auto LocalCAS::Splice( - bazel_re::Digest const& digest, - std::vector const& parts) const noexcept - -> expected { +auto LocalCAS::Splice(ArtifactDigest const& digest, + std::vector const& parts) + const noexcept -> expected { static constexpr bool kIsTree = IsTreeObject(kType); static constexpr bool kIsExec = IsExecutableObject(kType); @@ -414,7 +414,7 @@ auto LocalCAS::Splice( auto const stored_digest = kIsTree ? StoreTree(file_path) : StoreBlob(file_path, kIsExec); if (stored_digest) { - return std::move(*stored_digest); + return ArtifactDigest{std::move(*stored_digest)}; } return unexpected{ LargeObjectError{LargeObjectErrorCode::Internal, -- cgit v1.2.3