summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/storage/garbage_collector.cpp14
-rw-r--r--src/buildtool/storage/garbage_collector.hpp7
-rw-r--r--src/buildtool/storage/large_object_cas.hpp15
-rw-r--r--src/buildtool/storage/large_object_cas.tpp51
-rw-r--r--src/buildtool/storage/local_cas.hpp14
-rw-r--r--src/buildtool/storage/local_cas.tpp56
6 files changed, 151 insertions, 6 deletions
diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp
index cfc6daf5..232c8ab8 100644
--- a/src/buildtool/storage/garbage_collector.cpp
+++ b/src/buildtool/storage/garbage_collector.cpp
@@ -72,6 +72,20 @@ auto GarbageCollector::GlobalUplinkBlob(bazel_re::Digest const& digest,
return false;
}
+auto GarbageCollector::GlobalUplinkLargeBlob(
+ bazel_re::Digest const& digest) noexcept -> bool {
+ // Try to find large entry in all generations.
+ auto const& latest_cas = Storage::Generation(0).CAS();
+ for (std::size_t i = 0; i < StorageConfig::NumGenerations(); ++i) {
+ if (Storage::Generation(i)
+ .CAS()
+ .LocalUplinkLargeObject<ObjectType::File>(latest_cas, digest)) {
+ return true;
+ }
+ }
+ return false;
+}
+
auto GarbageCollector::GlobalUplinkTree(bazel_re::Digest const& digest) noexcept
-> bool {
// Try to find tree in all generations.
diff --git a/src/buildtool/storage/garbage_collector.hpp b/src/buildtool/storage/garbage_collector.hpp
index c4ceaf8a..28a5f08b 100644
--- a/src/buildtool/storage/garbage_collector.hpp
+++ b/src/buildtool/storage/garbage_collector.hpp
@@ -42,6 +42,13 @@ class GarbageCollector {
bool is_executable) noexcept
-> bool;
+ /// \brief Uplink large blob entry across LocalCASes from all generations to
+ /// latest. This method does not splice the large object.
+ /// \param digest Digest of the large blob entry to uplink.
+ /// \returns true if large entry was found and successfully uplinked.
+ [[nodiscard]] auto static GlobalUplinkLargeBlob(
+ bazel_re::Digest const& digest) noexcept -> bool;
+
/// \brief Uplink tree across LocalCASes from all generations to latest.
/// Note that the tree will be deeply uplinked, i.e., all entries referenced
/// by this tree will be uplinked before (including sub-trees).
diff --git a/src/buildtool/storage/large_object_cas.hpp b/src/buildtool/storage/large_object_cas.hpp
index 0d658af6..2c8a3348 100644
--- a/src/buildtool/storage/large_object_cas.hpp
+++ b/src/buildtool/storage/large_object_cas.hpp
@@ -133,6 +133,21 @@ class LargeObjectCAS final {
std::vector<bazel_re::Digest> const& parts)
const noexcept -> std::variant<LargeObjectError, LargeObject>;
+ /// \brief Uplink large entry from this generation to latest LocalCAS
+ /// generation. For the large entry it's parts get promoted first and then
+ /// the entry itself. This function is only available for instances that are
+ /// used as local GC generations (i.e., disabled global uplink).
+ /// \tparam kIsLocalGeneration True if this instance is a local generation.
+ /// \param latest The latest LocalCAS generation.
+ /// \param latest_large The latest LargeObjectCAS
+ /// \param digest The digest of the large entry to uplink.
+ /// \returns True if the large entry was successfully uplinked.
+ template <bool kIsLocalGeneration = not kDoGlobalUplink>
+ requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplink(
+ LocalCAS<false> const& latest,
+ LargeObjectCAS<false, kType> const& latest_large,
+ bazel_re::Digest const& digest) const noexcept -> bool;
+
private:
// By default, overwrite existing entries. Unless this is a generation
// (disabled global uplink), then we never want to overwrite any entries.
diff --git a/src/buildtool/storage/large_object_cas.tpp b/src/buildtool/storage/large_object_cas.tpp
index 71b4528e..4bab5201 100644
--- a/src/buildtool/storage/large_object_cas.tpp
+++ b/src/buildtool/storage/large_object_cas.tpp
@@ -21,9 +21,11 @@
#include "fmt/core.h"
#include "nlohmann/json.hpp"
+#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/compatibility/native_support.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/storage/file_chunker.hpp"
+#include "src/buildtool/storage/garbage_collector.hpp"
#include "src/buildtool/storage/large_object_cas.hpp"
#include "src/buildtool/storage/local_cas.hpp"
@@ -36,6 +38,18 @@ auto LargeObjectCAS<kDoGlobalUplink, kType>::GetEntryPath(
if (FileSystemManager::IsFile(file_path)) {
return file_path;
}
+
+ if constexpr (kDoGlobalUplink) {
+ // To promote parts of the tree properly, regular uplinking logic for
+ // trees is used:
+ bool uplinked =
+ IsTreeObject(kType) and not Compatibility::IsCompatible()
+ ? GarbageCollector::GlobalUplinkTree(digest)
+ : GarbageCollector::GlobalUplinkLargeBlob(digest);
+ if (uplinked and FileSystemManager::IsFile(file_path)) {
+ return file_path;
+ }
+ }
return std::nullopt;
}
@@ -213,4 +227,41 @@ auto LargeObjectCAS<kDoGlobalUplink, kType>::Splice(
return large_object;
}
+template <bool kDoGlobalUplink, ObjectType kType>
+template <bool kIsLocalGeneration>
+requires(kIsLocalGeneration) auto LargeObjectCAS<kDoGlobalUplink, kType>::
+ LocalUplink(LocalCAS<false> const& latest,
+ LargeObjectCAS<false, kType> const& latest_large,
+ bazel_re::Digest const& digest) const noexcept -> bool {
+ // Check the large entry in the youngest generation:
+ if (latest_large.GetEntryPath(digest)) {
+ return true;
+ }
+
+ // Check the large entry in the current generation:
+ auto parts = ReadEntry(digest);
+ if (not parts) {
+ // No large entry or the object is not large
+ return true;
+ }
+
+ // Promoting the parts of the large entry:
+ for (auto const& part : *parts) {
+ static constexpr bool is_executable = false;
+ static constexpr bool skip_sync = true;
+ if (not local_cas_.LocalUplinkBlob(
+ latest, part, is_executable, skip_sync)) {
+ return false;
+ }
+ }
+
+ auto path = GetEntryPath(digest);
+ if (not path) {
+ return false;
+ }
+
+ const auto hash = NativeSupport::Unprefix(digest.hash());
+ return latest_large.file_store_.AddFromFile(hash, *path, /*is_owner=*/true);
+}
+
#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_TPP
diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp
index e4477011..057087ef 100644
--- a/src/buildtool/storage/local_cas.hpp
+++ b/src/buildtool/storage/local_cas.hpp
@@ -206,6 +206,20 @@ class LocalCAS {
LocalGenerationCAS const& latest,
bazel_re::Digest const& digest) const noexcept -> bool;
+ /// \brief Uplink large entry from this generation to latest LocalCAS
+ /// generation. This function is only available for instances that are used
+ /// as local GC generations (i.e., disabled global uplink).
+ /// \tparam kType Type of the large entry to be uplinked.
+ /// \tparam kIsLocalGeneration True if this instance is a local generation.
+ /// \param latest The latest LocalCAS generation.
+ /// \param latest_large The latest LargeObjectCAS
+ /// \param digest The digest of the large entry to uplink.
+ /// \returns True if the large entry was successfully uplinked.
+ template <ObjectType kType, bool kIsLocalGeneration = not kDoGlobalUplink>
+ requires(kIsLocalGeneration) [[nodiscard]] auto LocalUplinkLargeObject(
+ LocalGenerationCAS const& latest,
+ bazel_re::Digest const& digest) const noexcept -> bool;
+
private:
ObjectCAS<ObjectType::File> cas_file_;
ObjectCAS<ObjectType::Executable> cas_exec_;
diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp
index 8c20ded9..0c2d794d 100644
--- a/src/buildtool/storage/local_cas.tpp
+++ b/src/buildtool/storage/local_cas.tpp
@@ -294,8 +294,19 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkBlob(
}
// Uplink blob from older generation to the latest generation.
- return blob_path_latest.has_value() or
- latest.StoreBlob</*kOwner=*/true>(*blob_path, is_executable);
+ bool uplinked =
+ blob_path_latest.has_value() or
+ latest.StoreBlob</*kOwner=*/true>(*blob_path, is_executable);
+
+ if (uplinked) {
+ // The result of uplinking of a large object must not affect the
+ // result of uplinking in general. In other case, two sequential calls
+ // to BlobPath might return different results: The first call splices
+ // and uplinks the object, but fails at large entry uplinking. The
+ // second call finds the tree in the youngest generation and returns.
+ std::ignore = LocalUplinkLargeObject<ObjectType::File>(latest, digest);
+ }
+ return uplinked;
}
template <bool kDoGlobalUplink>
@@ -386,10 +397,17 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree(
}
// Uplink tree from older generation to the latest generation.
- return latest.cas_tree_
- .StoreBlobFromFile(*tree_path,
- /*is_owner=*/true)
- .has_value();
+ if (latest.cas_tree_.StoreBlobFromFile(*tree_path, /*is owner=*/true)) {
+ // Uplink the large entry afterwards:
+ // The result of uplinking of a large object must not affect the
+ // result of uplinking in general. In other case, two sequential calls
+ // to TreePath might return different results: The first call splices
+ // and uplinks the object, but fails at large entry uplinking. The
+ // second call finds the tree in the youngest generation and returns.
+ std::ignore = LocalUplinkLargeObject<ObjectType::Tree>(latest, digest);
+ return true;
+ }
+ return false;
}
template <bool kDoGlobalUplink>
@@ -445,6 +463,16 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::
/*is_owner=*/true)) {
try {
seen->emplace(digest);
+
+ // Uplink the large entry afterwards:
+ // The result of uplinking of a large object must not affect the
+ // result of uplinking in general. In other case, two sequential
+ // calls to TreePath might return different results: The first call
+ // splices and uplinks the object, but fails at large entry
+ // uplinking. The second call finds the tree in the youngest
+ // generation and returns.
+ std::ignore =
+ LocalUplinkLargeObject<ObjectType::Tree>(latest, digest);
return true;
} catch (...) {
}
@@ -454,6 +482,22 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::
template <bool kDoGlobalUplink>
template <ObjectType kType, bool kIsLocalGeneration>
+requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::
+ LocalUplinkLargeObject(LocalGenerationCAS const& latest,
+ bazel_re::Digest const& digest) const noexcept
+ -> bool {
+ if constexpr (IsTreeObject(kType)) {
+ return cas_tree_large_.LocalUplink(
+ latest, latest.cas_tree_large_, digest);
+ }
+ else {
+ return cas_file_large_.LocalUplink(
+ latest, latest.cas_file_large_, digest);
+ }
+}
+
+template <bool kDoGlobalUplink>
+template <ObjectType kType, bool kIsLocalGeneration>
requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::TrySplice(
bazel_re::Digest const& digest) const noexcept
-> std::optional<LargeObject> {
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 16:38:23 +0000