summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/storage/large_object_cas.hpp44
-rw-r--r--src/buildtool/storage/large_object_cas.tpp63
-rw-r--r--src/buildtool/storage/local_cas.hpp5
-rw-r--r--src/buildtool/storage/local_cas.tpp59
4 files changed, 158 insertions, 13 deletions
diff --git a/src/buildtool/storage/large_object_cas.hpp b/src/buildtool/storage/large_object_cas.hpp
index d7af8e9f..0d658af6 100644
--- a/src/buildtool/storage/large_object_cas.hpp
+++ b/src/buildtool/storage/large_object_cas.hpp
@@ -25,6 +25,8 @@
#include "src/buildtool/common/bazel_types.hpp"
#include "src/buildtool/file_system/file_storage.hpp"
#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/storage/config.hpp"
+#include "src/utils/cpp/tmp_dir.hpp"
template <bool>
class LocalCAS;
@@ -58,6 +60,29 @@ class LargeObjectError final {
std::string message_;
};
+/// \brief Stores a temporary directory containing a result of splicing.
+class LargeObject final {
+ public:
+ LargeObject() noexcept
+ : directory_(StorageConfig::CreateTypedTmpDir("splice")),
+ path_(directory_ ? directory_->GetPath() / "result" : ".") {}
+
+ /// \brief Check whether the large object is valid.
+ [[nodiscard]] auto IsValid() const noexcept -> bool {
+ return directory_ != nullptr;
+ }
+
+ /// \brief Obtain the path to the spliced result.
+ [[nodiscard]] auto GetPath() const noexcept
+ -> std::filesystem::path const& {
+ return path_;
+ }
+
+ private:
+ TmpDirPtr directory_;
+ std::filesystem::path path_;
+};
+
/// \brief Stores auxiliary information for reconstructing large objects.
/// The entries are keyed by the hash of the spliced result and the value of an
/// entry is the concatenation of the hashes of chunks the large object is
@@ -89,6 +114,25 @@ class LargeObjectCAS final {
[[nodiscard]] auto Split(bazel_re::Digest const& digest) const noexcept
-> std::variant<LargeObjectError, std::vector<bazel_re::Digest>>;
+ /// \brief Splice an object based on the reconstruction rules from the
+ /// storage. This method doesn't check whether the result of splicing is
+ /// already in the CAS.
+ /// \param digest The digest of the object to be spliced.
+ /// \return A temporary directory that contains a single file
+ /// "result" on success or an error on failure.
+ [[nodiscard]] auto TrySplice(bazel_re::Digest const& digest) const noexcept
+ -> std::variant<LargeObjectError, LargeObject>;
+
+ /// \brief Splice an object from parts. This method doesn't check whether
+ /// the result of splicing is already in the CAS.
+ /// \param digest The digest of the resulting object.
+ /// \param parts Parts to be concatenated.
+ /// \return A temporary directory that contains a single file
+ /// "result" on success or an error on failure.
+ [[nodiscard]] auto Splice(bazel_re::Digest const& digest,
+ std::vector<bazel_re::Digest> const& parts)
+ const noexcept -> std::variant<LargeObjectError, LargeObject>;
+
private:
// By default, overwrite existing entries. Unless this is a generation
// (disabled global uplink), then we never want to overwrite any entries.
diff --git a/src/buildtool/storage/large_object_cas.tpp b/src/buildtool/storage/large_object_cas.tpp
index 5adcb6a3..71b4528e 100644
--- a/src/buildtool/storage/large_object_cas.tpp
+++ b/src/buildtool/storage/large_object_cas.tpp
@@ -150,4 +150,67 @@ auto LargeObjectCAS<kDoGlobalUplink, kType>::Split(
return parts;
}
+template <bool kDoGlobalUplink, ObjectType kType>
+auto LargeObjectCAS<kDoGlobalUplink, kType>::TrySplice(
+ bazel_re::Digest const& digest) const noexcept
+ -> std::variant<LargeObjectError, LargeObject> {
+ auto parts = ReadEntry(digest);
+ if (not parts) {
+ return LargeObjectError{
+ LargeObjectErrorCode::FileNotFound,
+ fmt::format("could not find large entry for {}", digest.hash())};
+ }
+ return Splice(digest, *parts);
+}
+
+template <bool kDoGlobalUplink, ObjectType kType>
+auto LargeObjectCAS<kDoGlobalUplink, kType>::Splice(
+ bazel_re::Digest const& digest,
+ std::vector<bazel_re::Digest> const& parts) const noexcept
+ -> std::variant<LargeObjectError, LargeObject> {
+ // Create temporary space for splicing:
+ LargeObject large_object;
+ if (not large_object.IsValid()) {
+ return LargeObjectError{
+ LargeObjectErrorCode::Internal,
+ fmt::format("could not create a temporary space for {}",
+ digest.hash())};
+ }
+
+ // Splice the object from parts
+ try {
+ std::ofstream stream(large_object.GetPath());
+ for (auto const& part : parts) {
+ auto part_path = local_cas_.BlobPath(part, /*is_executable=*/false);
+ if (not part_path) {
+ return LargeObjectError{
+ LargeObjectErrorCode::FileNotFound,
+ fmt::format("could not find the part {}", part.hash())};
+ }
+
+ auto part_content = FileSystemManager::ReadFile(*part_path);
+ if (not part_content) {
+ return LargeObjectError{
+ LargeObjectErrorCode::Internal,
+ fmt::format("could not read the part content {}",
+ part.hash())};
+ }
+
+ if (stream.good()) {
+ stream << *part_content;
+ }
+ else {
+ return LargeObjectError{
+ LargeObjectErrorCode::Internal,
+ fmt::format("could not splice {}", digest.hash())};
+ }
+ }
+ stream.close();
+ } catch (...) {
+ return LargeObjectError{LargeObjectErrorCode::Internal,
+ "an unknown error occured"};
+ }
+ return large_object;
+}
+
#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LARGE_OBJECT_CAS_TPP
diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp
index ee7ecdf8..e4477011 100644
--- a/src/buildtool/storage/local_cas.hpp
+++ b/src/buildtool/storage/local_cas.hpp
@@ -266,6 +266,11 @@ class LocalCAS {
bazel_re::Digest const& digest,
gsl::not_null<std::unordered_set<bazel_re::Digest>*> const& seen)
const noexcept -> bool;
+
+ template <ObjectType kType, bool kIsLocalGeneration = not kDoGlobalUplink>
+ requires(kIsLocalGeneration) [[nodiscard]] auto TrySplice(
+ bazel_re::Digest const& digest) const noexcept
+ -> std::optional<LargeObject>;
};
#ifndef BOOTSTRAP_BUILD_TOOL
diff --git a/src/buildtool/storage/local_cas.tpp b/src/buildtool/storage/local_cas.tpp
index 744a14e9..8c20ded9 100644
--- a/src/buildtool/storage/local_cas.tpp
+++ b/src/buildtool/storage/local_cas.tpp
@@ -284,6 +284,11 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkBlob(
// Determine blob path of given generation.
auto blob_path = skip_sync ? BlobPathNoSync(digest, is_executable)
: BlobPath(digest, is_executable);
+ std::optional<LargeObject> spliced;
+ if (not blob_path) {
+ spliced = TrySplice<ObjectType::File>(digest);
+ blob_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt;
+ }
if (not blob_path) {
return false;
}
@@ -318,6 +323,11 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree(
// Determine tree path of given generation.
auto tree_path = cas_tree_.BlobPath(digest);
+ std::optional<LargeObject> spliced;
+ if (not tree_path) {
+ spliced = TrySplice<ObjectType::Tree>(digest);
+ tree_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt;
+ }
if (not tree_path) {
return false;
}
@@ -326,20 +336,26 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::LocalUplinkGitTree(
auto content = FileSystemManager::ReadFile(*tree_path);
auto id = NativeSupport::Unprefix(digest.hash());
auto check_symlinks =
- [cas = &cas_file_](std::vector<bazel_re::Digest> const& ids) {
- for (auto const& id : ids) {
- auto link_path = cas->BlobPath(id);
- if (not link_path) {
- return false;
- }
- // in the local CAS we store as files
- auto content = FileSystemManager::ReadFile(*link_path);
- if (not content or not PathIsNonUpwards(*content)) {
- return false;
- }
+ [this](std::vector<bazel_re::Digest> const& ids) -> bool {
+ for (auto const& id : ids) {
+ auto link_path = cas_file_.BlobPath(id);
+ std::optional<LargeObject> spliced;
+ if (not link_path) {
+ spliced = TrySplice<ObjectType::File>(id);
+ link_path =
+ spliced ? std::optional{spliced->GetPath()} : std::nullopt;
}
- return true;
- };
+ if (not link_path) {
+ return false;
+ }
+ // in the local CAS we store as files
+ auto content = FileSystemManager::ReadFile(*link_path);
+ if (not content or not PathIsNonUpwards(*content)) {
+ return false;
+ }
+ }
+ return true;
+ };
auto tree_entries = GitRepo::ReadTreeData(*content,
id,
check_symlinks,
@@ -391,6 +407,11 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::
// Determine bazel directory path of given generation.
auto dir_path = cas_tree_.BlobPath(digest);
+ std::optional<LargeObject> spliced;
+ if (not dir_path) {
+ spliced = TrySplice<ObjectType::Tree>(digest);
+ dir_path = spliced ? std::optional{spliced->GetPath()} : std::nullopt;
+ }
if (not dir_path) {
return false;
}
@@ -431,4 +452,16 @@ requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::
return false;
}
+template <bool kDoGlobalUplink>
+template <ObjectType kType, bool kIsLocalGeneration>
+requires(kIsLocalGeneration) auto LocalCAS<kDoGlobalUplink>::TrySplice(
+ bazel_re::Digest const& digest) const noexcept
+ -> std::optional<LargeObject> {
+ auto spliced = IsTreeObject(kType) ? cas_tree_large_.TrySplice(digest)
+ : cas_file_large_.TrySplice(digest);
+ auto* large = std::get_if<LargeObject>(&spliced);
+ return large and large->IsValid() ? std::optional{std::move(*large)}
+ : std::nullopt;
+}
+
#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_LOCAL_CAS_TPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 16:26:15 +0000