summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaksim Denisov <denisov.maksim@huawei.com>2024-07-23 14:07:48 +0200
committerMaksim Denisov <denisov.maksim@huawei.com>2024-08-26 17:02:14 +0200
commit7a4bf22ac5330024b4b807c9f7cf42292a5135d1 (patch)
treecdfe1a62167f993c1bbcf61aa09cbfcdaae9d58b
parent3966dab59e45e3cbacb52781ef74e73f6db2b142 (diff)
downloadjustbuild-7a4bf22ac5330024b4b807c9f7cf42292a5135d1.tar.gz
Reduce the size of large object entries.
Removed named keys of entries (hash, size). These prefixes were added for every chunk creating an additional overhead without any benefit. Removed prefixes of hashes (*62*hash) appearing in native mode and creating an additional overhead as well: it is known that all of them are blobs.
-rw-r--r--CHANGELOG.md3
-rw-r--r--src/buildtool/storage/large_object_cas.tpp32
2 files changed, 21 insertions, 14 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3c53b23a..c120f104 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -54,6 +54,9 @@ A feature release on top of `1.3.0`, backwards compatible.
- Local execution no longer has the requirement that there exist no
more files with identical content than the hardlink limit of the
underlying file system.
+- The size of large object entries has been reduced. The cache and
+ CAS must be cleaned up since stable versions before `1.4.0` cannot use
+ the new format.
- Various improvements to the tests: dispatching of the summary
action is now possible, tests are independent of a .just-mrrc
file the user might have in their home directory
diff --git a/src/buildtool/storage/large_object_cas.tpp b/src/buildtool/storage/large_object_cas.tpp
index 0f79b231..dae1d696 100644
--- a/src/buildtool/storage/large_object_cas.tpp
+++ b/src/buildtool/storage/large_object_cas.tpp
@@ -29,6 +29,11 @@
#include "src/buildtool/storage/large_object_cas.hpp"
#include "src/buildtool/storage/local_cas.hpp"
+namespace {
+inline constexpr std::size_t kHashIndex = 0;
+inline constexpr std::size_t kSizeIndex = 1;
+} // namespace
+
template <bool kDoGlobalUplink, ObjectType kType>
auto LargeObjectCAS<kDoGlobalUplink, kType>::GetEntryPath(
bazel_re::Digest const& digest) const noexcept
@@ -66,15 +71,14 @@ auto LargeObjectCAS<kDoGlobalUplink, kType>::ReadEntry(
try {
std::ifstream stream(*file_path);
nlohmann::json j = nlohmann::json::parse(stream);
- const std::size_t size = j.at("size").template get<std::size_t>();
- parts.reserve(size);
-
- auto const& j_parts = j.at("parts");
- for (std::size_t i = 0; i < size; ++i) {
- bazel_re::Digest& d = parts.emplace_back();
- d.set_hash(j_parts.at(i).at("hash").template get<std::string>());
- d.set_size_bytes(
- j_parts.at(i).at("size").template get<std::int64_t>());
+ parts.reserve(j.size());
+
+ for (auto const& j_part : j) {
+ auto hash = j_part.at(kHashIndex).template get<std::string>();
+ auto size = j_part.at(kSizeIndex).template get<std::size_t>();
+
+ parts.emplace_back(
+ ArtifactDigest{std::move(hash), size, /*is_tree=*/false});
}
} catch (...) {
return std::nullopt;
@@ -100,12 +104,12 @@ auto LargeObjectCAS<kDoGlobalUplink, kType>::WriteEntry(
nlohmann::json j;
try {
- j["size"] = parts.size();
- auto& j_parts = j["parts"];
for (auto const& part : parts) {
- auto& j = j_parts.emplace_back();
- j["hash"] = part.hash();
- j["size"] = part.size_bytes();
+ auto& j_part = j.emplace_back();
+
+ ArtifactDigest const a_digest(part);
+ j_part[kHashIndex] = a_digest.hash();
+ j_part[kSizeIndex] = a_digest.size();
}
} catch (...) {
return false;