diff options
author | Maksim Denisov <denisov.maksim@huawei.com> | 2024-07-23 14:07:48 +0200 |
---|---|---|
committer | Maksim Denisov <denisov.maksim@huawei.com> | 2024-08-26 17:02:14 +0200 |
commit | 7a4bf22ac5330024b4b807c9f7cf42292a5135d1 (patch) | |
tree | cdfe1a62167f993c1bbcf61aa09cbfcdaae9d58b | |
parent | 3966dab59e45e3cbacb52781ef74e73f6db2b142 (diff) | |
download | justbuild-7a4bf22ac5330024b4b807c9f7cf42292a5135d1.tar.gz |
Reduce the size of large object entries.
Removed named keys of entries (hash, size). These prefixes were added for every chunk creating an additional overhead without any benefit.
Removed prefixes of hashes (*62*hash) appearing in native mode and creating an additional overhead as well: it is known that all of them are blobs.
-rw-r--r-- | CHANGELOG.md | 3 | ||||
-rw-r--r-- | src/buildtool/storage/large_object_cas.tpp | 32 |
2 files changed, 21 insertions, 14 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c53b23a..c120f104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,9 @@ A feature release on top of `1.3.0`, backwards compatible. - Local execution no longer has the requirement that there exist no more files with identical content than the hardlink limit of the underlying file system. +- The size of large object entries has been reduced. The cache and + CAS must be cleaned up since stable versions before `1.4.0` cannot use + the new format. - Various improvements to the tests: dispatching of the summary action is now possible, tests are independent of a .just-mrrc file the user might have in their home directory diff --git a/src/buildtool/storage/large_object_cas.tpp b/src/buildtool/storage/large_object_cas.tpp index 0f79b231..dae1d696 100644 --- a/src/buildtool/storage/large_object_cas.tpp +++ b/src/buildtool/storage/large_object_cas.tpp @@ -29,6 +29,11 @@ #include "src/buildtool/storage/large_object_cas.hpp" #include "src/buildtool/storage/local_cas.hpp" +namespace { +inline constexpr std::size_t kHashIndex = 0; +inline constexpr std::size_t kSizeIndex = 1; +} // namespace + template <bool kDoGlobalUplink, ObjectType kType> auto LargeObjectCAS<kDoGlobalUplink, kType>::GetEntryPath( bazel_re::Digest const& digest) const noexcept @@ -66,15 +71,14 @@ auto LargeObjectCAS<kDoGlobalUplink, kType>::ReadEntry( try { std::ifstream stream(*file_path); nlohmann::json j = nlohmann::json::parse(stream); - const std::size_t size = j.at("size").template get<std::size_t>(); - parts.reserve(size); - - auto const& j_parts = j.at("parts"); - for (std::size_t i = 0; i < size; ++i) { - bazel_re::Digest& d = parts.emplace_back(); - d.set_hash(j_parts.at(i).at("hash").template get<std::string>()); - d.set_size_bytes( - j_parts.at(i).at("size").template get<std::int64_t>()); + parts.reserve(j.size()); + + for (auto const& j_part : j) { + auto hash = j_part.at(kHashIndex).template get<std::string>(); + auto size = j_part.at(kSizeIndex).template get<std::size_t>(); + + parts.emplace_back( + ArtifactDigest{std::move(hash), size, /*is_tree=*/false}); } } catch (...) { return std::nullopt; @@ -100,12 +104,12 @@ auto LargeObjectCAS<kDoGlobalUplink, kType>::WriteEntry( nlohmann::json j; try { - j["size"] = parts.size(); - auto& j_parts = j["parts"]; for (auto const& part : parts) { - auto& j = j_parts.emplace_back(); - j["hash"] = part.hash(); - j["size"] = part.size_bytes(); + auto& j_part = j.emplace_back(); + + ArtifactDigest const a_digest(part); + j_part[kHashIndex] = a_digest.hash(); + j_part[kSizeIndex] = a_digest.size(); } } catch (...) { return false; |