summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/storage/config.hpp44
-rw-r--r--src/buildtool/storage/garbage_collector.cpp4
-rw-r--r--src/buildtool/storage/large_object_cas.hpp15
-rw-r--r--src/buildtool/storage/large_object_cas.tpp2
-rw-r--r--src/buildtool/storage/local_ac.hpp9
-rw-r--r--src/buildtool/storage/local_ac.tpp24
-rw-r--r--src/buildtool/storage/local_cas.hpp16
-rw-r--r--src/buildtool/storage/storage.hpp46
-rw-r--r--src/buildtool/storage/target_cache.hpp14
-rw-r--r--src/buildtool/storage/target_cache.tpp21
10 files changed, 113 insertions, 82 deletions
diff --git a/src/buildtool/storage/config.hpp b/src/buildtool/storage/config.hpp
index 38cdc45a..417d3e88 100644
--- a/src/buildtool/storage/config.hpp
+++ b/src/buildtool/storage/config.hpp
@@ -27,6 +27,19 @@
#include "src/utils/cpp/gsl.hpp"
#include "src/utils/cpp/tmp_dir.hpp"
+class StorageConfig;
+
+struct GenerationConfig final {
+ gsl::not_null<StorageConfig const*> const storage_config;
+ std::filesystem::path const cas_f;
+ std::filesystem::path const cas_x;
+ std::filesystem::path const cas_t;
+ std::filesystem::path const cas_large_f;
+ std::filesystem::path const cas_large_t;
+ std::filesystem::path const action_cache;
+ std::filesystem::path const target_cache;
+};
+
/// \brief Global storage configuration.
class StorageConfig {
public:
@@ -76,8 +89,7 @@ class StorageConfig {
return BuildRoot() / "git";
}
- /// \brief Root directory of specific storage generation for compatible and
- /// non-compatible protocol types.
+ /// \brief Root directory of specific storage generation
[[nodiscard]] auto GenerationCacheRoot(std::size_t index) const noexcept
-> std::filesystem::path {
ExpectsAudit(index < num_generations_);
@@ -85,15 +97,6 @@ class StorageConfig {
return CacheRoot() / generation;
}
- /// \brief Storage directory of specific generation and protocol type.
- [[nodiscard]] auto GenerationCacheDir(
- std::size_t index,
- bool is_compatible = Compatibility::IsCompatible()) const noexcept
- -> std::filesystem::path {
- return UpdatePathForCompatibility(GenerationCacheRoot(index),
- is_compatible);
- }
-
/// \brief Root directory for all ephemeral directories, i.e., directories
/// that can (and should) be removed immediately by garbage collection.
[[nodiscard]] auto EphemeralRoot() const noexcept -> std::filesystem::path {
@@ -115,6 +118,25 @@ class StorageConfig {
return TmpDir::Create(parent_path);
}
+ [[nodiscard]] auto CreateGenerationConfig(
+ std::size_t generation) const noexcept -> GenerationConfig {
+ bool const compatible = Compatibility::IsCompatible();
+ auto const cache_root = GenerationCacheRoot(generation);
+ auto const cache_dir =
+ UpdatePathForCompatibility(cache_root, compatible);
+
+ return GenerationConfig{
+ .storage_config = this,
+ .cas_f = cache_dir / "casf",
+ .cas_x = cache_dir / "casx",
+ .cas_t = cache_dir / (compatible ? "casf" : "cast"),
+ .cas_large_f = cache_dir / "cas-large-f",
+ .cas_large_t =
+ cache_dir / (compatible ? "cas-large-f" : "cas-large-t"),
+ .action_cache = cache_dir / "ac",
+ .target_cache = cache_dir / "tc"};
+ };
+
private:
// Build root directory. All the storage dirs are subdirs of build_root.
// By default, build_root is set to $HOME/.cache/just.
diff --git a/src/buildtool/storage/garbage_collector.cpp b/src/buildtool/storage/garbage_collector.cpp
index ccb8b83c..76a225c3 100644
--- a/src/buildtool/storage/garbage_collector.cpp
+++ b/src/buildtool/storage/garbage_collector.cpp
@@ -312,8 +312,8 @@ auto GarbageCollector::Compactify(StorageConfig const& storage_config,
// Compactification must be done for both native and compatible storages.
auto compactify = [&storage_config, threshold](bool compatible) -> bool {
Compatibility::SetCompatible(compatible);
- auto const storage =
- ::Generation(storage_config.GenerationCacheDir(0, compatible));
+ auto gen_config = storage_config.CreateGenerationConfig(0);
+ auto const storage = ::Generation(gen_config);
return Compactifier::RemoveInvalid(storage.CAS()) and
Compactifier::RemoveSpliced(storage.CAS()) and
diff --git a/src/buildtool/storage/large_object_cas.hpp b/src/buildtool/storage/large_object_cas.hpp
index 86c00171..e442aa90 100644
--- a/src/buildtool/storage/large_object_cas.hpp
+++ b/src/buildtool/storage/large_object_cas.hpp
@@ -69,8 +69,8 @@ class LargeObjectError final {
/// \brief Stores a temporary directory containing a result of splicing.
class LargeObject final {
public:
- LargeObject() noexcept
- : directory_(StorageConfig::Instance().CreateTypedTmpDir("splice")),
+ explicit LargeObject(StorageConfig const& storage_config) noexcept
+ : directory_(storage_config.CreateTypedTmpDir("splice")),
path_(directory_ ? directory_->GetPath() / "result" : ".") {}
/// \brief Check whether the large object is valid.
@@ -96,9 +96,13 @@ class LargeObject final {
template <bool kDoGlobalUplink, ObjectType kType>
class LargeObjectCAS final {
public:
- LargeObjectCAS(LocalCAS<kDoGlobalUplink> const& local_cas,
- std::filesystem::path const& store_path) noexcept
- : local_cas_(local_cas), file_store_(store_path) {}
+ explicit LargeObjectCAS(
+ gsl::not_null<LocalCAS<kDoGlobalUplink> const*> const& local_cas,
+ GenerationConfig const& config) noexcept
+ : local_cas_(*local_cas),
+ file_store_(IsTreeObject(kType) ? config.cas_large_t
+ : config.cas_large_f),
+ storage_config_{*config.storage_config} {}
LargeObjectCAS(LargeObjectCAS const&) = delete;
LargeObjectCAS(LargeObjectCAS&&) = delete;
@@ -167,6 +171,7 @@ class LargeObjectCAS final {
kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins;
LocalCAS<kDoGlobalUplink> const& local_cas_;
+ StorageConfig const& storage_config_;
FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false>
file_store_;
diff --git a/src/buildtool/storage/large_object_cas.tpp b/src/buildtool/storage/large_object_cas.tpp
index d216c8e8..ca8ac8c0 100644
--- a/src/buildtool/storage/large_object_cas.tpp
+++ b/src/buildtool/storage/large_object_cas.tpp
@@ -194,7 +194,7 @@ auto LargeObjectCAS<kDoGlobalUplink, kType>::Splice(
std::vector<bazel_re::Digest> const& parts) const noexcept
-> expected<LargeObject, LargeObjectError> {
// Create temporary space for splicing:
- LargeObject large_object;
+ LargeObject large_object(storage_config_);
if (not large_object.IsValid()) {
return unexpected{LargeObjectError{
LargeObjectErrorCode::Internal,
diff --git a/src/buildtool/storage/local_ac.hpp b/src/buildtool/storage/local_ac.hpp
index 41f7b7ac..98061e8a 100644
--- a/src/buildtool/storage/local_ac.hpp
+++ b/src/buildtool/storage/local_ac.hpp
@@ -22,6 +22,7 @@
#include "src/buildtool/file_system/file_storage.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/storage/config.hpp"
#include "src/buildtool/storage/garbage_collector.hpp"
#include "src/buildtool/storage/local_cas.hpp"
@@ -43,9 +44,9 @@ class LocalAC {
/// Local AC generation used by GC without global uplink.
using LocalGenerationAC = LocalAC</*kDoGlobalUplink=*/false>;
- LocalAC(std::shared_ptr<LocalCAS<kDoGlobalUplink>> cas,
- std::filesystem::path const& store_path) noexcept
- : cas_{std::move(cas)}, file_store_{store_path} {};
+ explicit LocalAC(gsl::not_null<LocalCAS<kDoGlobalUplink> const*> const& cas,
+ GenerationConfig const& config) noexcept
+ : cas_{*cas}, file_store_{config.action_cache} {};
LocalAC(LocalAC const&) = default;
LocalAC(LocalAC&&) noexcept = default;
@@ -88,7 +89,7 @@ class LocalAC {
kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins;
std::shared_ptr<Logger> logger_{std::make_shared<Logger>("LocalAC")};
- gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_;
+ LocalCAS<kDoGlobalUplink> const& cas_;
FileStorage<ObjectType::File, kStoreMode, /*kSetEpochTime=*/false>
file_store_;
diff --git a/src/buildtool/storage/local_ac.tpp b/src/buildtool/storage/local_ac.tpp
index 3f74d97e..38facf21 100644
--- a/src/buildtool/storage/local_ac.tpp
+++ b/src/buildtool/storage/local_ac.tpp
@@ -24,7 +24,7 @@ auto LocalAC<kDoGlobalUplink>::StoreResult(
bazel_re::Digest const& action_id,
bazel_re::ActionResult const& result) const noexcept -> bool {
auto bytes = result.SerializeAsString();
- auto digest = cas_->StoreBlob(bytes);
+ auto digest = cas_.StoreBlob(bytes);
return (digest and
file_store_.AddFromBytes(NativeSupport::Unprefix(action_id.hash()),
digest->SerializeAsString()));
@@ -95,14 +95,14 @@ requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry(
// Uplink result content
for (auto const& file : result->output_files()) {
- if (not cas_->LocalUplinkBlob(
- *latest.cas_, file.digest(), file.is_executable())) {
+ if (not cas_.LocalUplinkBlob(
+ latest.cas_, file.digest(), file.is_executable())) {
return false;
}
}
for (auto const& link : result->output_file_symlinks()) {
- if (not cas_->LocalUplinkBlob(
- *latest.cas_,
+ if (not cas_.LocalUplinkBlob(
+ latest.cas_,
bazel_re::Digest(
ArtifactDigest::Create<ObjectType::File>(link.target())),
/*is_executable=*/false)) {
@@ -110,8 +110,8 @@ requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry(
}
}
for (auto const& link : result->output_directory_symlinks()) {
- if (not cas_->LocalUplinkBlob(
- *latest.cas_,
+ if (not cas_.LocalUplinkBlob(
+ latest.cas_,
bazel_re::Digest(
ArtifactDigest::Create<ObjectType::File>(link.target())),
/*is_executable=*/false)) {
@@ -119,15 +119,15 @@ requires(kIsLocalGeneration) auto LocalAC<kDoGlobalUplink>::LocalUplinkEntry(
}
}
for (auto const& directory : result->output_directories()) {
- if (not cas_->LocalUplinkTree(*latest.cas_, directory.tree_digest())) {
+ if (not cas_.LocalUplinkTree(latest.cas_, directory.tree_digest())) {
return false;
}
}
// Uplink result (cache value)
- if (not cas_->LocalUplinkBlob(*latest.cas_,
- result_digest,
- /*is_executable=*/false)) {
+ if (not cas_.LocalUplinkBlob(latest.cas_,
+ result_digest,
+ /*is_executable=*/false)) {
return false;
}
@@ -140,7 +140,7 @@ template <bool kDoGlobalUplink>
auto LocalAC<kDoGlobalUplink>::ReadResult(bazel_re::Digest const& digest)
const noexcept -> std::optional<bazel_re::ActionResult> {
bazel_re::ActionResult result{};
- if (auto src_path = cas_->BlobPath(digest, /*is_executable=*/false)) {
+ if (auto src_path = cas_.BlobPath(digest, /*is_executable=*/false)) {
auto const bytes = FileSystemManager::ReadFile(*src_path);
if (bytes.has_value() and result.ParseFromString(*bytes)) {
return result;
diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp
index b29c870b..64811a62 100644
--- a/src/buildtool/storage/local_cas.hpp
+++ b/src/buildtool/storage/local_cas.hpp
@@ -23,6 +23,7 @@
#include "gsl/gsl"
#include "src/buildtool/file_system/git_repo.hpp"
#include "src/buildtool/file_system/object_cas.hpp"
+#include "src/buildtool/storage/config.hpp"
#include "src/buildtool/storage/garbage_collector.hpp"
#include "src/buildtool/storage/large_object_cas.hpp"
#include "src/utils/cpp/expected.hpp"
@@ -44,15 +45,12 @@ class LocalCAS {
/// Note that the base path is concatenated by a single character
/// 'f'/'x'/'t' for each internally used physical CAS.
/// \param base The base path for the CAS.
- explicit LocalCAS(std::filesystem::path const& base)
- : cas_file_{base.string() + 'f', Uplinker<ObjectType::File>()},
- cas_exec_{base.string() + 'x', Uplinker<ObjectType::Executable>()},
- cas_tree_{base.string() + (Compatibility::IsCompatible() ? 'f' : 't'),
- Uplinker<ObjectType::Tree>()},
- cas_file_large_{*this, base.string() + "-large-f"},
- cas_tree_large_{*this,
- base.string() + "-large-" +
- (Compatibility::IsCompatible() ? 'f' : 't')} {}
+ explicit LocalCAS(GenerationConfig const& config)
+ : cas_file_{config.cas_f, Uplinker<ObjectType::File>()},
+ cas_exec_{config.cas_x, Uplinker<ObjectType::Executable>()},
+ cas_tree_{config.cas_t, Uplinker<ObjectType::Tree>()},
+ cas_file_large_{this, config},
+ cas_tree_large_{this, config} {}
/// \brief Obtain path to the storage root.
/// \param type Type of the storage to be obtained.
diff --git a/src/buildtool/storage/storage.hpp b/src/buildtool/storage/storage.hpp
index 960699f1..ae56ef91 100644
--- a/src/buildtool/storage/storage.hpp
+++ b/src/buildtool/storage/storage.hpp
@@ -17,6 +17,7 @@
#include <cstddef>
#include <filesystem>
+#include <memory>
#include <optional>
#include <vector>
@@ -37,34 +38,34 @@
template <bool kDoGlobalUplink>
class LocalStorage {
public:
- explicit LocalStorage(std::filesystem::path const& storage_path)
- : cas_{std::make_shared<LocalCAS<kDoGlobalUplink>>(storage_path /
- "cas")},
- ac_{cas_, storage_path / "ac"},
- tc_{cas_, storage_path / "tc"} {}
+ static constexpr std::size_t kYoungest = 0U;
+
+ using CAS_t = LocalCAS<kDoGlobalUplink>;
+ using AC_t = LocalAC<kDoGlobalUplink>;
+ using TC_t = ::TargetCache<kDoGlobalUplink>;
+
+ explicit LocalStorage(GenerationConfig const& config)
+ : cas_{std::make_shared<CAS_t>(config)},
+ ac_{std::make_shared<AC_t>(&*cas_, config)},
+ tc_{std::make_shared<TC_t>(&*cas_, config)} {}
/// \brief Get the CAS instance.
- [[nodiscard]] auto CAS() const noexcept
- -> LocalCAS<kDoGlobalUplink> const& {
- return *cas_;
- }
+ [[nodiscard]] auto CAS() const noexcept -> CAS_t const& { return *cas_; }
/// \brief Get the action cache instance.
- [[nodiscard]] auto ActionCache() const noexcept
- -> LocalAC<kDoGlobalUplink> const& {
- return ac_;
+ [[nodiscard]] auto ActionCache() const noexcept -> AC_t const& {
+ return *ac_;
}
/// \brief Get the target cache instance.
- [[nodiscard]] auto TargetCache() const noexcept
- -> TargetCache<kDoGlobalUplink> const& {
- return tc_;
+ [[nodiscard]] auto TargetCache() const noexcept -> TC_t const& {
+ return *tc_;
}
private:
- gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_;
- LocalAC<kDoGlobalUplink> ac_;
- ::TargetCache<kDoGlobalUplink> tc_;
+ std::shared_ptr<CAS_t const> cas_;
+ std::shared_ptr<AC_t const> ac_;
+ std::shared_ptr<TC_t const> tc_;
};
#ifdef BOOTSTRAP_BUILD_TOOL
@@ -110,7 +111,9 @@ class Storage : public LocalStorage<kDefaultDoGlobalUplink> {
using LocalStorage<kDefaultDoGlobalUplink>::LocalStorage;
[[nodiscard]] static auto CreateStorage() noexcept -> Storage {
- return Storage{StorageConfig::Instance().GenerationCacheDir(0)};
+ auto gen_config = StorageConfig::Instance().CreateGenerationConfig(
+ Storage::kYoungest);
+ return Storage{gen_config};
}
[[nodiscard]] static auto CreateGenerations() noexcept
@@ -119,8 +122,9 @@ class Storage : public LocalStorage<kDefaultDoGlobalUplink> {
std::vector<::Generation> generations{};
generations.reserve(count);
for (std::size_t i = 0; i < count; ++i) {
- generations.emplace_back(
- StorageConfig::Instance().GenerationCacheDir(i));
+ auto gen_config =
+ StorageConfig::Instance().CreateGenerationConfig(i);
+ generations.emplace_back(gen_config);
}
return generations;
}
diff --git a/src/buildtool/storage/target_cache.hpp b/src/buildtool/storage/target_cache.hpp
index 2458b727..787a26b3 100644
--- a/src/buildtool/storage/target_cache.hpp
+++ b/src/buildtool/storage/target_cache.hpp
@@ -31,6 +31,7 @@
#include "src/buildtool/file_system/file_storage.hpp"
#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/storage/config.hpp"
#include "src/buildtool/storage/local_cas.hpp"
#include "src/buildtool/storage/target_cache_entry.hpp"
#include "src/buildtool/storage/target_cache_key.hpp"
@@ -51,15 +52,16 @@ class TargetCache {
using ArtifactDownloader =
std::function<bool(std::vector<Artifact::ObjectInfo> const&)>;
- explicit TargetCache(std::shared_ptr<LocalCAS<kDoGlobalUplink>> cas,
- std::filesystem::path const& store_path)
- : cas_{std::move(cas)},
- file_store_{store_path / ComputeShard()},
+ explicit TargetCache(
+ gsl::not_null<LocalCAS<kDoGlobalUplink> const*> const& cas,
+ GenerationConfig const& config)
+ : cas_{*cas},
+ file_store_{config.target_cache / ComputeShard()},
explicit_shard_{std::nullopt} {
if constexpr (kDoGlobalUplink) {
// write backend description (shard) to CAS
[[maybe_unused]] auto id =
- cas_->StoreBlob(RemoteExecutionConfig::DescribeBackend());
+ cas_.StoreBlob(RemoteExecutionConfig::DescribeBackend());
EnsuresAudit(id and ArtifactDigest{*id}.hash() == ComputeShard());
}
}
@@ -126,7 +128,7 @@ class TargetCache {
kDoGlobalUplink ? StoreMode::LastWins : StoreMode::FirstWins;
std::shared_ptr<Logger> logger_{std::make_shared<Logger>("TargetCache")};
- gsl::not_null<std::shared_ptr<LocalCAS<kDoGlobalUplink>>> cas_;
+ LocalCAS<kDoGlobalUplink> const& cas_;
FileStorage<ObjectType::File,
kStoreMode,
/*kSetEpochTime=*/false>
diff --git a/src/buildtool/storage/target_cache.tpp b/src/buildtool/storage/target_cache.tpp
index 7579d562..c2e89134 100644
--- a/src/buildtool/storage/target_cache.tpp
+++ b/src/buildtool/storage/target_cache.tpp
@@ -30,7 +30,7 @@ auto TargetCache<kDoGlobalUplink>::Store(
if (not DownloadKnownArtifacts(value, downloader)) {
return false;
}
- if (auto digest = cas_->StoreBlob(value.ToJson().dump(2))) {
+ if (auto digest = cas_.StoreBlob(value.ToJson().dump(2))) {
auto data =
Artifact::ObjectInfo{ArtifactDigest{*digest}, ObjectType::File}
.ToString();
@@ -57,7 +57,7 @@ auto TargetCache<kDoGlobalUplink>::ComputeKey(
nlohmann::json{target_name.module, target_name.name}.dump()},
{"effective_config", effective_config.ToString()}};
if (auto target_key =
- cas_->StoreBlob(target_desc.dump(2), /*is_executable=*/false)) {
+ cas_.StoreBlob(target_desc.dump(2), /*is_executable=*/false)) {
return TargetCacheKey{
{ArtifactDigest{*target_key}, ObjectType::File}};
}
@@ -92,7 +92,7 @@ auto TargetCache<kDoGlobalUplink>::Read(
return std::nullopt;
}
if (auto info = Artifact::ObjectInfo::FromString(*entry)) {
- if (auto path = cas_->BlobPath(info->digest, /*is_executable=*/false)) {
+ if (auto path = cas_.BlobPath(info->digest, /*is_executable=*/false)) {
if (auto value = FileSystemManager::ReadFile(*path)) {
try {
return std::make_pair(
@@ -147,7 +147,7 @@ requires(kIsLocalGeneration) auto TargetCache<kDoGlobalUplink>::
// Determine target cache entry blob path of given generation.
auto cache_entry =
- cas_->BlobPath(entry_info->digest, /*is_executable=*/false);
+ cas_.BlobPath(entry_info->digest, /*is_executable=*/false);
if (not cache_entry) {
return false;
}
@@ -182,21 +182,20 @@ requires(kIsLocalGeneration) auto TargetCache<kDoGlobalUplink>::
// Uplink referenced artifacts.
for (auto const& info : artifacts_info) {
if (info.type == ObjectType::Tree) {
- if (not cas_->LocalUplinkTree(*latest.cas_, info.digest)) {
+ if (not cas_.LocalUplinkTree(latest.cas_, info.digest)) {
return false;
}
}
- else if (not cas_->LocalUplinkBlob(*latest.cas_,
- info.digest,
- IsExecutableObject(info.type))) {
+ else if (not cas_.LocalUplinkBlob(
+ latest.cas_, info.digest, IsExecutableObject(info.type))) {
return false;
}
}
// Uplink target cache entry blob.
- if (not cas_->LocalUplinkBlob(*latest.cas_,
- entry_info->digest,
- /*is_executable=*/false)) {
+ if (not cas_.LocalUplinkBlob(latest.cas_,
+ entry_info->digest,
+ /*is_executable=*/false)) {
return false;
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-06 10:06:33 +0000