diff options
author | Alberto Sartori <alberto.sartori@huawei.com> | 2023-07-13 13:20:59 +0200 |
---|---|---|
committer | Alberto Sartori <alberto.sartori@huawei.com> | 2023-07-13 14:37:46 +0200 |
commit | ba28beba51ff7d495e5165471b96f7496bf585c2 (patch) | |
tree | afb7fbcc344541203fdee384f18e745561565d36 | |
parent | 1802fcd90b48900bd0e46102b392146297a5ebbb (diff) | |
download | justbuild-ba28beba51ff7d495e5165471b96f7496bf585c2.tar.gz |
bytestream_server: store temporary files under the local-buil-root
The bytestream server implementation (deployed by just execute) now
stores the temporary files under
$local_build_root/protocol-depenedent/generation-0
so that they can be garbage collected if "just exectue" is terminated
before they are cleaned up.
-rw-r--r-- | CHANGELOG.md | 2 | ||||
-rw-r--r-- | src/buildtool/execution_api/execution_service/bytestream_server.cpp | 20 | ||||
-rw-r--r-- | src/buildtool/storage/local_cas.hpp | 4 |
3 files changed, 16 insertions, 10 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index ffa17d97..761fe55c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ A feature release on top of `1.1.0`, backwards compatible. all entries in a Git repository. Previously it was possible for entries to be skipped inadvertently in, e.g., imported archives if `gitignore` files were present. +- Temporary files generated by `just execute` are now created inside + the local build root. ## Release `1.1.0` (2023-05-19) diff --git a/src/buildtool/execution_api/execution_service/bytestream_server.cpp b/src/buildtool/execution_api/execution_service/bytestream_server.cpp index 234c4adc..b450c697 100644 --- a/src/buildtool/execution_api/execution_service/bytestream_server.cpp +++ b/src/buildtool/execution_api/execution_service/bytestream_server.cpp @@ -113,7 +113,14 @@ auto BytestreamServiceImpl::Write( *hash, request.write_offset(), request.finish_write()); - auto tmp_dir = TmpDir::Create("execution-service"); + auto lock = GarbageCollector::SharedLock(); + if (!lock) { + auto str = fmt::format("Could not acquire SharedLock"); + logger_.Emit(LogLevel::Error, str); + return grpc::Status{grpc::StatusCode::INTERNAL, str}; + } + auto tmp_dir = TmpDir::Create(StorageConfig::GenerationCacheRoot(0) / + "execution-service"); if (!tmp_dir) { return ::grpc::Status{::grpc::StatusCode::INTERNAL, "could not create TmpDir"}; @@ -129,21 +136,16 @@ auto BytestreamServiceImpl::Write( of.write(request.data().data(), static_cast<std::streamsize>(request.data().size())); } - auto lock = GarbageCollector::SharedLock(); - if (!lock) { - auto str = fmt::format("Could not acquire SharedLock"); - logger_.Emit(LogLevel::Error, str); - return grpc::Status{grpc::StatusCode::INTERNAL, str}; - } if (NativeSupport::IsTree(*hash)) { - if (not storage_->CAS().StoreTree(tmp)) { + if (not storage_->CAS().StoreTree</*kOwner=*/true>(tmp)) { auto str = fmt::format("could not store tree {}", *hash); logger_.Emit(LogLevel::Error, str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; } } else { - if (not storage_->CAS().StoreBlob(tmp, /*is_executable=*/false)) { + if (not storage_->CAS().StoreBlob</*kOwner=*/true>( + tmp, /*is_executable=*/false)) { auto str = fmt::format("could not store blob {}", *hash); logger_.Emit(LogLevel::Error, str); return ::grpc::Status{::grpc::StatusCode::INVALID_ARGUMENT, str}; diff --git a/src/buildtool/storage/local_cas.hpp b/src/buildtool/storage/local_cas.hpp index af613b1d..4726f8b8 100644 --- a/src/buildtool/storage/local_cas.hpp +++ b/src/buildtool/storage/local_cas.hpp @@ -70,11 +70,13 @@ class LocalCAS { } /// \brief Store tree from file path. + /// \tparam kOwner Indicates ownership for optimization (hardlink). /// \param file_path The path of the file to store as tree. /// \returns Digest of the stored tree or nullopt otherwise. + template <bool kOwner = false> [[nodiscard]] auto StoreTree(std::filesystem::path const& file_path) const noexcept -> std::optional<bazel_re::Digest> { - return cas_tree_.StoreBlobFromFile(file_path); + return cas_tree_.StoreBlobFromFile(file_path, kOwner); } /// \brief Store tree from bytes. |