summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/build_engine/target_map/TARGETS16
-rw-r--r--src/buildtool/build_engine/target_map/target_cache.cpp115
-rw-r--r--src/buildtool/build_engine/target_map/target_cache.hpp111
-rw-r--r--src/buildtool/execution_api/local/config.hpp6
4 files changed, 248 insertions, 0 deletions
diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS
index 3854039c..a4704623 100644
--- a/src/buildtool/build_engine/target_map/TARGETS
+++ b/src/buildtool/build_engine/target_map/TARGETS
@@ -35,6 +35,7 @@
, "deps":
[ "configured_target"
, "result_map"
+ , ["src/buildtool/common", "common"]
, ["src/buildtool/build_engine/analysed_target", "target"]
, ["src/buildtool/build_engine/base_maps", "entity_name"]
, ["src/buildtool/build_engine/base_maps", "field_reader"]
@@ -57,4 +58,19 @@
, "deps": ["target_map"]
, "stage": ["src", "buildtool", "build_engine", "target_map"]
}
+, "target_cache":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["target_cache"]
+ , "hdrs": ["target_cache.hpp"]
+ , "srcs": ["target_cache.cpp"]
+ , "deps":
+ [ ["src/buildtool/build_engine/analysed_target", "target"]
+ , ["src/buildtool/build_engine/base_maps", "entity_name"]
+ , ["src/buildtool/execution_api/local", "local"]
+ , ["src/buildtool/execution_api/remote", "config"]
+ , ["src/buildtool/file_system", "file_root"]
+ , ["src/buildtool/logging", "logging"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "target_map"]
+ }
}
diff --git a/src/buildtool/build_engine/target_map/target_cache.cpp b/src/buildtool/build_engine/target_map/target_cache.cpp
new file mode 100644
index 00000000..aca14363
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_cache.cpp
@@ -0,0 +1,115 @@
+#include "src/buildtool/build_engine/target_map/target_cache.hpp"
+
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/execution_api/local/file_storage.hpp"
+#include "src/buildtool/execution_api/local/local_cas.hpp"
+#include "src/buildtool/execution_api/remote/config.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+auto TargetCache::Key::Create(BuildMaps::Base::EntityName const& target,
+ Configuration const& effective_config) noexcept
+ -> std::optional<TargetCache::Key> {
+ static auto const& repos = RepositoryConfig::Instance();
+ try {
+ if (auto repo_key =
+ repos.RepositoryKey(target.GetNamedTarget().repository)) {
+ // target's repository is content-fixed, we can compute a cache key
+ auto target_desc = nlohmann::json{
+ {{"repo_key", *repo_key},
+ {"target_name", target.ToString()},
+ {"effective_config", effective_config.ToString()}}};
+ static auto const& cas = LocalCAS<ObjectType::File>::Instance();
+ if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) {
+ return Key{
+ {ArtifactDigest{std::move(*target_key)}, ObjectType::File}};
+ }
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "Creating target cache key failed with:\n{}",
+ ex.what());
+ }
+ return std::nullopt;
+}
+
+auto TargetCache::Entry::FromTarget(
+ AnalysedTargetPtr const& target,
+ std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
+ replacements) noexcept -> std::optional<TargetCache::Entry> {
+ auto result = TargetResult{
+ target->Artifacts(), target->Provides(), target->RunFiles()};
+ if (auto desc = result.ReplaceNonKnownAndToJson(replacements)) {
+ return Entry{*desc};
+ }
+ return std::nullopt;
+}
+
+auto TargetCache::Entry::ToResult() const -> std::optional<TargetResult> {
+ return TargetResult::FromJson(desc_);
+}
+
+auto TargetCache::Store(Key const& key, Entry const& value) const noexcept
+ -> bool {
+ if (auto digest = CAS().StoreBlobFromBytes(value.ToJson().dump(2))) {
+ auto data = Artifact::ObjectInfo{ArtifactDigest{std::move(*digest)},
+ ObjectType::File}
+ .ToString();
+ logger_.Emit(LogLevel::Debug,
+ "Adding entry for key {} as {}",
+ key.Id().ToString(),
+ data);
+ return file_store_.AddFromBytes(key.Id().digest.hash(), data);
+ }
+ return false;
+}
+
+auto TargetCache::Read(Key const& key) const noexcept
+ -> std::optional<std::pair<Entry, Artifact::ObjectInfo>> {
+ auto entry_path = file_store_.GetPath(key.Id().digest.hash());
+ auto const entry =
+ FileSystemManager::ReadFile(entry_path, ObjectType::File);
+ if (not entry.has_value()) {
+ logger_.Emit(LogLevel::Debug,
+ "Cache miss, entry not found {}",
+ entry_path.string());
+ return std::nullopt;
+ }
+ if (auto info = Artifact::ObjectInfo::FromString(*entry)) {
+ if (auto path = CAS().BlobPath(info->digest)) {
+ if (auto value = FileSystemManager::ReadFile(*path)) {
+ try {
+ return std::make_pair(Entry{nlohmann::json::parse(*value)},
+ std::move(*info));
+ } catch (std::exception const& ex) {
+ logger_.Emit(LogLevel::Warning,
+ "Parsing entry for key {} failed with:\n{}",
+ key.Id().ToString(),
+ ex.what());
+ }
+ }
+ }
+ }
+ logger_.Emit(LogLevel::Warning,
+ "Reading entry for key {} failed",
+ key.Id().ToString());
+ return std::nullopt;
+}
+
+auto TargetCache::ComputeCacheDir() -> std::filesystem::path {
+ return LocalExecutionConfig::TargetCacheDir() / ExecutionBackendId();
+}
+
+auto TargetCache::ExecutionBackendId() -> std::string {
+ auto address = RemoteExecutionConfig::RemoteAddress();
+ auto properties = RemoteExecutionConfig::PlatformProperties();
+ auto backend_desc = nlohmann::json{
+ {"remote_address",
+ address ? nlohmann::json{fmt::format(
+ "{}:{}", address->host, address->port)}
+ : nlohmann::json{}},
+ {"platform_properties",
+ properties}}.dump(2);
+ return CAS().StoreBlobFromBytes(backend_desc).value().hash();
+}
diff --git a/src/buildtool/build_engine/target_map/target_cache.hpp b/src/buildtool/build_engine/target_map/target_cache.hpp
new file mode 100644
index 00000000..eb31b831
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_cache.hpp
@@ -0,0 +1,111 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP
+
+#include <unordered_map>
+
+#include <nlohmann/json.hpp>
+
+#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/execution_api/local/file_storage.hpp"
+#include "src/buildtool/execution_api/local/local_cas.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+class TargetCache {
+ public:
+ // Key for target cache. Created from target name and effective config.
+ class Key {
+ public:
+ [[nodiscard]] static auto Create(
+ BuildMaps::Base::EntityName const& target,
+ Configuration const& effective_config) noexcept
+ -> std::optional<Key>;
+
+ [[nodiscard]] auto Id() const& -> Artifact::ObjectInfo const& {
+ return id_;
+ }
+ [[nodiscard]] auto Id() && -> Artifact::ObjectInfo {
+ return std::move(id_);
+ }
+ [[nodiscard]] auto operator==(Key const& other) const -> bool {
+ return this->Id() == other.Id();
+ }
+
+ private:
+ explicit Key(Artifact::ObjectInfo id) : id_{std::move(id)} {}
+ Artifact::ObjectInfo id_;
+ };
+
+ // Entry for target cache. Created from target, contains TargetResult.
+ class Entry {
+ friend class TargetCache;
+
+ public:
+ // Create the entry from target with replacement artifacts/infos.
+ // Replacement artifacts must replace all non-known artifacts by known.
+ [[nodiscard]] static auto FromTarget(
+ AnalysedTargetPtr const& target,
+ std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
+ replacements) noexcept -> std::optional<Entry>;
+
+ // Obtain TargetResult from cache entry.
+ [[nodiscard]] auto ToResult() const -> std::optional<TargetResult>;
+
+ private:
+ nlohmann::json desc_;
+
+ explicit Entry(nlohmann::json desc) : desc_{std::move(desc)} {}
+ [[nodiscard]] auto ToJson() const& -> nlohmann::json const& {
+ return desc_;
+ }
+ [[nodiscard]] auto ToJson() && -> nlohmann::json {
+ return std::move(desc_);
+ }
+ };
+
+ TargetCache() = default;
+ TargetCache(TargetCache const&) = delete;
+ TargetCache(TargetCache&&) = delete;
+ auto operator=(TargetCache const&) -> TargetCache& = delete;
+ auto operator=(TargetCache&&) -> TargetCache& = delete;
+ ~TargetCache() noexcept = default;
+
+ [[nodiscard]] static auto Instance() -> TargetCache& {
+ static TargetCache instance;
+ return instance;
+ }
+
+ // Store new key entry pair in the target cache.
+ [[nodiscard]] auto Store(Key const& key, Entry const& value) const noexcept
+ -> bool;
+
+ // Read existing entry and object info for given key from the target cache.
+ [[nodiscard]] auto Read(Key const& key) const noexcept
+ -> std::optional<std::pair<Entry, Artifact::ObjectInfo>>;
+
+ private:
+ Logger logger_{"TargetCache"};
+ FileStorage<ObjectType::File,
+ StoreMode::LastWins,
+ /*kSetEpochTime=*/false>
+ file_store_{ComputeCacheDir()};
+
+ [[nodiscard]] static auto CAS() noexcept -> LocalCAS<ObjectType::File>& {
+ return LocalCAS<ObjectType::File>::Instance();
+ }
+ [[nodiscard]] static auto ComputeCacheDir() -> std::filesystem::path;
+ [[nodiscard]] static auto ExecutionBackendId() -> std::string;
+};
+
+namespace std {
+template <>
+struct hash<TargetCache::Key> {
+ [[nodiscard]] auto operator()(TargetCache::Key const& k) const {
+ return std::hash<Artifact::ObjectInfo>{}(k.Id());
+ }
+};
+} // namespace std
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP
diff --git a/src/buildtool/execution_api/local/config.hpp b/src/buildtool/execution_api/local/config.hpp
index 2dc4466b..5ce8b9ef 100644
--- a/src/buildtool/execution_api/local/config.hpp
+++ b/src/buildtool/execution_api/local/config.hpp
@@ -123,6 +123,12 @@ class LocalExecutionConfig {
return CacheRoot() / "ac";
}
+ /// \brief Target cache directory
+ [[nodiscard]] static auto TargetCacheDir() noexcept
+ -> std::filesystem::path {
+ return CacheRoot() / "tc";
+ }
+
[[nodiscard]] static auto GetLauncher() noexcept
-> std::vector<std::string> {
return Data().launcher;
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 11:00:06 +0000