TargetCache: Initial implementation

4 files changed, 248 insertions, 0 deletions

diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS
index 3854039c..a4704623 100644
--- a/src/buildtool/build_engine/target_map/TARGETS
+++ b/src/buildtool/build_engine/target_map/TARGETS
@@ -35,6 +35,7 @@
   , "deps":
     [ "configured_target"
     , "result_map"
+    , ["src/buildtool/common", "common"]
     , ["src/buildtool/build_engine/analysed_target", "target"]
     , ["src/buildtool/build_engine/base_maps", "entity_name"]
     , ["src/buildtool/build_engine/base_maps", "field_reader"]
@@ -57,4 +58,19 @@
   , "deps": ["target_map"]
   , "stage": ["src", "buildtool", "build_engine", "target_map"]
   }
+, "target_cache":
+  { "type": ["@", "rules", "CC", "library"]
+  , "name": ["target_cache"]
+  , "hdrs": ["target_cache.hpp"]
+  , "srcs": ["target_cache.cpp"]
+  , "deps":
+    [ ["src/buildtool/build_engine/analysed_target", "target"]
+    , ["src/buildtool/build_engine/base_maps", "entity_name"]
+    , ["src/buildtool/execution_api/local", "local"]
+    , ["src/buildtool/execution_api/remote", "config"]
+    , ["src/buildtool/file_system", "file_root"]
+    , ["src/buildtool/logging", "logging"]
+    ]
+  , "stage": ["src", "buildtool", "build_engine", "target_map"]
+  }
 }
diff --git a/src/buildtool/build_engine/target_map/target_cache.cpp b/src/buildtool/build_engine/target_map/target_cache.cpp
new file mode 100644
index 00000000..aca14363
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_cache.cpp
@@ -0,0 +1,115 @@
+#include "src/buildtool/build_engine/target_map/target_cache.hpp"
+
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/execution_api/local/file_storage.hpp"
+#include "src/buildtool/execution_api/local/local_cas.hpp"
+#include "src/buildtool/execution_api/remote/config.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+auto TargetCache::Key::Create(BuildMaps::Base::EntityName const& target,
+                              Configuration const& effective_config) noexcept
+    -> std::optional<TargetCache::Key> {
+    static auto const& repos = RepositoryConfig::Instance();
+    try {
+        if (auto repo_key =
+                repos.RepositoryKey(target.GetNamedTarget().repository)) {
+            // target's repository is content-fixed, we can compute a cache key
+            auto target_desc = nlohmann::json{
+                {{"repo_key", *repo_key},
+                 {"target_name", target.ToString()},
+                 {"effective_config", effective_config.ToString()}}};
+            static auto const& cas = LocalCAS<ObjectType::File>::Instance();
+            if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) {
+                return Key{
+                    {ArtifactDigest{std::move(*target_key)}, ObjectType::File}};
+            }
+        }
+    } catch (std::exception const& ex) {
+        Logger::Log(LogLevel::Error,
+                    "Creating target cache key failed with:\n{}",
+                    ex.what());
+    }
+    return std::nullopt;
+}
+
+auto TargetCache::Entry::FromTarget(
+    AnalysedTargetPtr const& target,
+    std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
+        replacements) noexcept -> std::optional<TargetCache::Entry> {
+    auto result = TargetResult{
+        target->Artifacts(), target->Provides(), target->RunFiles()};
+    if (auto desc = result.ReplaceNonKnownAndToJson(replacements)) {
+        return Entry{*desc};
+    }
+    return std::nullopt;
+}
+
+auto TargetCache::Entry::ToResult() const -> std::optional<TargetResult> {
+    return TargetResult::FromJson(desc_);
+}
+
+auto TargetCache::Store(Key const& key, Entry const& value) const noexcept
+    -> bool {
+    if (auto digest = CAS().StoreBlobFromBytes(value.ToJson().dump(2))) {
+        auto data = Artifact::ObjectInfo{ArtifactDigest{std::move(*digest)},
+                                         ObjectType::File}
+                        .ToString();
+        logger_.Emit(LogLevel::Debug,
+                     "Adding entry for key {} as {}",
+                     key.Id().ToString(),
+                     data);
+        return file_store_.AddFromBytes(key.Id().digest.hash(), data);
+    }
+    return false;
+}
+
+auto TargetCache::Read(Key const& key) const noexcept
+    -> std::optional<std::pair<Entry, Artifact::ObjectInfo>> {
+    auto entry_path = file_store_.GetPath(key.Id().digest.hash());
+    auto const entry =
+        FileSystemManager::ReadFile(entry_path, ObjectType::File);
+    if (not entry.has_value()) {
+        logger_.Emit(LogLevel::Debug,
+                     "Cache miss, entry not found {}",
+                     entry_path.string());
+        return std::nullopt;
+    }
+    if (auto info = Artifact::ObjectInfo::FromString(*entry)) {
+        if (auto path = CAS().BlobPath(info->digest)) {
+            if (auto value = FileSystemManager::ReadFile(*path)) {
+                try {
+                    return std::make_pair(Entry{nlohmann::json::parse(*value)},
+                                          std::move(*info));
+                } catch (std::exception const& ex) {
+                    logger_.Emit(LogLevel::Warning,
+                                 "Parsing entry for key {} failed with:\n{}",
+                                 key.Id().ToString(),
+                                 ex.what());
+                }
+            }
+        }
+    }
+    logger_.Emit(LogLevel::Warning,
+                 "Reading entry for key {} failed",
+                 key.Id().ToString());
+    return std::nullopt;
+}
+
+auto TargetCache::ComputeCacheDir() -> std::filesystem::path {
+    return LocalExecutionConfig::TargetCacheDir() / ExecutionBackendId();
+}
+
+auto TargetCache::ExecutionBackendId() -> std::string {
+    auto address = RemoteExecutionConfig::RemoteAddress();
+    auto properties = RemoteExecutionConfig::PlatformProperties();
+    auto backend_desc = nlohmann::json{
+        {"remote_address",
+         address ? nlohmann::json{fmt::format(
+                       "{}:{}", address->host, address->port)}
+                 : nlohmann::json{}},
+        {"platform_properties",
+         properties}}.dump(2);
+    return CAS().StoreBlobFromBytes(backend_desc).value().hash();
+}
diff --git a/src/buildtool/build_engine/target_map/target_cache.hpp b/src/buildtool/build_engine/target_map/target_cache.hpp
new file mode 100644
index 00000000..eb31b831
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_cache.hpp
@@ -0,0 +1,111 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP
+
+#include <unordered_map>
+
+#include <nlohmann/json.hpp>
+
+#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/execution_api/local/file_storage.hpp"
+#include "src/buildtool/execution_api/local/local_cas.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+class TargetCache {
+  public:
+    // Key for target cache. Created from target name and effective config.
+    class Key {
+      public:
+        [[nodiscard]] static auto Create(
+            BuildMaps::Base::EntityName const& target,
+            Configuration const& effective_config) noexcept
+            -> std::optional<Key>;
+
+        [[nodiscard]] auto Id() const& -> Artifact::ObjectInfo const& {
+            return id_;
+        }
+        [[nodiscard]] auto Id() && -> Artifact::ObjectInfo {
+            return std::move(id_);
+        }
+        [[nodiscard]] auto operator==(Key const& other) const -> bool {
+            return this->Id() == other.Id();
+        }
+
+      private:
+        explicit Key(Artifact::ObjectInfo id) : id_{std::move(id)} {}
+        Artifact::ObjectInfo id_;
+    };
+
+    // Entry for target cache. Created from target, contains TargetResult.
+    class Entry {
+        friend class TargetCache;
+
+      public:
+        // Create the entry from target with replacement artifacts/infos.
+        // Replacement artifacts must replace all non-known artifacts by known.
+        [[nodiscard]] static auto FromTarget(
+            AnalysedTargetPtr const& target,
+            std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
+                replacements) noexcept -> std::optional<Entry>;
+
+        // Obtain TargetResult from cache entry.
+        [[nodiscard]] auto ToResult() const -> std::optional<TargetResult>;
+
+      private:
+        nlohmann::json desc_;
+
+        explicit Entry(nlohmann::json desc) : desc_{std::move(desc)} {}
+        [[nodiscard]] auto ToJson() const& -> nlohmann::json const& {
+            return desc_;
+        }
+        [[nodiscard]] auto ToJson() && -> nlohmann::json {
+            return std::move(desc_);
+        }
+    };
+
+    TargetCache() = default;
+    TargetCache(TargetCache const&) = delete;
+    TargetCache(TargetCache&&) = delete;
+    auto operator=(TargetCache const&) -> TargetCache& = delete;
+    auto operator=(TargetCache&&) -> TargetCache& = delete;
+    ~TargetCache() noexcept = default;
+
+    [[nodiscard]] static auto Instance() -> TargetCache& {
+        static TargetCache instance;
+        return instance;
+    }
+
+    // Store new key entry pair in the target cache.
+    [[nodiscard]] auto Store(Key const& key, Entry const& value) const noexcept
+        -> bool;
+
+    // Read existing entry and object info for given key from the target cache.
+    [[nodiscard]] auto Read(Key const& key) const noexcept
+        -> std::optional<std::pair<Entry, Artifact::ObjectInfo>>;
+
+  private:
+    Logger logger_{"TargetCache"};
+    FileStorage<ObjectType::File,
+                StoreMode::LastWins,
+                /*kSetEpochTime=*/false>
+        file_store_{ComputeCacheDir()};
+
+    [[nodiscard]] static auto CAS() noexcept -> LocalCAS<ObjectType::File>& {
+        return LocalCAS<ObjectType::File>::Instance();
+    }
+    [[nodiscard]] static auto ComputeCacheDir() -> std::filesystem::path;
+    [[nodiscard]] static auto ExecutionBackendId() -> std::string;
+};
+
+namespace std {
+template <>
+struct hash<TargetCache::Key> {
+    [[nodiscard]] auto operator()(TargetCache::Key const& k) const {
+        return std::hash<Artifact::ObjectInfo>{}(k.Id());
+    }
+};
+}  // namespace std
+
+#endif  // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP
diff --git a/src/buildtool/execution_api/local/config.hpp b/src/buildtool/execution_api/local/config.hpp
index 2dc4466b..5ce8b9ef 100644
--- a/src/buildtool/execution_api/local/config.hpp
+++ b/src/buildtool/execution_api/local/config.hpp
@@ -123,6 +123,12 @@ class LocalExecutionConfig {
         return CacheRoot() / "ac";
     }
 
+    /// \brief Target cache directory
+    [[nodiscard]] static auto TargetCacheDir() noexcept
+        -> std::filesystem::path {
+        return CacheRoot() / "tc";
+    }
+
     [[nodiscard]] static auto GetLauncher() noexcept
         -> std::vector<std::string> {
         return Data().launcher;