summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSascha Roloff <sascha.roloff@huawei.com>2022-12-21 16:24:27 +0100
committerKlaus Aehlig <klaus.aehlig@huawei.com>2023-01-20 13:57:18 +0100
commitee549fa1ca1ff4a075c9b2634808525b014818ea (patch)
treece709bd351412e73766340439b991a5215e69d1e
parent56828d241a618c94f0526b5fd498b7f3ce540230 (diff)
downloadjustbuild-ee549fa1ca1ff4a075c9b2634808525b014818ea.tar.gz
Refactor target cache key and entry to separate files and targets
-rw-r--r--src/buildtool/build_engine/target_map/TARGETS55
-rw-r--r--src/buildtool/build_engine/target_map/export.cpp5
-rw-r--r--src/buildtool/build_engine/target_map/result_map.hpp9
-rw-r--r--src/buildtool/build_engine/target_map/target_cache.cpp134
-rw-r--r--src/buildtool/build_engine/target_map/target_cache.hpp95
-rw-r--r--src/buildtool/build_engine/target_map/target_cache_entry.cpp101
-rw-r--r--src/buildtool/build_engine/target_map/target_cache_entry.hpp64
-rw-r--r--src/buildtool/build_engine/target_map/target_cache_key.cpp53
-rw-r--r--src/buildtool/build_engine/target_map/target_cache_key.hpp48
-rw-r--r--src/buildtool/main/TARGETS2
-rw-r--r--src/buildtool/main/main.cpp11
11 files changed, 368 insertions, 209 deletions
diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS
index f7bd8cb3..ee65b3e3 100644
--- a/src/buildtool/build_engine/target_map/TARGETS
+++ b/src/buildtool/build_engine/target_map/TARGETS
@@ -52,6 +52,7 @@
, "stage": ["src", "buildtool", "build_engine", "target_map"]
, "private-deps":
[ "target_cache"
+ , "target_cache_key"
, ["src/buildtool/common", "common"]
, ["src/buildtool/build_engine/base_maps", "entity_name"]
, ["src/buildtool/build_engine/base_maps", "field_reader"]
@@ -85,18 +86,62 @@
, "hdrs": ["target_cache.hpp"]
, "srcs": ["target_cache.cpp"]
, "deps":
- [ ["src/buildtool/build_engine/analysed_target", "target"]
- , ["src/buildtool/build_engine/base_maps", "entity_name"]
+ [ "target_cache_key"
+ , "target_cache_entry"
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["@", "json", "", "json"]
+ , ["src/buildtool/common", "common"]
, ["src/buildtool/execution_api/local", "local"]
+ , ["src/buildtool/file_system", "object_type"]
, ["src/buildtool/logging", "logging"]
- , ["src/buildtool/common", "common"]
, ["src/buildtool/execution_api/common", "common"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "target_map"]
+ , "private-deps":
+ [ ["@", "fmt", "", "fmt"]
+ , ["src/buildtool/compatibility", "compatibility"]
+ , ["src/buildtool/execution_api/local", "config"]
+ , ["src/buildtool/execution_api/remote", "config"]
, ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/logging", "log_level"]
+ ]
+ }
+, "target_cache_key":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["target_cache_key"]
+ , "hdrs": ["target_cache_key.hpp"]
+ , "srcs": ["target_cache_key.cpp"]
+ , "deps":
+ [ ["src/buildtool/build_engine/base_maps", "entity_name_data"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/common", "common"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "target_map"]
+ , "private-deps":
+ [ ["src/buildtool/common", "config"]
+ , ["src/buildtool/execution_api/local", "local"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/logging", "log_level"]
+ ]
+ }
+, "target_cache_entry":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["target_cache_entry"]
+ , "hdrs": ["target_cache_entry.hpp"]
+ , "srcs": ["target_cache_entry.cpp"]
+ , "deps":
+ [ ["@", "gsl-lite", "", "gsl-lite"]
+ , ["@", "json", "", "json"]
+ , ["src/buildtool/build_engine/analysed_target", "target"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/common", "artifact_description"]
]
, "stage": ["src", "buildtool", "build_engine", "target_map"]
, "private-deps":
- [ ["src/buildtool/execution_api/remote", "config"]
- , ["src/buildtool/file_system", "file_root"]
+ [ ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/logging", "log_level"]
]
}
}
diff --git a/src/buildtool/build_engine/target_map/export.cpp b/src/buildtool/build_engine/target_map/export.cpp
index 657ca73b..7a53e78e 100644
--- a/src/buildtool/build_engine/target_map/export.cpp
+++ b/src/buildtool/build_engine/target_map/export.cpp
@@ -19,6 +19,7 @@
#include "src/buildtool/build_engine/base_maps/field_reader.hpp"
#include "src/buildtool/build_engine/expression/configuration.hpp"
#include "src/buildtool/build_engine/target_map/target_cache.hpp"
+#include "src/buildtool/build_engine/target_map/target_cache_key.hpp"
#include "src/buildtool/common/statistics.hpp"
#include "src/buildtool/execution_api/local/local_cas.hpp"
@@ -35,7 +36,7 @@ void FinalizeExport(
const BuildMaps::Base::EntityName& target,
const std::vector<std::string>& vars,
const Configuration& effective_config,
- const std::optional<TargetCache::Key>& target_cache_key,
+ const std::optional<TargetCacheKey>& target_cache_key,
const BuildMaps::Target::TargetMap::LoggerPtr& logger,
const BuildMaps::Target::TargetMap::SetterPtr& setter,
const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) {
@@ -129,7 +130,7 @@ void ExportRule(
auto target_config = effective_config.Update(fixed_config);
auto target_cache_key =
- TargetCache::Key::Create(*exported_target, target_config);
+ TargetCacheKey::Create(*exported_target, target_config);
if (target_cache_key) {
if (auto target_cache_value =
TargetCache::Instance().Read(*target_cache_key)) {
diff --git a/src/buildtool/build_engine/target_map/result_map.hpp b/src/buildtool/build_engine/target_map/result_map.hpp
index 700cf7bb..edb1a472 100644
--- a/src/buildtool/build_engine/target_map/result_map.hpp
+++ b/src/buildtool/build_engine/target_map/result_map.hpp
@@ -31,6 +31,7 @@
#include "src/buildtool/build_engine/expression/expression.hpp"
#include "src/buildtool/build_engine/target_map/configured_target.hpp"
#include "src/buildtool/build_engine/target_map/target_cache.hpp"
+#include "src/buildtool/build_engine/target_map/target_cache_key.hpp"
#include "src/buildtool/common/statistics.hpp"
#include "src/buildtool/common/tree.hpp"
#include "src/buildtool/logging/logger.hpp"
@@ -68,7 +69,7 @@ class ResultTargetMap {
BuildMaps::Base::EntityName name,
Configuration conf,
gsl::not_null<AnalysedTargetPtr> result,
- std::optional<TargetCache::Key> target_cache_key = std::nullopt,
+ std::optional<TargetCacheKey> target_cache_key = std::nullopt,
bool is_export_target = false) -> AnalysedTargetPtr {
auto part = std::hash<BuildMaps::Base::EntityName>{}(name) % width_;
std::unique_lock lock{m_[part]};
@@ -142,11 +143,11 @@ class ResultTargetMap {
}
[[nodiscard]] auto CacheTargets() const noexcept
- -> std::unordered_map<TargetCache::Key, AnalysedTargetPtr> {
+ -> std::unordered_map<TargetCacheKey, AnalysedTargetPtr> {
return std::accumulate(
cache_targets_.begin(),
cache_targets_.end(),
- std::unordered_map<TargetCache::Key, AnalysedTargetPtr>{},
+ std::unordered_map<TargetCacheKey, AnalysedTargetPtr>{},
[](auto&& l, auto const& r) {
l.insert(r.begin(), r.end());
return std::forward<decltype(l)>(l);
@@ -361,7 +362,7 @@ class ResultTargetMap {
std::unordered_map<ConfiguredTarget, gsl::not_null<AnalysedTargetPtr>>>
targets_{width_};
std::vector<
- std::unordered_map<TargetCache::Key, gsl::not_null<AnalysedTargetPtr>>>
+ std::unordered_map<TargetCacheKey, gsl::not_null<AnalysedTargetPtr>>>
cache_targets_{width_};
std::vector<std::unordered_set<ConfiguredTarget>> export_targets_{width_};
std::vector<std::size_t> num_actions_{std::vector<std::size_t>(width_)};
diff --git a/src/buildtool/build_engine/target_map/target_cache.cpp b/src/buildtool/build_engine/target_map/target_cache.cpp
index b2f3b209..727b7404 100644
--- a/src/buildtool/build_engine/target_map/target_cache.cpp
+++ b/src/buildtool/build_engine/target_map/target_cache.cpp
@@ -14,123 +14,20 @@
#include "src/buildtool/build_engine/target_map/target_cache.hpp"
-#include <gsl-lite/gsl-lite.hpp>
+#include <exception>
+#include <vector>
-#include "src/buildtool/common/repository_config.hpp"
+#include <fmt/core.h>
+
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/compatibility/native_support.hpp"
#include "src/buildtool/execution_api/local/config.hpp"
-#include "src/buildtool/execution_api/local/file_storage.hpp"
-#include "src/buildtool/execution_api/local/local_cas.hpp"
#include "src/buildtool/execution_api/remote/config.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
-#include "src/buildtool/logging/logger.hpp"
-
-auto TargetCache::Key::Create(BuildMaps::Base::EntityName const& target,
- Configuration const& effective_config) noexcept
- -> std::optional<TargetCache::Key> {
- static auto const& repos = RepositoryConfig::Instance();
- try {
- if (auto repo_key =
- repos.RepositoryKey(target.GetNamedTarget().repository)) {
- // target's repository is content-fixed, we can compute a cache key
- auto const& name = target.GetNamedTarget();
- auto target_desc = nlohmann::json{
- {{"repo_key", *repo_key},
- {"target_name", nlohmann::json{name.module, name.name}.dump()},
- {"effective_config", effective_config.ToString()}}};
- static auto const& cas = LocalCAS<ObjectType::File>::Instance();
- if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) {
- return Key{{ArtifactDigest{*target_key}, ObjectType::File}};
- }
- }
- } catch (std::exception const& ex) {
- Logger::Log(LogLevel::Error,
- "Creating target cache key failed with:\n{}",
- ex.what());
- }
- return std::nullopt;
-}
-
-auto TargetCache::Entry::FromTarget(
- AnalysedTargetPtr const& target,
- std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
- replacements) noexcept -> std::optional<TargetCache::Entry> {
- auto result = TargetResult{
- target->Artifacts(), target->Provides(), target->RunFiles()};
- if (auto desc = result.ReplaceNonKnownAndToJson(replacements)) {
- return Entry{*desc};
- }
- return std::nullopt;
-}
-
-auto TargetCache::Entry::ToResult() const noexcept
- -> std::optional<TargetResult> {
- return TargetResult::FromJson(desc_);
-}
-
-[[nodiscard]] auto ToObjectInfo(nlohmann::json const& json)
- -> Artifact::ObjectInfo {
- auto const& desc = ArtifactDescription::FromJson(json);
- // The assumption is that all artifacts mentioned in a target cache
- // entry are KNOWN to the remote side.
- gsl_ExpectsAudit(desc and desc->IsKnown());
- auto const& info = desc->ToArtifact().Info();
- gsl_ExpectsAudit(info);
- return *info;
-}
-
-[[nodiscard]] auto ScanArtifactMap(
- gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos,
- nlohmann::json const& json) -> bool {
- if (not json.is_object()) {
- return false;
- }
- infos->reserve(infos->size() + json.size());
- std::transform(json.begin(),
- json.end(),
- std::back_inserter(*infos),
- [](auto const& item) { return ToObjectInfo(item); });
- return true;
-}
-
-[[nodiscard]] auto ScanProvidesMap(
- gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos,
- nlohmann::json const& json) -> bool {
- if (not json.is_object()) {
- return false;
- }
- auto const& nodes = json["nodes"];
- auto const& provided_artifacts = json["provided_artifacts"];
- infos->reserve(infos->size() + provided_artifacts.size());
- std::transform(
- provided_artifacts.begin(),
- provided_artifacts.end(),
- std::back_inserter(*infos),
- [&nodes](auto const& item) {
- return ToObjectInfo(nodes[item.template get<std::string>()]);
- });
- return true;
-}
-
-auto TargetCache::Entry::ToArtifacts(
- gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos)
- const noexcept -> bool {
- try {
- if (ScanArtifactMap(infos, desc_["artifacts"]) and
- ScanArtifactMap(infos, desc_["runfiles"]) and
- ScanProvidesMap(infos, desc_["provides"])) {
- return true;
- }
- } catch (std::exception const& ex) {
- Logger::Log(
- LogLevel::Error,
- "Scanning target cache entry for artifacts failed with:\n{}",
- ex.what());
- }
- return false;
-}
+#include "src/buildtool/logging/log_level.hpp"
-auto TargetCache::Store(Key const& key, Entry const& value) const noexcept
- -> bool {
+auto TargetCache::Store(TargetCacheKey const& key,
+ TargetCacheEntry const& value) const noexcept -> bool {
// Before a target-cache entry is stored in local CAS, make sure any created
// artifact for this target is downloaded from the remote CAS to the local
// CAS.
@@ -150,8 +47,8 @@ auto TargetCache::Store(Key const& key, Entry const& value) const noexcept
return false;
}
-auto TargetCache::Read(Key const& key) const noexcept
- -> std::optional<std::pair<Entry, Artifact::ObjectInfo>> {
+auto TargetCache::Read(TargetCacheKey const& key) const noexcept
+ -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> {
auto entry_path = file_store_.GetPath(key.Id().digest.hash());
auto const entry =
FileSystemManager::ReadFile(entry_path, ObjectType::File);
@@ -165,8 +62,9 @@ auto TargetCache::Read(Key const& key) const noexcept
if (auto path = CAS().BlobPath(info->digest)) {
if (auto value = FileSystemManager::ReadFile(*path)) {
try {
- return std::make_pair(Entry{nlohmann::json::parse(*value)},
- std::move(*info));
+ return std::make_pair(
+ TargetCacheEntry{nlohmann::json::parse(*value)},
+ std::move(*info));
} catch (std::exception const& ex) {
logger_.Emit(LogLevel::Warning,
"Parsing entry for key {} failed with:\n{}",
@@ -182,8 +80,8 @@ auto TargetCache::Read(Key const& key) const noexcept
return std::nullopt;
}
-auto TargetCache::DownloadKnownArtifacts(Entry const& value) const noexcept
- -> bool {
+auto TargetCache::DownloadKnownArtifacts(
+ TargetCacheEntry const& value) const noexcept -> bool {
std::vector<Artifact::ObjectInfo> artifacts_info;
if (not value.ToArtifacts(&artifacts_info)) {
return false;
diff --git a/src/buildtool/build_engine/target_map/target_cache.hpp b/src/buildtool/build_engine/target_map/target_cache.hpp
index fe32d3dd..b6b54ce0 100644
--- a/src/buildtool/build_engine/target_map/target_cache.hpp
+++ b/src/buildtool/build_engine/target_map/target_cache.hpp
@@ -15,80 +15,28 @@
#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP
#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP
-#include <unordered_map>
+#include <filesystem>
+#include <functional>
+#include <optional>
+#include <string>
+#include <utility>
+#include <gsl-lite/gsl-lite.hpp>
#include <nlohmann/json.hpp>
-#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
-#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp"
+#include "src/buildtool/build_engine/target_map/target_cache_key.hpp"
#include "src/buildtool/common/artifact.hpp"
-#ifndef BOOTSTRAP_BUILD_TOOL
-#include "src/buildtool/execution_api/common/execution_api.hpp"
-#endif
#include "src/buildtool/execution_api/local/file_storage.hpp"
#include "src/buildtool/execution_api/local/local_cas.hpp"
-#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/logging/logger.hpp"
+#ifndef BOOTSTRAP_BUILD_TOOL
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#endif
class TargetCache {
public:
- // Key for target cache. Created from target name and effective config.
- class Key {
- public:
- [[nodiscard]] static auto Create(
- BuildMaps::Base::EntityName const& target,
- Configuration const& effective_config) noexcept
- -> std::optional<Key>;
-
- [[nodiscard]] auto Id() const& -> Artifact::ObjectInfo const& {
- return id_;
- }
- [[nodiscard]] auto Id() && -> Artifact::ObjectInfo {
- return std::move(id_);
- }
- [[nodiscard]] auto operator==(Key const& other) const -> bool {
- return this->Id() == other.Id();
- }
-
- private:
- explicit Key(Artifact::ObjectInfo id) : id_{std::move(id)} {}
- Artifact::ObjectInfo id_;
- };
-
- // Entry for target cache. Created from target, contains TargetResult.
- class Entry {
- friend class TargetCache;
-
- public:
- // Create the entry from target with replacement artifacts/infos.
- // Replacement artifacts must replace all non-known artifacts by known.
- [[nodiscard]] static auto FromTarget(
- AnalysedTargetPtr const& target,
- std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
- replacements) noexcept -> std::optional<Entry>;
-
- // Obtain TargetResult from cache entry.
- [[nodiscard]] auto ToResult() const noexcept
- -> std::optional<TargetResult>;
-
- // Obtain all artifacts from cache entry (all should be known
- // artifacts).
- [[nodiscard]] auto ToArtifacts(
- gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos)
- const noexcept -> bool;
-
- private:
- nlohmann::json desc_;
-
- explicit Entry(nlohmann::json desc) : desc_(std::move(desc)) {}
- [[nodiscard]] auto ToJson() const& -> nlohmann::json const& {
- return desc_;
- }
- [[nodiscard]] auto ToJson() && -> nlohmann::json {
- return std::move(desc_);
- }
- };
-
TargetCache() = default;
TargetCache(TargetCache const&) = delete;
TargetCache(TargetCache&&) = delete;
@@ -102,12 +50,13 @@ class TargetCache {
}
// Store new key entry pair in the target cache.
- [[nodiscard]] auto Store(Key const& key, Entry const& value) const noexcept
+ [[nodiscard]] auto Store(TargetCacheKey const& key,
+ TargetCacheEntry const& value) const noexcept
-> bool;
// Read existing entry and object info for given key from the target cache.
- [[nodiscard]] auto Read(Key const& key) const noexcept
- -> std::optional<std::pair<Entry, Artifact::ObjectInfo>>;
+ [[nodiscard]] auto Read(TargetCacheKey const& key) const noexcept
+ -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>;
#ifndef BOOTSTRAP_BUILD_TOOL
auto SetLocalApi(gsl::not_null<IExecutionApi*> const& api) noexcept
@@ -132,12 +81,8 @@ class TargetCache {
IExecutionApi* remote_api_{};
#endif
- [[nodiscard]] auto DownloadKnownArtifacts(Entry const& value) const noexcept
- -> bool;
- [[nodiscard]] auto CollectKnownArtifacts(
- gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos,
- gsl::not_null<std::unordered_set<std::string>*> const& hashes,
- ExpressionPtr const& expr) const noexcept -> bool;
+ [[nodiscard]] auto DownloadKnownArtifacts(
+ TargetCacheEntry const& value) const noexcept -> bool;
[[nodiscard]] static auto CAS() noexcept -> LocalCAS<ObjectType::File>& {
return LocalCAS<ObjectType::File>::Instance();
}
@@ -147,8 +92,8 @@ class TargetCache {
namespace std {
template <>
-struct hash<TargetCache::Key> {
- [[nodiscard]] auto operator()(TargetCache::Key const& k) const {
+struct hash<TargetCacheKey> {
+ [[nodiscard]] auto operator()(TargetCacheKey const& k) const {
return std::hash<Artifact::ObjectInfo>{}(k.Id());
}
};
diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.cpp b/src/buildtool/build_engine/target_map/target_cache_entry.cpp
new file mode 100644
index 00000000..45815056
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_cache_entry.cpp
@@ -0,0 +1,101 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp"
+
+#include <algorithm>
+#include <exception>
+#include <iterator>
+
+#include "src/buildtool/logging/log_level.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+auto TargetCacheEntry::FromTarget(
+ AnalysedTargetPtr const& target,
+ std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
+ replacements) noexcept -> std::optional<TargetCacheEntry> {
+ auto result = TargetResult{
+ target->Artifacts(), target->Provides(), target->RunFiles()};
+ if (auto desc = result.ReplaceNonKnownAndToJson(replacements)) {
+ return TargetCacheEntry{*desc};
+ }
+ return std::nullopt;
+}
+
+auto TargetCacheEntry::ToResult() const noexcept
+ -> std::optional<TargetResult> {
+ return TargetResult::FromJson(desc_);
+}
+
+[[nodiscard]] auto ToObjectInfo(nlohmann::json const& json)
+ -> Artifact::ObjectInfo {
+ auto const& desc = ArtifactDescription::FromJson(json);
+ // The assumption is that all artifacts mentioned in a target cache
+ // entry are KNOWN to the remote side.
+ gsl_ExpectsAudit(desc and desc->IsKnown());
+ auto const& info = desc->ToArtifact().Info();
+ gsl_ExpectsAudit(info);
+ return *info;
+}
+
+[[nodiscard]] auto ScanArtifactMap(
+ gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos,
+ nlohmann::json const& json) -> bool {
+ if (not json.is_object()) {
+ return false;
+ }
+ infos->reserve(infos->size() + json.size());
+ std::transform(json.begin(),
+ json.end(),
+ std::back_inserter(*infos),
+ [](auto const& item) { return ToObjectInfo(item); });
+ return true;
+}
+
+[[nodiscard]] auto ScanProvidesMap(
+ gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos,
+ nlohmann::json const& json) -> bool {
+ if (not json.is_object()) {
+ return false;
+ }
+ auto const& nodes = json["nodes"];
+ auto const& provided_artifacts = json["provided_artifacts"];
+ infos->reserve(infos->size() + provided_artifacts.size());
+ std::transform(
+ provided_artifacts.begin(),
+ provided_artifacts.end(),
+ std::back_inserter(*infos),
+ [&nodes](auto const& item) {
+ return ToObjectInfo(nodes[item.template get<std::string>()]);
+ });
+ return true;
+}
+
+auto TargetCacheEntry::ToArtifacts(
+ gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos)
+ const noexcept -> bool {
+ try {
+ if (ScanArtifactMap(infos, desc_["artifacts"]) and
+ ScanArtifactMap(infos, desc_["runfiles"]) and
+ ScanProvidesMap(infos, desc_["provides"])) {
+ return true;
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(
+ LogLevel::Error,
+ "Scanning target cache entry for artifacts failed with:\n{}",
+ ex.what());
+ }
+ return false;
+}
diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.hpp b/src/buildtool/build_engine/target_map/target_cache_entry.hpp
new file mode 100644
index 00000000..b6d1351c
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_cache_entry.hpp
@@ -0,0 +1,64 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP
+
+#include <optional>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <gsl-lite/gsl-lite.hpp>
+#include <nlohmann/json.hpp>
+
+#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
+#include "src/buildtool/build_engine/expression/target_result.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/artifact_description.hpp"
+
+// Entry for target cache. Created from target, contains TargetResult.
+class TargetCacheEntry {
+ friend class TargetCache;
+
+ public:
+ // Create the entry from target with replacement artifacts/infos.
+ // Replacement artifacts must replace all non-known artifacts by known.
+ [[nodiscard]] static auto FromTarget(
+ AnalysedTargetPtr const& target,
+ std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
+ replacements) noexcept -> std::optional<TargetCacheEntry>;
+
+ // Obtain TargetResult from cache entry.
+ [[nodiscard]] auto ToResult() const noexcept -> std::optional<TargetResult>;
+
+ // Obtain all artifacts from cache entry (all should be known
+ // artifacts).
+ [[nodiscard]] auto ToArtifacts(
+ gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos)
+ const noexcept -> bool;
+
+ private:
+ nlohmann::json desc_;
+
+ explicit TargetCacheEntry(nlohmann::json desc) : desc_(std::move(desc)) {}
+ [[nodiscard]] auto ToJson() const& -> nlohmann::json const& {
+ return desc_;
+ }
+ [[nodiscard]] auto ToJson() && -> nlohmann::json {
+ return std::move(desc_);
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP
diff --git a/src/buildtool/build_engine/target_map/target_cache_key.cpp b/src/buildtool/build_engine/target_map/target_cache_key.cpp
new file mode 100644
index 00000000..1ee6a7ef
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_cache_key.cpp
@@ -0,0 +1,53 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/build_engine/target_map/target_cache_key.hpp"
+
+#include <exception>
+
+#include <nlohmann/json.hpp>
+
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/execution_api/local/local_cas.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/logging/log_level.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+auto TargetCacheKey::Create(BuildMaps::Base::EntityName const& target,
+ Configuration const& effective_config) noexcept
+ -> std::optional<TargetCacheKey> {
+ static auto const& repos = RepositoryConfig::Instance();
+ try {
+ if (auto repo_key =
+ repos.RepositoryKey(target.GetNamedTarget().repository)) {
+ // target's repository is content-fixed, we can compute a cache key
+ auto const& name = target.GetNamedTarget();
+ auto target_desc = nlohmann::json{
+ {{"repo_key", *repo_key},
+ {"target_name", nlohmann::json{name.module, name.name}.dump()},
+ {"effective_config", effective_config.ToString()}}};
+ static auto const& cas = LocalCAS<ObjectType::File>::Instance();
+ if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) {
+ return TargetCacheKey{
+ {ArtifactDigest{*target_key}, ObjectType::File}};
+ }
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "Creating target cache key failed with:\n{}",
+ ex.what());
+ }
+ return std::nullopt;
+}
diff --git a/src/buildtool/build_engine/target_map/target_cache_key.hpp b/src/buildtool/build_engine/target_map/target_cache_key.hpp
new file mode 100644
index 00000000..bc648135
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_cache_key.hpp
@@ -0,0 +1,48 @@
+// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP
+
+#include <optional>
+#include <utility>
+
+#include "src/buildtool/build_engine/base_maps/entity_name_data.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/common/artifact.hpp"
+
+// Key for target cache. Created from target name and effective config.
+class TargetCacheKey {
+ public:
+ [[nodiscard]] static auto Create(
+ BuildMaps::Base::EntityName const& target,
+ Configuration const& effective_config) noexcept
+ -> std::optional<TargetCacheKey>;
+
+ [[nodiscard]] auto Id() const& -> Artifact::ObjectInfo const& {
+ return id_;
+ }
+ [[nodiscard]] auto Id() && -> Artifact::ObjectInfo {
+ return std::move(id_);
+ }
+ [[nodiscard]] auto operator==(TargetCacheKey const& other) const -> bool {
+ return this->Id() == other.Id();
+ }
+
+ private:
+ explicit TargetCacheKey(Artifact::ObjectInfo id) : id_{std::move(id)} {}
+ Artifact::ObjectInfo id_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP
diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS
index 70e361a2..d139d1d8 100644
--- a/src/buildtool/main/TARGETS
+++ b/src/buildtool/main/TARGETS
@@ -12,6 +12,8 @@
, ["src/buildtool/progress_reporting", "base_progress_reporter"]
, ["src/buildtool/build_engine/target_map", "result_map"]
, ["src/buildtool/build_engine/target_map", "target_cache"]
+ , ["src/buildtool/build_engine/target_map", "target_cache_key"]
+ , ["src/buildtool/build_engine/target_map", "target_cache_entry"]
, ["src/buildtool/build_engine/target_map", "target_map"]
, ["src/buildtool/multithreading", "task_system"]
, ["src/utils/cpp", "concepts"]
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index cf98bcf3..5bb707f5 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -25,6 +25,8 @@
#include "src/buildtool/build_engine/expression/evaluator.hpp"
#include "src/buildtool/build_engine/expression/expression.hpp"
#include "src/buildtool/build_engine/target_map/target_cache.hpp"
+#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp"
+#include "src/buildtool/build_engine/target_map/target_cache_key.hpp"
#include "src/buildtool/build_engine/target_map/target_map.hpp"
#include "src/buildtool/common/artifact_description.hpp"
#include "src/buildtool/common/cli.hpp"
@@ -1073,8 +1075,8 @@ void DumpArtifactsToBuild(
}
auto CollectNonKnownArtifacts(
- std::unordered_map<TargetCache::Key, AnalysedTargetPtr> const&
- cache_targets) -> std::vector<ArtifactDescription> {
+ std::unordered_map<TargetCacheKey, AnalysedTargetPtr> const& cache_targets)
+ -> std::vector<ArtifactDescription> {
auto cache_artifacts = std::unordered_set<ArtifactDescription>{};
for (auto const& [_, target] : cache_targets) {
auto artifacts = target->ContainedNonKnownArtifacts();
@@ -1087,8 +1089,7 @@ auto CollectNonKnownArtifacts(
#ifndef BOOTSTRAP_BUILD_TOOL
void WriteTargetCacheEntries(
- std::unordered_map<TargetCache::Key, AnalysedTargetPtr> const&
- cache_targets,
+ std::unordered_map<TargetCacheKey, AnalysedTargetPtr> const& cache_targets,
std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
extra_infos,
std::size_t jobs,
@@ -1100,7 +1101,7 @@ void WriteTargetCacheEntries(
for (auto const& [key, target] : cache_targets) {
ts.QueueTask([&key = key, &target = target, &extra_infos]() {
if (auto entry =
- TargetCache::Entry::FromTarget(target, extra_infos)) {
+ TargetCacheEntry::FromTarget(target, extra_infos)) {
if (not TargetCache::Instance().Store(key, *entry)) {
Logger::Log(LogLevel::Warning,
"Failed writing target cache entry for {}",