diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/buildtool/build_engine/target_map/TARGETS | 55 | ||||
| -rw-r--r-- | src/buildtool/build_engine/target_map/export.cpp | 5 | ||||
| -rw-r--r-- | src/buildtool/build_engine/target_map/result_map.hpp | 9 | ||||
| -rw-r--r-- | src/buildtool/build_engine/target_map/target_cache.cpp | 134 | ||||
| -rw-r--r-- | src/buildtool/build_engine/target_map/target_cache.hpp | 95 | ||||
| -rw-r--r-- | src/buildtool/build_engine/target_map/target_cache_entry.cpp | 101 | ||||
| -rw-r--r-- | src/buildtool/build_engine/target_map/target_cache_entry.hpp | 64 | ||||
| -rw-r--r-- | src/buildtool/build_engine/target_map/target_cache_key.cpp | 53 | ||||
| -rw-r--r-- | src/buildtool/build_engine/target_map/target_cache_key.hpp | 48 | ||||
| -rw-r--r-- | src/buildtool/main/TARGETS | 2 | ||||
| -rw-r--r-- | src/buildtool/main/main.cpp | 11 |
11 files changed, 368 insertions, 209 deletions
diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS index f7bd8cb3..ee65b3e3 100644 --- a/src/buildtool/build_engine/target_map/TARGETS +++ b/src/buildtool/build_engine/target_map/TARGETS @@ -52,6 +52,7 @@ , "stage": ["src", "buildtool", "build_engine", "target_map"] , "private-deps": [ "target_cache" + , "target_cache_key" , ["src/buildtool/common", "common"] , ["src/buildtool/build_engine/base_maps", "entity_name"] , ["src/buildtool/build_engine/base_maps", "field_reader"] @@ -85,18 +86,62 @@ , "hdrs": ["target_cache.hpp"] , "srcs": ["target_cache.cpp"] , "deps": - [ ["src/buildtool/build_engine/analysed_target", "target"] - , ["src/buildtool/build_engine/base_maps", "entity_name"] + [ "target_cache_key" + , "target_cache_entry" + , ["@", "gsl-lite", "", "gsl-lite"] + , ["@", "json", "", "json"] + , ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/local", "local"] + , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "logging"] - , ["src/buildtool/common", "common"] , ["src/buildtool/execution_api/common", "common"] + ] + , "stage": ["src", "buildtool", "build_engine", "target_map"] + , "private-deps": + [ ["@", "fmt", "", "fmt"] + , ["src/buildtool/compatibility", "compatibility"] + , ["src/buildtool/execution_api/local", "config"] + , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/file_system", "file_system_manager"] + , ["src/buildtool/logging", "log_level"] + ] + } +, "target_cache_key": + { "type": ["@", "rules", "CC", "library"] + , "name": ["target_cache_key"] + , "hdrs": ["target_cache_key.hpp"] + , "srcs": ["target_cache_key.cpp"] + , "deps": + [ ["src/buildtool/build_engine/base_maps", "entity_name_data"] + , ["src/buildtool/build_engine/expression", "expression"] + , ["src/buildtool/common", "common"] + ] + , "stage": ["src", "buildtool", "build_engine", "target_map"] + , "private-deps": + [ ["src/buildtool/common", "config"] + , ["src/buildtool/execution_api/local", "local"] + , ["src/buildtool/file_system", "object_type"] + , ["src/buildtool/logging", "logging"] + , ["src/buildtool/logging", "log_level"] + ] + } +, "target_cache_entry": + { "type": ["@", "rules", "CC", "library"] + , "name": ["target_cache_entry"] + , "hdrs": ["target_cache_entry.hpp"] + , "srcs": ["target_cache_entry.cpp"] + , "deps": + [ ["@", "gsl-lite", "", "gsl-lite"] + , ["@", "json", "", "json"] + , ["src/buildtool/build_engine/analysed_target", "target"] + , ["src/buildtool/build_engine/expression", "expression"] + , ["src/buildtool/common", "common"] + , ["src/buildtool/common", "artifact_description"] ] , "stage": ["src", "buildtool", "build_engine", "target_map"] , "private-deps": - [ ["src/buildtool/execution_api/remote", "config"] - , ["src/buildtool/file_system", "file_root"] + [ ["src/buildtool/logging", "logging"] + , ["src/buildtool/logging", "log_level"] ] } } diff --git a/src/buildtool/build_engine/target_map/export.cpp b/src/buildtool/build_engine/target_map/export.cpp index 657ca73b..7a53e78e 100644 --- a/src/buildtool/build_engine/target_map/export.cpp +++ b/src/buildtool/build_engine/target_map/export.cpp @@ -19,6 +19,7 @@ #include "src/buildtool/build_engine/base_maps/field_reader.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/build_engine/target_map/target_cache.hpp" +#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/execution_api/local/local_cas.hpp" @@ -35,7 +36,7 @@ void FinalizeExport( const BuildMaps::Base::EntityName& target, const std::vector<std::string>& vars, const Configuration& effective_config, - const std::optional<TargetCache::Key>& target_cache_key, + const std::optional<TargetCacheKey>& target_cache_key, const BuildMaps::Target::TargetMap::LoggerPtr& logger, const BuildMaps::Target::TargetMap::SetterPtr& setter, const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) { @@ -129,7 +130,7 @@ void ExportRule( auto target_config = effective_config.Update(fixed_config); auto target_cache_key = - TargetCache::Key::Create(*exported_target, target_config); + TargetCacheKey::Create(*exported_target, target_config); if (target_cache_key) { if (auto target_cache_value = TargetCache::Instance().Read(*target_cache_key)) { diff --git a/src/buildtool/build_engine/target_map/result_map.hpp b/src/buildtool/build_engine/target_map/result_map.hpp index 700cf7bb..edb1a472 100644 --- a/src/buildtool/build_engine/target_map/result_map.hpp +++ b/src/buildtool/build_engine/target_map/result_map.hpp @@ -31,6 +31,7 @@ #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/target_map/configured_target.hpp" #include "src/buildtool/build_engine/target_map/target_cache.hpp" +#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/logging/logger.hpp" @@ -68,7 +69,7 @@ class ResultTargetMap { BuildMaps::Base::EntityName name, Configuration conf, gsl::not_null<AnalysedTargetPtr> result, - std::optional<TargetCache::Key> target_cache_key = std::nullopt, + std::optional<TargetCacheKey> target_cache_key = std::nullopt, bool is_export_target = false) -> AnalysedTargetPtr { auto part = std::hash<BuildMaps::Base::EntityName>{}(name) % width_; std::unique_lock lock{m_[part]}; @@ -142,11 +143,11 @@ class ResultTargetMap { } [[nodiscard]] auto CacheTargets() const noexcept - -> std::unordered_map<TargetCache::Key, AnalysedTargetPtr> { + -> std::unordered_map<TargetCacheKey, AnalysedTargetPtr> { return std::accumulate( cache_targets_.begin(), cache_targets_.end(), - std::unordered_map<TargetCache::Key, AnalysedTargetPtr>{}, + std::unordered_map<TargetCacheKey, AnalysedTargetPtr>{}, [](auto&& l, auto const& r) { l.insert(r.begin(), r.end()); return std::forward<decltype(l)>(l); @@ -361,7 +362,7 @@ class ResultTargetMap { std::unordered_map<ConfiguredTarget, gsl::not_null<AnalysedTargetPtr>>> targets_{width_}; std::vector< - std::unordered_map<TargetCache::Key, gsl::not_null<AnalysedTargetPtr>>> + std::unordered_map<TargetCacheKey, gsl::not_null<AnalysedTargetPtr>>> cache_targets_{width_}; std::vector<std::unordered_set<ConfiguredTarget>> export_targets_{width_}; std::vector<std::size_t> num_actions_{std::vector<std::size_t>(width_)}; diff --git a/src/buildtool/build_engine/target_map/target_cache.cpp b/src/buildtool/build_engine/target_map/target_cache.cpp index b2f3b209..727b7404 100644 --- a/src/buildtool/build_engine/target_map/target_cache.cpp +++ b/src/buildtool/build_engine/target_map/target_cache.cpp @@ -14,123 +14,20 @@ #include "src/buildtool/build_engine/target_map/target_cache.hpp" -#include <gsl-lite/gsl-lite.hpp> +#include <exception> +#include <vector> -#include "src/buildtool/common/repository_config.hpp" +#include <fmt/core.h> + +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/compatibility/native_support.hpp" #include "src/buildtool/execution_api/local/config.hpp" -#include "src/buildtool/execution_api/local/file_storage.hpp" -#include "src/buildtool/execution_api/local/local_cas.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" -#include "src/buildtool/logging/logger.hpp" - -auto TargetCache::Key::Create(BuildMaps::Base::EntityName const& target, - Configuration const& effective_config) noexcept - -> std::optional<TargetCache::Key> { - static auto const& repos = RepositoryConfig::Instance(); - try { - if (auto repo_key = - repos.RepositoryKey(target.GetNamedTarget().repository)) { - // target's repository is content-fixed, we can compute a cache key - auto const& name = target.GetNamedTarget(); - auto target_desc = nlohmann::json{ - {{"repo_key", *repo_key}, - {"target_name", nlohmann::json{name.module, name.name}.dump()}, - {"effective_config", effective_config.ToString()}}}; - static auto const& cas = LocalCAS<ObjectType::File>::Instance(); - if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) { - return Key{{ArtifactDigest{*target_key}, ObjectType::File}}; - } - } - } catch (std::exception const& ex) { - Logger::Log(LogLevel::Error, - "Creating target cache key failed with:\n{}", - ex.what()); - } - return std::nullopt; -} - -auto TargetCache::Entry::FromTarget( - AnalysedTargetPtr const& target, - std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const& - replacements) noexcept -> std::optional<TargetCache::Entry> { - auto result = TargetResult{ - target->Artifacts(), target->Provides(), target->RunFiles()}; - if (auto desc = result.ReplaceNonKnownAndToJson(replacements)) { - return Entry{*desc}; - } - return std::nullopt; -} - -auto TargetCache::Entry::ToResult() const noexcept - -> std::optional<TargetResult> { - return TargetResult::FromJson(desc_); -} - -[[nodiscard]] auto ToObjectInfo(nlohmann::json const& json) - -> Artifact::ObjectInfo { - auto const& desc = ArtifactDescription::FromJson(json); - // The assumption is that all artifacts mentioned in a target cache - // entry are KNOWN to the remote side. - gsl_ExpectsAudit(desc and desc->IsKnown()); - auto const& info = desc->ToArtifact().Info(); - gsl_ExpectsAudit(info); - return *info; -} - -[[nodiscard]] auto ScanArtifactMap( - gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos, - nlohmann::json const& json) -> bool { - if (not json.is_object()) { - return false; - } - infos->reserve(infos->size() + json.size()); - std::transform(json.begin(), - json.end(), - std::back_inserter(*infos), - [](auto const& item) { return ToObjectInfo(item); }); - return true; -} - -[[nodiscard]] auto ScanProvidesMap( - gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos, - nlohmann::json const& json) -> bool { - if (not json.is_object()) { - return false; - } - auto const& nodes = json["nodes"]; - auto const& provided_artifacts = json["provided_artifacts"]; - infos->reserve(infos->size() + provided_artifacts.size()); - std::transform( - provided_artifacts.begin(), - provided_artifacts.end(), - std::back_inserter(*infos), - [&nodes](auto const& item) { - return ToObjectInfo(nodes[item.template get<std::string>()]); - }); - return true; -} - -auto TargetCache::Entry::ToArtifacts( - gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos) - const noexcept -> bool { - try { - if (ScanArtifactMap(infos, desc_["artifacts"]) and - ScanArtifactMap(infos, desc_["runfiles"]) and - ScanProvidesMap(infos, desc_["provides"])) { - return true; - } - } catch (std::exception const& ex) { - Logger::Log( - LogLevel::Error, - "Scanning target cache entry for artifacts failed with:\n{}", - ex.what()); - } - return false; -} +#include "src/buildtool/logging/log_level.hpp" -auto TargetCache::Store(Key const& key, Entry const& value) const noexcept - -> bool { +auto TargetCache::Store(TargetCacheKey const& key, + TargetCacheEntry const& value) const noexcept -> bool { // Before a target-cache entry is stored in local CAS, make sure any created // artifact for this target is downloaded from the remote CAS to the local // CAS. @@ -150,8 +47,8 @@ auto TargetCache::Store(Key const& key, Entry const& value) const noexcept return false; } -auto TargetCache::Read(Key const& key) const noexcept - -> std::optional<std::pair<Entry, Artifact::ObjectInfo>> { +auto TargetCache::Read(TargetCacheKey const& key) const noexcept + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>> { auto entry_path = file_store_.GetPath(key.Id().digest.hash()); auto const entry = FileSystemManager::ReadFile(entry_path, ObjectType::File); @@ -165,8 +62,9 @@ auto TargetCache::Read(Key const& key) const noexcept if (auto path = CAS().BlobPath(info->digest)) { if (auto value = FileSystemManager::ReadFile(*path)) { try { - return std::make_pair(Entry{nlohmann::json::parse(*value)}, - std::move(*info)); + return std::make_pair( + TargetCacheEntry{nlohmann::json::parse(*value)}, + std::move(*info)); } catch (std::exception const& ex) { logger_.Emit(LogLevel::Warning, "Parsing entry for key {} failed with:\n{}", @@ -182,8 +80,8 @@ auto TargetCache::Read(Key const& key) const noexcept return std::nullopt; } -auto TargetCache::DownloadKnownArtifacts(Entry const& value) const noexcept - -> bool { +auto TargetCache::DownloadKnownArtifacts( + TargetCacheEntry const& value) const noexcept -> bool { std::vector<Artifact::ObjectInfo> artifacts_info; if (not value.ToArtifacts(&artifacts_info)) { return false; diff --git a/src/buildtool/build_engine/target_map/target_cache.hpp b/src/buildtool/build_engine/target_map/target_cache.hpp index fe32d3dd..b6b54ce0 100644 --- a/src/buildtool/build_engine/target_map/target_cache.hpp +++ b/src/buildtool/build_engine/target_map/target_cache.hpp @@ -15,80 +15,28 @@ #ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP #define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_HPP -#include <unordered_map> +#include <filesystem> +#include <functional> +#include <optional> +#include <string> +#include <utility> +#include <gsl-lite/gsl-lite.hpp> #include <nlohmann/json.hpp> -#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" -#include "src/buildtool/build_engine/base_maps/entity_name.hpp" +#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" +#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/common/artifact.hpp" -#ifndef BOOTSTRAP_BUILD_TOOL -#include "src/buildtool/execution_api/common/execution_api.hpp" -#endif #include "src/buildtool/execution_api/local/file_storage.hpp" #include "src/buildtool/execution_api/local/local_cas.hpp" -#include "src/buildtool/file_system/file_system_manager.hpp" +#include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/logger.hpp" +#ifndef BOOTSTRAP_BUILD_TOOL +#include "src/buildtool/execution_api/common/execution_api.hpp" +#endif class TargetCache { public: - // Key for target cache. Created from target name and effective config. - class Key { - public: - [[nodiscard]] static auto Create( - BuildMaps::Base::EntityName const& target, - Configuration const& effective_config) noexcept - -> std::optional<Key>; - - [[nodiscard]] auto Id() const& -> Artifact::ObjectInfo const& { - return id_; - } - [[nodiscard]] auto Id() && -> Artifact::ObjectInfo { - return std::move(id_); - } - [[nodiscard]] auto operator==(Key const& other) const -> bool { - return this->Id() == other.Id(); - } - - private: - explicit Key(Artifact::ObjectInfo id) : id_{std::move(id)} {} - Artifact::ObjectInfo id_; - }; - - // Entry for target cache. Created from target, contains TargetResult. - class Entry { - friend class TargetCache; - - public: - // Create the entry from target with replacement artifacts/infos. - // Replacement artifacts must replace all non-known artifacts by known. - [[nodiscard]] static auto FromTarget( - AnalysedTargetPtr const& target, - std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const& - replacements) noexcept -> std::optional<Entry>; - - // Obtain TargetResult from cache entry. - [[nodiscard]] auto ToResult() const noexcept - -> std::optional<TargetResult>; - - // Obtain all artifacts from cache entry (all should be known - // artifacts). - [[nodiscard]] auto ToArtifacts( - gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos) - const noexcept -> bool; - - private: - nlohmann::json desc_; - - explicit Entry(nlohmann::json desc) : desc_(std::move(desc)) {} - [[nodiscard]] auto ToJson() const& -> nlohmann::json const& { - return desc_; - } - [[nodiscard]] auto ToJson() && -> nlohmann::json { - return std::move(desc_); - } - }; - TargetCache() = default; TargetCache(TargetCache const&) = delete; TargetCache(TargetCache&&) = delete; @@ -102,12 +50,13 @@ class TargetCache { } // Store new key entry pair in the target cache. - [[nodiscard]] auto Store(Key const& key, Entry const& value) const noexcept + [[nodiscard]] auto Store(TargetCacheKey const& key, + TargetCacheEntry const& value) const noexcept -> bool; // Read existing entry and object info for given key from the target cache. - [[nodiscard]] auto Read(Key const& key) const noexcept - -> std::optional<std::pair<Entry, Artifact::ObjectInfo>>; + [[nodiscard]] auto Read(TargetCacheKey const& key) const noexcept + -> std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>; #ifndef BOOTSTRAP_BUILD_TOOL auto SetLocalApi(gsl::not_null<IExecutionApi*> const& api) noexcept @@ -132,12 +81,8 @@ class TargetCache { IExecutionApi* remote_api_{}; #endif - [[nodiscard]] auto DownloadKnownArtifacts(Entry const& value) const noexcept - -> bool; - [[nodiscard]] auto CollectKnownArtifacts( - gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos, - gsl::not_null<std::unordered_set<std::string>*> const& hashes, - ExpressionPtr const& expr) const noexcept -> bool; + [[nodiscard]] auto DownloadKnownArtifacts( + TargetCacheEntry const& value) const noexcept -> bool; [[nodiscard]] static auto CAS() noexcept -> LocalCAS<ObjectType::File>& { return LocalCAS<ObjectType::File>::Instance(); } @@ -147,8 +92,8 @@ class TargetCache { namespace std { template <> -struct hash<TargetCache::Key> { - [[nodiscard]] auto operator()(TargetCache::Key const& k) const { +struct hash<TargetCacheKey> { + [[nodiscard]] auto operator()(TargetCacheKey const& k) const { return std::hash<Artifact::ObjectInfo>{}(k.Id()); } }; diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.cpp b/src/buildtool/build_engine/target_map/target_cache_entry.cpp new file mode 100644 index 00000000..45815056 --- /dev/null +++ b/src/buildtool/build_engine/target_map/target_cache_entry.cpp @@ -0,0 +1,101 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" + +#include <algorithm> +#include <exception> +#include <iterator> + +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" + +auto TargetCacheEntry::FromTarget( + AnalysedTargetPtr const& target, + std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const& + replacements) noexcept -> std::optional<TargetCacheEntry> { + auto result = TargetResult{ + target->Artifacts(), target->Provides(), target->RunFiles()}; + if (auto desc = result.ReplaceNonKnownAndToJson(replacements)) { + return TargetCacheEntry{*desc}; + } + return std::nullopt; +} + +auto TargetCacheEntry::ToResult() const noexcept + -> std::optional<TargetResult> { + return TargetResult::FromJson(desc_); +} + +[[nodiscard]] auto ToObjectInfo(nlohmann::json const& json) + -> Artifact::ObjectInfo { + auto const& desc = ArtifactDescription::FromJson(json); + // The assumption is that all artifacts mentioned in a target cache + // entry are KNOWN to the remote side. + gsl_ExpectsAudit(desc and desc->IsKnown()); + auto const& info = desc->ToArtifact().Info(); + gsl_ExpectsAudit(info); + return *info; +} + +[[nodiscard]] auto ScanArtifactMap( + gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos, + nlohmann::json const& json) -> bool { + if (not json.is_object()) { + return false; + } + infos->reserve(infos->size() + json.size()); + std::transform(json.begin(), + json.end(), + std::back_inserter(*infos), + [](auto const& item) { return ToObjectInfo(item); }); + return true; +} + +[[nodiscard]] auto ScanProvidesMap( + gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos, + nlohmann::json const& json) -> bool { + if (not json.is_object()) { + return false; + } + auto const& nodes = json["nodes"]; + auto const& provided_artifacts = json["provided_artifacts"]; + infos->reserve(infos->size() + provided_artifacts.size()); + std::transform( + provided_artifacts.begin(), + provided_artifacts.end(), + std::back_inserter(*infos), + [&nodes](auto const& item) { + return ToObjectInfo(nodes[item.template get<std::string>()]); + }); + return true; +} + +auto TargetCacheEntry::ToArtifacts( + gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos) + const noexcept -> bool { + try { + if (ScanArtifactMap(infos, desc_["artifacts"]) and + ScanArtifactMap(infos, desc_["runfiles"]) and + ScanProvidesMap(infos, desc_["provides"])) { + return true; + } + } catch (std::exception const& ex) { + Logger::Log( + LogLevel::Error, + "Scanning target cache entry for artifacts failed with:\n{}", + ex.what()); + } + return false; +} diff --git a/src/buildtool/build_engine/target_map/target_cache_entry.hpp b/src/buildtool/build_engine/target_map/target_cache_entry.hpp new file mode 100644 index 00000000..b6d1351c --- /dev/null +++ b/src/buildtool/build_engine/target_map/target_cache_entry.hpp @@ -0,0 +1,64 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP +#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP + +#include <optional> +#include <unordered_map> +#include <utility> +#include <vector> + +#include <gsl-lite/gsl-lite.hpp> +#include <nlohmann/json.hpp> + +#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp" +#include "src/buildtool/build_engine/expression/target_result.hpp" +#include "src/buildtool/common/artifact.hpp" +#include "src/buildtool/common/artifact_description.hpp" + +// Entry for target cache. Created from target, contains TargetResult. +class TargetCacheEntry { + friend class TargetCache; + + public: + // Create the entry from target with replacement artifacts/infos. + // Replacement artifacts must replace all non-known artifacts by known. + [[nodiscard]] static auto FromTarget( + AnalysedTargetPtr const& target, + std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const& + replacements) noexcept -> std::optional<TargetCacheEntry>; + + // Obtain TargetResult from cache entry. + [[nodiscard]] auto ToResult() const noexcept -> std::optional<TargetResult>; + + // Obtain all artifacts from cache entry (all should be known + // artifacts). + [[nodiscard]] auto ToArtifacts( + gsl::not_null<std::vector<Artifact::ObjectInfo>*> const& infos) + const noexcept -> bool; + + private: + nlohmann::json desc_; + + explicit TargetCacheEntry(nlohmann::json desc) : desc_(std::move(desc)) {} + [[nodiscard]] auto ToJson() const& -> nlohmann::json const& { + return desc_; + } + [[nodiscard]] auto ToJson() && -> nlohmann::json { + return std::move(desc_); + } +}; + +#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_ENTRY_HPP diff --git a/src/buildtool/build_engine/target_map/target_cache_key.cpp b/src/buildtool/build_engine/target_map/target_cache_key.cpp new file mode 100644 index 00000000..1ee6a7ef --- /dev/null +++ b/src/buildtool/build_engine/target_map/target_cache_key.cpp @@ -0,0 +1,53 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" + +#include <exception> + +#include <nlohmann/json.hpp> + +#include "src/buildtool/common/artifact_digest.hpp" +#include "src/buildtool/common/repository_config.hpp" +#include "src/buildtool/execution_api/local/local_cas.hpp" +#include "src/buildtool/file_system/object_type.hpp" +#include "src/buildtool/logging/log_level.hpp" +#include "src/buildtool/logging/logger.hpp" + +auto TargetCacheKey::Create(BuildMaps::Base::EntityName const& target, + Configuration const& effective_config) noexcept + -> std::optional<TargetCacheKey> { + static auto const& repos = RepositoryConfig::Instance(); + try { + if (auto repo_key = + repos.RepositoryKey(target.GetNamedTarget().repository)) { + // target's repository is content-fixed, we can compute a cache key + auto const& name = target.GetNamedTarget(); + auto target_desc = nlohmann::json{ + {{"repo_key", *repo_key}, + {"target_name", nlohmann::json{name.module, name.name}.dump()}, + {"effective_config", effective_config.ToString()}}}; + static auto const& cas = LocalCAS<ObjectType::File>::Instance(); + if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) { + return TargetCacheKey{ + {ArtifactDigest{*target_key}, ObjectType::File}}; + } + } + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "Creating target cache key failed with:\n{}", + ex.what()); + } + return std::nullopt; +} diff --git a/src/buildtool/build_engine/target_map/target_cache_key.hpp b/src/buildtool/build_engine/target_map/target_cache_key.hpp new file mode 100644 index 00000000..bc648135 --- /dev/null +++ b/src/buildtool/build_engine/target_map/target_cache_key.hpp @@ -0,0 +1,48 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP +#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP + +#include <optional> +#include <utility> + +#include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" +#include "src/buildtool/build_engine/expression/configuration.hpp" +#include "src/buildtool/common/artifact.hpp" + +// Key for target cache. Created from target name and effective config. +class TargetCacheKey { + public: + [[nodiscard]] static auto Create( + BuildMaps::Base::EntityName const& target, + Configuration const& effective_config) noexcept + -> std::optional<TargetCacheKey>; + + [[nodiscard]] auto Id() const& -> Artifact::ObjectInfo const& { + return id_; + } + [[nodiscard]] auto Id() && -> Artifact::ObjectInfo { + return std::move(id_); + } + [[nodiscard]] auto operator==(TargetCacheKey const& other) const -> bool { + return this->Id() == other.Id(); + } + + private: + explicit TargetCacheKey(Artifact::ObjectInfo id) : id_{std::move(id)} {} + Artifact::ObjectInfo id_; +}; + +#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_CACHE_KEY_HPP diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS index 70e361a2..d139d1d8 100644 --- a/src/buildtool/main/TARGETS +++ b/src/buildtool/main/TARGETS @@ -12,6 +12,8 @@ , ["src/buildtool/progress_reporting", "base_progress_reporter"] , ["src/buildtool/build_engine/target_map", "result_map"] , ["src/buildtool/build_engine/target_map", "target_cache"] + , ["src/buildtool/build_engine/target_map", "target_cache_key"] + , ["src/buildtool/build_engine/target_map", "target_cache_entry"] , ["src/buildtool/build_engine/target_map", "target_map"] , ["src/buildtool/multithreading", "task_system"] , ["src/utils/cpp", "concepts"] diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index cf98bcf3..5bb707f5 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -25,6 +25,8 @@ #include "src/buildtool/build_engine/expression/evaluator.hpp" #include "src/buildtool/build_engine/expression/expression.hpp" #include "src/buildtool/build_engine/target_map/target_cache.hpp" +#include "src/buildtool/build_engine/target_map/target_cache_entry.hpp" +#include "src/buildtool/build_engine/target_map/target_cache_key.hpp" #include "src/buildtool/build_engine/target_map/target_map.hpp" #include "src/buildtool/common/artifact_description.hpp" #include "src/buildtool/common/cli.hpp" @@ -1073,8 +1075,8 @@ void DumpArtifactsToBuild( } auto CollectNonKnownArtifacts( - std::unordered_map<TargetCache::Key, AnalysedTargetPtr> const& - cache_targets) -> std::vector<ArtifactDescription> { + std::unordered_map<TargetCacheKey, AnalysedTargetPtr> const& cache_targets) + -> std::vector<ArtifactDescription> { auto cache_artifacts = std::unordered_set<ArtifactDescription>{}; for (auto const& [_, target] : cache_targets) { auto artifacts = target->ContainedNonKnownArtifacts(); @@ -1087,8 +1089,7 @@ auto CollectNonKnownArtifacts( #ifndef BOOTSTRAP_BUILD_TOOL void WriteTargetCacheEntries( - std::unordered_map<TargetCache::Key, AnalysedTargetPtr> const& - cache_targets, + std::unordered_map<TargetCacheKey, AnalysedTargetPtr> const& cache_targets, std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const& extra_infos, std::size_t jobs, @@ -1100,7 +1101,7 @@ void WriteTargetCacheEntries( for (auto const& [key, target] : cache_targets) { ts.QueueTask([&key = key, &target = target, &extra_infos]() { if (auto entry = - TargetCache::Entry::FromTarget(target, extra_infos)) { + TargetCacheEntry::FromTarget(target, extra_infos)) { if (not TargetCache::Instance().Store(key, *entry)) { Logger::Log(LogLevel::Warning, "Failed writing target cache entry for {}", |