diff options
| -rw-r--r-- | src/buildtool/computed_roots/TARGETS | 8 | ||||
| -rw-r--r-- | src/buildtool/computed_roots/evaluate.cpp | 75 | ||||
| -rw-r--r-- | src/buildtool/computed_roots/lookup_cache.cpp | 138 | ||||
| -rw-r--r-- | src/buildtool/computed_roots/lookup_cache.hpp | 35 | ||||
| -rw-r--r-- | test/end-to-end/computed-roots/basic.sh | 2 | ||||
| -rw-r--r-- | test/end-to-end/computed-roots/mr_computed_setup.sh | 3 |
6 files changed, 254 insertions, 7 deletions
diff --git a/src/buildtool/computed_roots/TARGETS b/src/buildtool/computed_roots/TARGETS index 0bf40d6c..46f9ea0b 100644 --- a/src/buildtool/computed_roots/TARGETS +++ b/src/buildtool/computed_roots/TARGETS @@ -46,7 +46,8 @@ { "type": ["@", "rules", "CC", "library"] , "name": ["evaluate"] , "hdrs": ["evaluate.hpp"] - , "srcs": ["evaluate.cpp"] + , "private-hdrs": ["lookup_cache.hpp"] + , "srcs": ["evaluate.cpp", "lookup_cache.cpp"] , "stage": ["src", "buildtool", "computed_roots"] , "deps": [ ["@", "gsl", "", "gsl"] @@ -57,10 +58,15 @@ ] , "private-deps": [ "analyse_and_build" + , "artifacts_root" , ["@", "fmt", "", "fmt"] , ["@", "json", "", "json"] , ["src/buildtool/build_engine/base_maps", "entity_name_data"] + , ["src/buildtool/build_engine/base_maps", "field_reader"] + , ["src/buildtool/build_engine/base_maps", "module_name"] + , ["src/buildtool/build_engine/base_maps", "targets_file_map"] , ["src/buildtool/build_engine/expression", "expression"] + , ["src/buildtool/build_engine/expression", "expression_ptr_interface"] , ["src/buildtool/build_engine/target_map", "configured_target"] , ["src/buildtool/common", "cli"] , ["src/buildtool/common", "common"] diff --git a/src/buildtool/computed_roots/evaluate.cpp b/src/buildtool/computed_roots/evaluate.cpp index b20f144e..06beccec 100644 --- a/src/buildtool/computed_roots/evaluate.cpp +++ b/src/buildtool/computed_roots/evaluate.cpp @@ -26,6 +26,7 @@ #include <shared_mutex> #include <sstream> #include <utility> +#include <variant> #include <vector> #include "fmt/core.h" @@ -38,6 +39,7 @@ #include "src/buildtool/common/cli.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/computed_roots/analyse_and_build.hpp" +#include "src/buildtool/computed_roots/lookup_cache.hpp" #include "src/buildtool/file_system/file_root.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_cas.hpp" @@ -235,6 +237,27 @@ auto ImportToGitCas(std::filesystem::path root_dir, return *std::move(res); } +auto IsTreePresent(std::string const& tree_id, + gsl::not_null<const StorageConfig*> const& storage_config, + GitRepo::anon_logger_ptr const& logger) + -> std::optional<bool> { + auto just_git_cas = GitCAS::Open(storage_config->GitRoot()); + if (not just_git_cas) { + (*logger)(fmt::format("Failed to open Git ODB at {}", + storage_config->GitRoot().string()), + true); + return std::nullopt; + } + auto just_git_repo = GitRepo::Open(just_git_cas); + if (not just_git_repo) { + (*logger)(fmt::format("Failed to open Git repository at {}", + storage_config->GitRoot().string()), + true); + return std::nullopt; + } + return just_git_repo->CheckTreeExists(tree_id, logger); +} + void ComputeAndFill( FileRoot::ComputedRoot const& key, gsl::not_null<RepositoryConfig*> const& repository_config, @@ -277,15 +300,59 @@ void ComputeAndFill( context->remote_context, &statistics, &progress}; + + auto cache_lookup = + expected<std::optional<std::string>, std::monostate>(std::nullopt); + { + std::shared_lock computing{*config_lock}; + cache_lookup = LookupCache(target, repository_config, storage, logger); + } + if (not cache_lookup) { + // prerequisite failure; fatal logger call already handled by + // LookupCache + return; + } + if (*cache_lookup) { + std::string root = **cache_lookup; + auto wrapped_logger = std::make_shared<GitRepo::anon_logger_t>( + [&logger, &root](auto const& msg, bool fatal) { + (*logger)(fmt::format("While checking presence of tree {} in " + "local git repo:\n{}", + root, + msg), + fatal); + }); + auto tree_present = IsTreePresent(root, storage_config, wrapped_logger); + if (not tree_present) { + // fatal error; logger already called by IsTreePresent + return; + } + if (*tree_present) { + Logger::Log(LogLevel::Performance, + "Root {} taken from cache to be {}", + target.ToString(), + root); + auto root_result = + FileRoot::FromGit(storage_config->GitRoot(), root); + if (not root_result) { + (*logger)(fmt::format("Failed to create git root for {}", root), + /*fatal=*/true); + return; + } + { + std::unique_lock setting{*config_lock}; + repository_config->SetComputedRoot(key, *root_result); + } + (*setter)(std::move(root)); + return; + } + } + GraphTraverser traverser{ root_build_args, &root_exec_context, reporter, &build_logger}; std::optional<AnalyseAndBuildResult> build_result{}; { std::shared_lock computing{*config_lock}; - // TODO(aehlig): ensure that target is an export target of a - // content-fixed repo - // TODO(aehlig): avoid installing and importing to git if the - // resulting root is available already build_result = AnalyseAndBuild(&analyse_context, traverser, target, diff --git a/src/buildtool/computed_roots/lookup_cache.cpp b/src/buildtool/computed_roots/lookup_cache.cpp new file mode 100644 index 00000000..9021463c --- /dev/null +++ b/src/buildtool/computed_roots/lookup_cache.cpp @@ -0,0 +1,138 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "src/buildtool/computed_roots/lookup_cache.hpp" + +#include <functional> +#include <memory> +#include <vector> + +#include "fmt/core.h" +#include "nlohmann/json.hpp" +#include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" +#include "src/buildtool/build_engine/base_maps/field_reader.hpp" +#include "src/buildtool/build_engine/base_maps/module_name.hpp" +#include "src/buildtool/build_engine/base_maps/targets_file_map.hpp" +#include "src/buildtool/build_engine/expression/configuration.hpp" +#include "src/buildtool/build_engine/expression/target_result.hpp" +#include "src/buildtool/computed_roots/artifacts_root.hpp" +#include "src/buildtool/file_system/file_root.hpp" +#include "src/buildtool/multithreading/task_system.hpp" +#include "src/buildtool/storage/target_cache_entry.hpp" +#include "src/buildtool/storage/target_cache_key.hpp" + +auto LookupCache(BuildMaps::Target::ConfiguredTarget const& ctarget, + gsl::not_null<RepositoryConfig*> const& repository_config, + Storage const& storage, + AsyncMapConsumerLoggerPtr const& logger) + -> expected<std::optional<std::string>, std::monostate> { + auto const* target_root = + repository_config->TargetRoot(ctarget.target.ToModule().repository); + if ((target_root == nullptr) or target_root->IsAbsent()) { + // TODO(aehlig): avoid local installing in case of absent target root of + // the base repository + return expected<std::optional<std::string>, std::monostate>( + std::nullopt); + } + auto repo_key = repository_config->RepositoryKey( + storage, ctarget.target.GetNamedTarget().repository); + if (not repo_key) { + (*logger)(fmt::format("Repository {} is not content-fixed", + nlohmann::json( + ctarget.target.GetNamedTarget().repository)), + /*fatal=*/true); + return unexpected(std::monostate{}); + } + auto targets_file_map = + BuildMaps::Base::CreateTargetsFileMap(repository_config, 1); + nlohmann::json targets_file{}; + bool failed{false}; + { + TaskSystem ts{1}; + targets_file_map.ConsumeAfterKeysReady( + &ts, + {ctarget.target.ToModule()}, + [&targets_file](auto values) { targets_file = *values[0]; }, + [&logger, &failed](auto const& msg, bool fatal) { + (*logger)( + fmt::format("While searching for target description:\n{}", + msg), + fatal); + failed = failed or fatal; + }); + } + if (failed) { + return unexpected(std::monostate{}); + } + auto desc_it = targets_file.find(ctarget.target.GetNamedTarget().name); + if (desc_it == targets_file.end()) { + (*logger)("Not referring to a defined target", /*fatal=*/true); + return unexpected(std::monostate{}); + } + nlohmann::json const& desc = *desc_it; + auto rule_it = desc.find("type"); + if (rule_it == desc.end()) { + (*logger)(fmt::format("No type specified in target-description {}", + desc.dump()), + true); + return unexpected(std::monostate{}); + } + auto const& rule = *rule_it; + if (not(rule.is_string() and rule.get<std::string>() == "export")) { + (*logger)(fmt::format("Target not an export target, but of type {}", + rule.dump()), + true); + return unexpected(std::monostate{}); + } + auto reader = BuildMaps::Base::FieldReader::CreatePtr( + desc, ctarget.target, "export target", logger); + auto flexible_vars = reader->ReadStringList("flexible_config"); + if (not flexible_vars) { + return unexpected(std::monostate{}); + } + auto effective_config = ctarget.config.Prune(*flexible_vars); + auto cache_key = storage.TargetCache().ComputeKey( + *repo_key, ctarget.target.GetNamedTarget(), effective_config); + if (not cache_key) { + (*logger)("Target-cache key generation failed", true); + return unexpected(std::monostate{}); + } + auto target_cache_value = storage.TargetCache().Read(*cache_key); + if (not target_cache_value) { + return expected<std::optional<std::string>, std::monostate>( + std::nullopt); + } + auto const& [entry, info] = *target_cache_value; + auto result = entry.ToResult(); + if (not result) { + (*logger)(fmt::format("Failed to deserialize cache entry {} for key {}", + info.ToString(), + cache_key->Id().ToString()), + true); + return unexpected(std::monostate{}); + } + + auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>( + [&logger](auto const& msg, bool fatal) { + (*logger)( + fmt::format("While computing git tree for artifacts stage:\n{}", + msg), + fatal); + }); + auto root = ArtifactsRoot(result->artifact_stage, wrapped_logger); + if (not root) { + return unexpected(std::monostate{}); + } + return root; +} diff --git a/src/buildtool/computed_roots/lookup_cache.hpp b/src/buildtool/computed_roots/lookup_cache.hpp new file mode 100644 index 00000000..0f8096d1 --- /dev/null +++ b/src/buildtool/computed_roots/lookup_cache.hpp @@ -0,0 +1,35 @@ +// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_LOOKUP_CACHE_HPP +#define INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_LOOKUP_CACHE_HPP + +#include <optional> +#include <string> +#include <variant> + +#include "gsl/gsl" +#include "src/buildtool/build_engine/target_map/configured_target.hpp" +#include "src/buildtool/common/repository_config.hpp" +#include "src/buildtool/multithreading/async_map_consumer.hpp" +#include "src/buildtool/storage/storage.hpp" +#include "src/utils/cpp/expected.hpp" + +auto LookupCache(BuildMaps::Target::ConfiguredTarget const& ctarget, + gsl::not_null<RepositoryConfig*> const& repository_config, + Storage const& storage, + AsyncMapConsumerLoggerPtr const& logger) + -> expected<std::optional<std::string>, std::monostate>; + +#endif diff --git a/test/end-to-end/computed-roots/basic.sh b/test/end-to-end/computed-roots/basic.sh index 59470ef4..16e363c1 100644 --- a/test/end-to-end/computed-roots/basic.sh +++ b/test/end-to-end/computed-roots/basic.sh @@ -140,6 +140,6 @@ echo --log-limit 4 -f "${OUT}/log2" \ --main 'other derived' 2>&1 echo -grep '[Ee]xport.*from cache' "${OUT}/log2" +grep '[Rr]oot.*from cache' "${OUT}/log2" echo OK diff --git a/test/end-to-end/computed-roots/mr_computed_setup.sh b/test/end-to-end/computed-roots/mr_computed_setup.sh index 02b2e0ce..47b17e42 100644 --- a/test/end-to-end/computed-roots/mr_computed_setup.sh +++ b/test/end-to-end/computed-roots/mr_computed_setup.sh @@ -73,6 +73,7 @@ cat > repo-config.json <<EOF { "repository": { "type": "file" , "path": "${BASE_ROOT}" + , "pragma": {"to_git": true} } } , "derived": @@ -127,4 +128,4 @@ echo [ "$(cat "${OUT}/other-derived/out" | wc -l)" -eq 78 ] -echo OK
\ No newline at end of file +echo OK |