// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "src/buildtool/build_engine/target_map/absent_target_map.hpp" #ifndef BOOTSTRAP_BUILD_TOOL #include #include #include #include #include #include // std::move #include #include "fmt/core.h" #include "src/buildtool/build_engine/analysed_target/target_graph_information.hpp" #include "src/buildtool/build_engine/base_maps/entity_name_data.hpp" #include "src/buildtool/build_engine/expression/configuration.hpp" #include "src/buildtool/common/action_description.hpp" #include "src/buildtool/common/artifact.hpp" #include "src/buildtool/common/artifact_digest.hpp" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" #include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/storage.hpp" #include "src/buildtool/storage/target_cache_entry.hpp" #include "src/buildtool/storage/target_cache_key.hpp" #include "src/utils/cpp/json.hpp" #endif // BOOTSTRAP_BUILD_TOOL #ifndef BOOTSTRAP_BUILD_TOOL namespace { void WithFlexibleVariables( const gsl::not_null& context, const BuildMaps::Target::ConfiguredTarget& key, std::vector flexible_vars, const BuildMaps::Target::AbsentTargetMap::SubCallerPtr& subcaller, const BuildMaps::Target::AbsentTargetMap::SetterPtr& setter, const BuildMaps::Target::AbsentTargetMap::LoggerPtr& logger, const gsl::not_null result_map, BuildMaps::Target::ServeFailureLogReporter* serve_failure_reporter) { auto effective_config = key.config.Prune(flexible_vars); if (key.config != effective_config) { (*subcaller)( {BuildMaps::Target::ConfiguredTarget{.target = key.target, .config = effective_config}}, [setter](auto const& values) { AnalysedTargetPtr result = *(values[0]); (*setter)(std::move(result)); }, logger); return; } // TODO(asartori): avoid code duplication in export.cpp context->statistics->IncrementExportsFoundCounter(); auto target_name = key.target.GetNamedTarget(); auto repo_key = context->repo_config->RepositoryKey(*context->storage, target_name.repository); if (not repo_key) { (*logger)(fmt::format("Failed to obtain repository key for repo \"{}\"", target_name.repository), /*fatal=*/true); return; } auto target_cache_key = context->storage->TargetCache().ComputeKey( *repo_key, target_name, effective_config); if (not target_cache_key) { (*logger)(fmt::format("Could not produce cache key for target {}", key.target.ToString()), /*fatal=*/true); return; } std::optional> target_cache_value{std::nullopt}; target_cache_value = context->storage->TargetCache().Read(*target_cache_key); bool from_just_serve = false; if (not target_cache_value and context->serve != nullptr) { auto task = fmt::format("[{},{}]", key.target.ToString(), PruneJson(effective_config.ToJson()).dump()); Logger::Log( LogLevel::Debug, "Querying serve endpoint for absent export target {} with key {}", task, key.target.ToString()); context->progress->TaskTracker().Start(task); auto res = context->serve->ServeTarget(*target_cache_key, *repo_key); // process response from serve endpoint if (not res) { // report target not found (*logger)(fmt::format("Absent target {} was not found on serve " "endpoint", key.target.ToString()), /*fatal=*/true); return; } switch (auto const& ind = res->index(); ind) { case 0: { if (serve_failure_reporter != nullptr) { (*serve_failure_reporter)(key, std::get<0>(*res)); } // target found but failed to analyse/build: log it as fatal (*logger)( fmt::format("Failure to remotely analyse or build absent " "target {}\nDetailed log available on the " "remote-execution endpoint as blob {}", key.target.ToString(), std::get<0>(*res)), /*fatal=*/true); return; } case 1: // fallthrough case 2: { // Other errors, including INTERNAL: log it as fatal (*logger)(fmt::format( "While querying serve endpoint for absent export " "target {}:\n{}", key.target.ToString(), ind == 1 ? std::get<1>(*res) : std::get<2>(*res)), /*fatal=*/true); return; } default: { // index == 3 target_cache_value = std::get<3>(*res); context->progress->TaskTracker().Stop(task); from_just_serve = true; } } } if (not target_cache_value) { (*logger)(fmt::format("Could not get target cache value for key {}", target_cache_key->Id().ToString()), /*fatal=*/true); return; } auto const& [entry, info] = *target_cache_value; if (auto result = entry.ToResult()) { auto deps_info = TargetGraphInformation{ std::make_shared( BuildMaps::Target::ConfiguredTarget{ .target = key.target, .config = effective_config}), {}, {}, {}}; auto analysis_result = std::make_shared( *result, std::vector{}, std::vector{}, std::vector{}, std::vector{}, std::unordered_set{flexible_vars.begin(), flexible_vars.end()}, std::set{}, entry.ToImplied(), deps_info); analysis_result = result_map->Add( key.target, effective_config, analysis_result, std::nullopt, true); Logger::Log(LogLevel::Performance, "Absent export target {} taken from {}: {} -> {}", key.target.ToString(), (from_just_serve ? "serve endpoint" : "cache"), target_cache_key->Id().ToString(), info.ToString()); (*setter)(std::move(analysis_result)); if (from_just_serve) { context->statistics->IncrementExportsServedCounter(); } else { context->statistics->IncrementExportsCachedCounter(); } return; } (*logger)(fmt::format("Reading target entry for key {} failed", target_cache_key->Id().ToString()), /*fatal=*/true); } } // namespace #endif // BOOTSTRAP_BUILD_TOOL auto BuildMaps::Target::CreateAbsentTargetVariablesMap( const gsl::not_null& context, std::size_t jobs) -> AbsentTargetVariablesMap { #ifdef BOOTSTRAP_BUILD_TOOL auto target_variables = [](auto /*ts*/, auto /*setter*/, auto /*logger*/, auto /*subcaller*/, auto /*key*/) {}; #else auto target_variables = [context](auto /*ts*/, auto setter, auto logger, auto /*subcaller*/, auto key) { std::optional> vars; if (context->serve != nullptr) { vars = context->serve->ServeTargetVariables( key.target_root_id, key.target_file, key.target); } if (not vars) { (*logger)(fmt::format("Failed to obtain flexible config variables " "for absent target {}", key.target), /*fatal=*/true); return; } (*setter)(std::move(vars.value())); }; #endif return AbsentTargetVariablesMap{target_variables, jobs}; } auto BuildMaps::Target::CreateAbsentTargetMap( const gsl::not_null& context, const gsl::not_null& result_map, const gsl::not_null& absent_variables, std::size_t jobs, BuildMaps::Target::ServeFailureLogReporter* serve_failure_reporter) -> AbsentTargetMap { #ifndef BOOTSTRAP_BUILD_TOOL auto target_reader = [context, result_map, absent_variables, serve_failure_reporter]( auto ts, auto setter, auto logger, auto subcaller, auto key) { // assumptions: // - target with absent targets file requested // - ServeApi correctly configured auto const& repo_name = key.target.ToModule().repository; auto target_root_id = context->repo_config->TargetRoot(repo_name)->GetAbsentTreeId(); if (not target_root_id) { (*logger)(fmt::format("Failed to get the target root id for " "repository \"{}\"", repo_name), /*fatal=*/true); return; } std::filesystem::path module{key.target.ToModule().module}; auto vars_request = AbsentTargetDescription{ .target_root_id = *target_root_id, .target_file = (module / *(context->repo_config->TargetFileName( repo_name))) .string(), .target = key.target.GetNamedTarget().name}; absent_variables->ConsumeAfterKeysReady( ts, {vars_request}, [context, key, setter, logger, serve_failure_reporter, result_map, subcaller](auto const& values) { WithFlexibleVariables(context, key, *(values[0]), subcaller, setter, logger, result_map, serve_failure_reporter); }, [logger, target = key.target](auto const& msg, auto fatal) { (*logger)(fmt::format("While requested the flexible " "variables of {}:\n{}", target.ToString(), msg), fatal); }); }; #else auto target_reader = [](auto /*ts*/, auto /*setter*/, auto /*logger*/, auto /*subcaller*/, auto /*key*/) {}; #endif return AbsentTargetMap{target_reader, jobs}; }