summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/buildtool/computed_roots/TARGETS31
-rw-r--r--src/buildtool/computed_roots/evaluate.cpp74
-rw-r--r--src/buildtool/computed_roots/inquire_serve.cpp143
-rw-r--r--src/buildtool/computed_roots/inquire_serve.hpp35
-rw-r--r--src/buildtool/computed_roots/lookup_cache.cpp2
5 files changed, 283 insertions, 2 deletions
diff --git a/src/buildtool/computed_roots/TARGETS b/src/buildtool/computed_roots/TARGETS
index fa216d22..a0a7aa8c 100644
--- a/src/buildtool/computed_roots/TARGETS
+++ b/src/buildtool/computed_roots/TARGETS
@@ -22,6 +22,36 @@
, ["src/buildtool/storage", "storage"]
]
}
+, "inquire_serve":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["inquire_serve"]
+ , "hdrs": ["inquire_serve.hpp"]
+ , "srcs": ["inquire_serve.cpp"]
+ , "deps":
+ [ ["@", "gsl", "", "gsl"]
+ , ["src/buildtool/build_engine/target_map", "configured_target"]
+ , ["src/buildtool/execution_api/common", "api_bundle"]
+ , ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/main", "analyse_context"]
+ ]
+ , "private-deps":
+ [ "artifacts_root"
+ , ["@", "json", "", "json"]
+ , ["src/buildtool/build_engine/base_maps", "entity_name_data"]
+ , ["src/buildtool/build_engine/base_maps", "module_name"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/build_engine/expression", "expression_ptr_interface"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/common", "config"]
+ , ["src/buildtool/execution_api/utils", "rehash_utils"]
+ , ["src/buildtool/file_system", "file_root"]
+ , ["src/buildtool/logging", "log_level"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ , ["src/buildtool/serve_api/remote", "serve_api"]
+ , ["src/buildtool/storage", "storage"]
+ ]
+ , "stage": ["src", "buildtool", "computed_roots"]
+ }
, "artifacts_root":
{ "type": ["@", "rules", "CC", "library"]
, "name": ["artifacts_root"]
@@ -63,6 +93,7 @@
, "private-deps":
[ "analyse_and_build"
, "artifacts_root"
+ , "inquire_serve"
, ["@", "fmt", "", "fmt"]
, ["@", "json", "", "json"]
, ["src/buildtool/build_engine/base_maps", "entity_name_data"]
diff --git a/src/buildtool/computed_roots/evaluate.cpp b/src/buildtool/computed_roots/evaluate.cpp
index 5df902fd..ebcbed8a 100644
--- a/src/buildtool/computed_roots/evaluate.cpp
+++ b/src/buildtool/computed_roots/evaluate.cpp
@@ -42,6 +42,7 @@
#include "src/buildtool/common/protocol_traits.hpp"
#include "src/buildtool/common/statistics.hpp"
#include "src/buildtool/computed_roots/analyse_and_build.hpp"
+#include "src/buildtool/computed_roots/inquire_serve.hpp"
#include "src/buildtool/computed_roots/lookup_cache.hpp"
#include "src/buildtool/crypto/hash_function.hpp"
#include "src/buildtool/execution_api/common/api_bundle.hpp"
@@ -174,6 +175,7 @@ void ComputeAndFill(
std::size_t jobs,
RootMap::LoggerPtr const& logger,
RootMap::SetterPtr const& setter) {
+
auto tmpdir = storage_config->CreateTypedTmpDir("computed-root");
if (not tmpdir) {
(*logger)("Failed to create temporary directory", true);
@@ -257,6 +259,34 @@ void ComputeAndFill(
}
}
+ if (key.absent) {
+ if (storage_config->hash_function.GetType() !=
+ HashFunction::Type::GitSHA1) {
+ Logger::Log(LogLevel::Performance,
+ "Computing root {} locally as rehahing would have to "
+ "be done locally",
+ key.ToString());
+ }
+ else {
+ auto serve_result = InquireServe(
+ &analyse_context, target, context->apis, &build_logger);
+ if (serve_result) {
+ auto root_result = FileRoot(*serve_result);
+ Logger::Log(LogLevel::Performance,
+ "Absent root {} obtained from serve to be {}",
+ target.ToString(),
+ *serve_result);
+ {
+ std::unique_lock setting{*config_lock};
+ repository_config->SetPrecomputedRoot(PrecomputedRoot{key},
+ root_result);
+ }
+ (*setter)(std::move(*serve_result));
+ return;
+ }
+ }
+ }
+
GraphTraverser traverser{
root_build_args, &root_exec_context, reporter, &build_logger};
std::optional<AnalyseAndBuildResult> build_result{};
@@ -314,6 +344,50 @@ void ComputeAndFill(
/*fatal=*/true);
return;
}
+ if (key.absent) {
+ if (serve == nullptr) {
+ (*logger)(fmt::format("Requested root {} to be absent, without "
+ "providing serve endpoint",
+ key.ToString()),
+ /*fatal=*/true);
+ return;
+ }
+ auto known = serve->CheckRootTree(*result);
+ if (known.has_value() and not *known) {
+ auto tree_digest =
+ ArtifactDigestFactory::Create(HashFunction::Type::GitSHA1,
+ *result,
+ /*size_unknown=*/0,
+ /*is_tree=*/true);
+ if (not tree_digest) {
+ (*logger)(
+ fmt::format("Internal error getting digest for tree {}: {}",
+ *known,
+ tree_digest.error()),
+ /*fatal=*/true);
+ return;
+ }
+ auto uploaded_to_serve = UploadToServe(*serve,
+ *context->apis,
+ *storage_config,
+ *tree_digest,
+ storage_config->GitRoot());
+ if (not uploaded_to_serve) {
+ (*logger)(fmt::format("Failed to sync {} to serve:{}",
+ *result,
+ uploaded_to_serve.error()),
+ /*fatal=*/true);
+ return;
+ }
+ Logger::Log(LogLevel::Performance, "Uploaded {} to serve", *result);
+ known = true;
+ }
+ if (not known.has_value() or not *known) {
+ (*logger)(
+ fmt::format("Failed to ensure {} is known to serve", *result),
+ /*fatal=*/true);
+ }
+ }
{
// For setting, we need an exclusiver lock; so get one after we
// dropped the shared one
diff --git a/src/buildtool/computed_roots/inquire_serve.cpp b/src/buildtool/computed_roots/inquire_serve.cpp
new file mode 100644
index 00000000..5f5c12ef
--- /dev/null
+++ b/src/buildtool/computed_roots/inquire_serve.cpp
@@ -0,0 +1,143 @@
+// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/computed_roots/inquire_serve.hpp"
+
+#include <filesystem>
+#include <memory>
+#include <utility>
+#include <variant>
+#include <vector>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name_data.hpp"
+#include "src/buildtool/build_engine/base_maps/module_name.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
+#include "src/buildtool/build_engine/expression/target_result.hpp"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/computed_roots/artifacts_root.hpp"
+#include "src/buildtool/execution_api/utils/rehash_utils.hpp"
+#include "src/buildtool/file_system/file_root.hpp"
+#include "src/buildtool/logging/log_level.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+#include "src/buildtool/serve_api/remote/serve_api.hpp"
+#include "src/buildtool/storage/storage.hpp"
+#include "src/buildtool/storage/target_cache_entry.hpp"
+#include "src/buildtool/storage/target_cache_key.hpp"
+
+[[nodiscard]] auto InquireServe(
+ gsl::not_null<AnalyseContext*> const& analyse_context,
+ BuildMaps::Target::ConfiguredTarget const& id,
+ gsl::not_null<ApiBundle const*> const& /* apis */,
+ gsl::not_null<Logger const*> const& logger) -> std::optional<std::string> {
+
+ auto const& repo_name = id.target.ToModule().repository;
+ if (not analyse_context->repo_config->TargetRoot(repo_name)->IsAbsent()) {
+ logger->Emit(LogLevel::Info,
+ "Base root is concrete, will manage build locally.");
+ return std::nullopt;
+ }
+
+ if (analyse_context->serve == nullptr) {
+ logger->Emit(LogLevel::Warning,
+ "Cannot treat a root absent without serve");
+ return std::nullopt;
+ }
+
+ auto target = id.target.GetNamedTarget();
+
+ auto target_root_id =
+ analyse_context->repo_config->TargetRoot(repo_name)->GetAbsentTreeId();
+ if (not target_root_id) {
+ logger->Emit(LogLevel::Warning,
+ "Failed to get the target root id for repository {}",
+ nlohmann::json(repo_name).dump());
+ return std::nullopt;
+ }
+ std::filesystem::path module{id.target.ToModule().module};
+ auto vars = analyse_context->serve->ServeTargetVariables(
+ *target_root_id,
+ (module / *(analyse_context->repo_config->TargetFileName(repo_name)))
+ .string(),
+ target.name);
+
+ if (not vars) {
+ logger->Emit(LogLevel::Warning,
+ "Failed to obtain variables for {}",
+ id.target.ToString());
+ return std::nullopt;
+ }
+
+ auto effective_config = id.config.Prune(*vars);
+ logger->Emit(LogLevel::Info,
+ "Effective configuration {}",
+ effective_config.ToString());
+
+ auto repo_key = analyse_context->repo_config->RepositoryKey(
+ *analyse_context->storage, target.repository);
+ if (not repo_key) {
+ logger->Emit(LogLevel::Warning, "Cannot obtain repository key");
+ return std::nullopt;
+ }
+ auto target_cache_key = analyse_context->storage->TargetCache().ComputeKey(
+ *repo_key, target, effective_config);
+
+ if (not target_cache_key) {
+ logger->Emit(LogLevel::Warning, "Failed to obtain target-cache key");
+ return std::nullopt;
+ }
+
+ logger->Emit(LogLevel::Info,
+ "Target cache key {}",
+ target_cache_key->Id().ToString());
+
+ auto res = analyse_context->serve->ServeTarget(
+ *target_cache_key, *repo_key, /*keep_artifacts_root=*/true);
+ if (not res) {
+ logger->Emit(LogLevel::Warning, "Could not obtain target from serve");
+ return std::nullopt;
+ }
+ if (res->index() != 3) {
+ logger->Emit(LogLevel::Warning, "Failed to obtain root from serve");
+ return std::nullopt;
+ }
+ auto target_cache_value = std::get<3>(*res);
+ auto const& [entry, info] = target_cache_value;
+ auto result = entry.ToResult();
+ if (not result) {
+ logger->Emit(LogLevel::Warning, "Reading entry cache entry failed.");
+ return std::nullopt;
+ }
+
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [&logger](auto const& msg, bool fatal) {
+ logger->Emit(fatal ? LogLevel::Warning : LogLevel::Info,
+ "While computing root from stage:{}",
+ msg);
+ });
+ auto git_tree = ArtifactsRoot(
+ result->artifact_stage, wrapped_logger, /*rehash=*/std::nullopt);
+ if (not git_tree) {
+ logger->Emit(
+ LogLevel::Warning,
+ "Failed to compute git tree from obtained artifact stage {}",
+ result->artifact_stage->ToString());
+ return std::nullopt;
+ }
+ logger->Emit(LogLevel::Info, "Tree identifier for root is {}.", *git_tree);
+ return git_tree;
+}
diff --git a/src/buildtool/computed_roots/inquire_serve.hpp b/src/buildtool/computed_roots/inquire_serve.hpp
new file mode 100644
index 00000000..cec39400
--- /dev/null
+++ b/src/buildtool/computed_roots/inquire_serve.hpp
@@ -0,0 +1,35 @@
+// Copyright 2025 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDOOL_COMPUTED_ROOTS_INQUIRE_SERVE_HPP
+#define INCLUDED_SRC_BUILDOOL_COMPUTED_ROOTS_INQUIRE_SERVE_HPP
+
+#include <optional>
+#include <string>
+
+#include "gsl/gsl"
+#include "src/buildtool/build_engine/target_map/configured_target.hpp"
+#include "src/buildtool/execution_api/common/api_bundle.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/main/analyse_context.hpp"
+
+// Inquire serve for a given target and report the artifact stage as git tree
+// identifier.
+[[nodiscard]] auto InquireServe(
+ gsl::not_null<AnalyseContext*> const& analyse_context,
+ BuildMaps::Target::ConfiguredTarget const& id,
+ gsl::not_null<ApiBundle const*> const& apis,
+ gsl::not_null<Logger const*> const& logger) -> std::optional<std::string>;
+
+#endif
diff --git a/src/buildtool/computed_roots/lookup_cache.cpp b/src/buildtool/computed_roots/lookup_cache.cpp
index 0555c74b..ac3eeaef 100644
--- a/src/buildtool/computed_roots/lookup_cache.cpp
+++ b/src/buildtool/computed_roots/lookup_cache.cpp
@@ -41,8 +41,6 @@ auto LookupCache(BuildMaps::Target::ConfiguredTarget const& ctarget,
auto const* target_root =
repository_config->TargetRoot(ctarget.target.ToModule().repository);
if ((target_root == nullptr) or target_root->IsAbsent()) {
- // TODO(aehlig): avoid local installing in case of absent target root of
- // the base repository
return expected<std::optional<std::string>, std::monostate>(
std::nullopt);
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 17:02:01 +0000