summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/execution_engine/executor/TARGETS1
-rw-r--r--src/buildtool/execution_engine/executor/executor.hpp27
-rw-r--r--src/buildtool/graph_traverser/graph_traverser.hpp32
-rw-r--r--src/buildtool/main/main.cpp15
-rw-r--r--src/buildtool/serve_api/serve_service/target.cpp56
-rw-r--r--src/buildtool/serve_api/serve_service/target.hpp7
6 files changed, 98 insertions, 40 deletions
diff --git a/src/buildtool/execution_engine/executor/TARGETS b/src/buildtool/execution_engine/executor/TARGETS
index bc334b4c..97c7fea4 100644
--- a/src/buildtool/execution_engine/executor/TARGETS
+++ b/src/buildtool/execution_engine/executor/TARGETS
@@ -16,6 +16,7 @@
, ["src/utils/cpp", "hex_string"]
, ["@", "gsl", "", "gsl"]
, ["src/buildtool/common", "common"]
+ , ["src/buildtool/common/remote", "remote_common"]
]
, "stage": ["src", "buildtool", "execution_engine", "executor"]
}
diff --git a/src/buildtool/execution_engine/executor/executor.hpp b/src/buildtool/execution_engine/executor/executor.hpp
index 29b16a1d..4e33060c 100644
--- a/src/buildtool/execution_engine/executor/executor.hpp
+++ b/src/buildtool/execution_engine/executor/executor.hpp
@@ -22,9 +22,11 @@
#include <map>
#include <optional>
#include <type_traits>
+#include <utility>
#include <vector>
#include "gsl/gsl"
+#include "src/buildtool/common/remote/remote_common.hpp"
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/common/statistics.hpp"
#include "src/buildtool/common/tree.hpp"
@@ -49,6 +51,8 @@ class ExecutorImpl {
gsl::not_null<DependencyGraph::ActionNode const*> const& action,
gsl::not_null<IExecutionApi*> const& api,
std::map<std::string, std::string> const& properties,
+ std::vector<std::pair<std::map<std::string, std::string>,
+ ServerAddress>> const& dispatch_list,
std::chrono::milliseconds const& timeout,
IExecutionAction::CacheFlag cache_flag)
-> std::optional<IExecutionResponse::Ptr> {
@@ -94,7 +98,8 @@ class ExecutorImpl {
Statistics::Instance().IncrementActionsQueuedCounter();
}
- auto alternative_api = GetAlternativeEndpoint(properties);
+ auto alternative_api =
+ GetAlternativeEndpoint(properties, dispatch_list);
if (alternative_api) {
if (not api->RetrieveToCas(
std::vector<Artifact::ObjectInfo>{Artifact::ObjectInfo{
@@ -616,10 +621,11 @@ class ExecutorImpl {
private:
[[nodiscard]] static inline auto GetAlternativeEndpoint(
- const std::map<std::string, std::string>& properties)
+ const std::map<std::string, std::string>& properties,
+ const std::vector<std::pair<std::map<std::string, std::string>,
+ ServerAddress>>& dispatch_list)
-> std::unique_ptr<BazelApi> {
- for (auto const& [pred, endpoint] :
- RemoteExecutionConfig::DispatchList()) {
+ for (auto const& [pred, endpoint] : dispatch_list) {
bool match = true;
for (auto const& [k, v] : pred) {
auto v_it = properties.find(k);
@@ -655,11 +661,14 @@ class Executor {
IExecutionApi* local_api,
IExecutionApi* remote_api,
std::map<std::string, std::string> properties,
+ std::vector<std::pair<std::map<std::string, std::string>,
+ ServerAddress>> dispatch_list,
std::chrono::milliseconds timeout = IExecutionAction::kDefaultTimeout)
: repo_config_{repo_config},
local_api_{local_api},
remote_api_{remote_api},
properties_{std::move(properties)},
+ dispatch_list_{std::move(dispatch_list)},
timeout_{timeout} {}
/// \brief Run an action in a blocking manner
@@ -676,6 +685,7 @@ class Executor {
action,
remote_api_,
Impl::MergeProperties(properties_, action->ExecutionProperties()),
+ dispatch_list_,
Impl::ScaleTime(timeout_, action->TimeoutScale()),
action->NoCache() ? CF::DoNotCacheOutput : CF::CacheOutput);
@@ -700,6 +710,8 @@ class Executor {
gsl::not_null<IExecutionApi*> local_api_;
gsl::not_null<IExecutionApi*> remote_api_;
std::map<std::string, std::string> properties_;
+ std::vector<std::pair<std::map<std::string, std::string>, ServerAddress>>
+ dispatch_list_;
std::chrono::milliseconds timeout_;
};
@@ -720,12 +732,15 @@ class Rebuilder {
IExecutionApi* remote_api,
IExecutionApi* api_cached,
std::map<std::string, std::string> properties,
+ std::vector<std::pair<std::map<std::string, std::string>,
+ ServerAddress>> dispatch_list,
std::chrono::milliseconds timeout = IExecutionAction::kDefaultTimeout)
: repo_config_{repo_config},
local_api_{local_api},
remote_api_{remote_api},
api_cached_{api_cached},
properties_{std::move(properties)},
+ dispatch_list_{std::move(dispatch_list)},
timeout_{timeout} {}
[[nodiscard]] auto Process(
@@ -738,6 +753,7 @@ class Rebuilder {
action,
remote_api_,
Impl::MergeProperties(properties_, action->ExecutionProperties()),
+ dispatch_list_,
Impl::ScaleTime(timeout_, action->TimeoutScale()),
CF::PretendCached);
@@ -751,6 +767,7 @@ class Rebuilder {
action,
api_cached_,
Impl::MergeProperties(properties_, action->ExecutionProperties()),
+ dispatch_list_,
Impl::ScaleTime(timeout_, action->TimeoutScale()),
CF::FromCacheOnly);
@@ -791,6 +808,8 @@ class Rebuilder {
gsl::not_null<IExecutionApi*> remote_api_;
gsl::not_null<IExecutionApi*> api_cached_;
std::map<std::string, std::string> properties_;
+ std::vector<std::pair<std::map<std::string, std::string>, ServerAddress>>
+ dispatch_list_;
std::chrono::milliseconds timeout_;
mutable std::mutex m_;
mutable std::vector<std::string> cache_misses_{};
diff --git a/src/buildtool/graph_traverser/graph_traverser.hpp b/src/buildtool/graph_traverser/graph_traverser.hpp
index 205bac32..611ea3e0 100644
--- a/src/buildtool/graph_traverser/graph_traverser.hpp
+++ b/src/buildtool/graph_traverser/graph_traverser.hpp
@@ -26,6 +26,7 @@
#include <thread>
#include <unordered_map>
#include <utility>
+#include <vector>
#include "fmt/core.h"
#include "gsl/gsl"
@@ -69,21 +70,33 @@ class GraphTraverser {
bool failed_artifacts;
};
- explicit GraphTraverser(CommandLineArguments clargs,
- gsl::not_null<RepositoryConfig*> const& repo_config)
+ explicit GraphTraverser(
+ CommandLineArguments clargs,
+ gsl::not_null<RepositoryConfig*> const& repo_config,
+ std::map<std::string, std::string> platform_properties,
+ std::vector<std::pair<std::map<std::string, std::string>,
+ ServerAddress>> dispatch_list)
: clargs_{std::move(clargs)},
repo_config_{repo_config},
+ platform_properties_{std::move(platform_properties)},
+ dispatch_list_{std::move(dispatch_list)},
local_api_{CreateExecutionApi(std::nullopt,
std::make_optional(repo_config))},
remote_api_{CreateExecutionApi(RemoteExecutionConfig::RemoteAddress(),
std::make_optional(repo_config))},
reporter_{[](auto done, auto cv) {}} {}
- explicit GraphTraverser(CommandLineArguments clargs,
- gsl::not_null<RepositoryConfig*> const& repo_config,
- progress_reporter_t reporter)
+ explicit GraphTraverser(
+ CommandLineArguments clargs,
+ gsl::not_null<RepositoryConfig*> const& repo_config,
+ std::map<std::string, std::string> platform_properties,
+ std::vector<std::pair<std::map<std::string, std::string>,
+ ServerAddress>> dispatch_list,
+ progress_reporter_t reporter)
: clargs_{std::move(clargs)},
repo_config_{repo_config},
+ platform_properties_{std::move(platform_properties)},
+ dispatch_list_{std::move(dispatch_list)},
local_api_{CreateExecutionApi(std::nullopt,
std::make_optional(repo_config))},
remote_api_{CreateExecutionApi(RemoteExecutionConfig::RemoteAddress(),
@@ -230,6 +243,9 @@ class GraphTraverser {
private:
CommandLineArguments const clargs_;
gsl::not_null<RepositoryConfig*> repo_config_;
+ std::map<std::string, std::string> platform_properties_;
+ std::vector<std::pair<std::map<std::string, std::string>, ServerAddress>>
+ dispatch_list_;
gsl::not_null<IExecutionApi::Ptr> const local_api_;
gsl::not_null<IExecutionApi::Ptr> const remote_api_;
progress_reporter_t reporter_;
@@ -350,7 +366,8 @@ class GraphTraverser {
Executor executor{repo_config_,
&(*local_api_),
&(*remote_api_),
- RemoteExecutionConfig::PlatformProperties(),
+ platform_properties_,
+ dispatch_list_,
clargs_.build.timeout};
bool traversing{};
std::atomic<bool> done = false;
@@ -380,7 +397,8 @@ class GraphTraverser {
&(*local_api_),
&(*remote_api_),
&(*api_cached),
- RemoteExecutionConfig::PlatformProperties(),
+ platform_properties_,
+ dispatch_list_,
clargs_.build.timeout};
bool traversing{false};
std::atomic<bool> done = false;
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index 3f5bfd76..d584f960 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -907,12 +907,15 @@ auto main(int argc, char* argv[]) -> int {
#ifndef BOOTSTRAP_BUILD_TOOL
SetupRetryConfig(arguments.retry);
- GraphTraverser const traverser{{jobs,
- std::move(arguments.build),
- std::move(stage_args),
- std::move(rebuild_args)},
- &repo_config,
- ProgressReporter::Reporter()};
+ GraphTraverser const traverser{
+ {jobs,
+ std::move(arguments.build),
+ std::move(stage_args),
+ std::move(rebuild_args)},
+ &repo_config,
+ RemoteExecutionConfig::PlatformProperties(),
+ RemoteExecutionConfig::DispatchList(),
+ ProgressReporter::Reporter()};
if (arguments.cmd == SubCommand::kInstallCas) {
if (not repo_config.SetGitCAS(StorageConfig::GitRoot())) {
diff --git a/src/buildtool/serve_api/serve_service/target.cpp b/src/buildtool/serve_api/serve_service/target.cpp
index b2cd06db..70373e05 100644
--- a/src/buildtool/serve_api/serve_service/target.cpp
+++ b/src/buildtool/serve_api/serve_service/target.cpp
@@ -14,8 +14,6 @@
#include "src/buildtool/serve_api/serve_service/target.hpp"
-#include <vector>
-
#include "fmt/core.h"
#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
#include "src/buildtool/build_engine/base_maps/entity_name_data.hpp"
@@ -25,7 +23,6 @@
#include "src/buildtool/build_engine/target_map/configured_target.hpp"
#include "src/buildtool/build_engine/target_map/result_map.hpp"
#include "src/buildtool/common/artifact.hpp"
-#include "src/buildtool/common/remote/remote_common.hpp"
#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/graph_traverser/graph_traverser.hpp"
#include "src/buildtool/main/analyse.hpp"
@@ -40,8 +37,8 @@
#include "src/utils/cpp/verify_hash.hpp"
auto TargetService::GetDispatchList(ArtifactDigest const& dispatch_digest)
- -> std::variant<::grpc::Status, nlohmann::json> {
- using result_t = std::variant<::grpc::Status, nlohmann::json>;
+ -> std::variant<::grpc::Status, dispatch_t> {
+ using result_t = std::variant<::grpc::Status, dispatch_t>;
// get blob from remote cas
auto const& dispatch_info = Artifact::ObjectInfo{.digest = dispatch_digest,
.type = ObjectType::File};
@@ -73,15 +70,7 @@ auto TargetService::GetDispatchList(ArtifactDigest const& dispatch_digest)
::grpc::Status{::grpc::StatusCode::FAILED_PRECONDITION,
std::get<0>(parsed)});
}
- auto const& dispatch = std::get<1>(parsed);
- auto dispatch_list = nlohmann::json::array();
- for (auto const& [props, endpoint] : dispatch) {
- auto entry = nlohmann::json::array();
- entry.push_back(nlohmann::json(props));
- entry.push_back(endpoint.ToJson());
- dispatch_list.push_back(entry);
- }
- return result_t(std::in_place_index<1>, std::move(dispatch_list));
+ return result_t(std::in_place_index<1>, std::get<1>(parsed));
}
auto TargetService::ServeTarget(
@@ -107,14 +96,17 @@ auto TargetService::ServeTarget(
// start filling in the backend description
auto address = RemoteExecutionConfig::RemoteAddress();
- auto description = nlohmann::json{
- {"remote_address", address ? address->ToJson() : nlohmann::json{}}};
-
- // read in the execution properties and add it to the description
- description["platform_properties"] = std::map<std::string, std::string>{};
+ // read in the execution properties and add it to the description;
+ // Important: we will need to pass these platform properties also to the
+ // executor (via the graph_traverser) in order for the build to be properly
+ // dispatched to the correct remote-execution endpoint.
+ auto platform_properties = std::map<std::string, std::string>{};
for (auto const& p : request->execution_properties()) {
- description["platform_properties"][p.name()] = p.value();
+ platform_properties[p.name()] = p.value();
}
+ auto description = nlohmann::json{
+ {"remote_address", address ? address->ToJson() : nlohmann::json{}},
+ {"platform_properties", platform_properties}};
// read in the dispatch list and add it to the description, if not empty
if (auto error_msg = IsAHash(request->dispatch_info().hash()); error_msg) {
@@ -128,8 +120,26 @@ auto TargetService::ServeTarget(
logger_->Emit(LogLevel::Error, err.error_message());
return err;
}
- if (auto dispatch_list = std::get<1>(res); not dispatch_list.empty()) {
- description["endpoint dispatch list"] = std::move(dispatch_list);
+ // keep dispatch list, as it needs to be passed to the executor (via the
+ // graph_traverser)
+ auto dispatch_list = std::get<1>(res);
+ // parse dispatch list to json and add to description
+ auto dispatch_json = nlohmann::json::array();
+ try {
+ for (auto const& [props, endpoint] : dispatch_list) {
+ auto entry = nlohmann::json::array();
+ entry.push_back(nlohmann::json(props));
+ entry.push_back(endpoint.ToJson());
+ dispatch_json.push_back(entry);
+ }
+ } catch (std::exception const& ex) {
+ logger_->Emit(
+ LogLevel::Info,
+ fmt::format("Parsing dispatch list to JSON failed with:\n{}",
+ ex.what()));
+ }
+ if (not dispatch_json.empty()) {
+ description["endpoint dispatch list"] = std::move(dispatch_json);
}
// add backend description to CAS;
@@ -426,6 +436,8 @@ auto TargetService::ServeTarget(
traverser_args.rebuild = std::nullopt;
GraphTraverser const traverser{std::move(traverser_args),
&repository_config,
+ std::move(platform_properties),
+ std::move(dispatch_list),
ProgressReporter::Reporter()};
// perform build
diff --git a/src/buildtool/serve_api/serve_service/target.hpp b/src/buildtool/serve_api/serve_service/target.hpp
index 5f9e2d15..9ab10d09 100644
--- a/src/buildtool/serve_api/serve_service/target.hpp
+++ b/src/buildtool/serve_api/serve_service/target.hpp
@@ -22,6 +22,7 @@
#include <string>
#include <utility>
#include <variant>
+#include <vector>
#include "gsl/gsl"
#include "justbuild/just_serve/just_serve.grpc.pb.h"
@@ -100,6 +101,10 @@ class TargetService final : public justbuild::just_serve::Target::Service {
private:
std::shared_ptr<Logger> logger_{std::make_shared<Logger>("target-service")};
+ // type of dispatch list; reduces verbosity
+ using dispatch_t = std::vector<
+ std::pair<std::map<std::string, std::string>, ServerAddress>>;
+
// remote execution endpoint used for remote building
gsl::not_null<IExecutionApi::Ptr> const remote_api_{
CreateExecutionApi(RemoteExecutionConfig::RemoteAddress(),
@@ -114,7 +119,7 @@ class TargetService final : public justbuild::just_serve::Target::Service {
/// \returns An error + data union, with a pair of grpc status at index 0
/// and the dispatch list stored as a JSON object at index 1.
auto GetDispatchList(ArtifactDigest const& dispatch_digest)
- -> std::variant<::grpc::Status, nlohmann::json>;
+ -> std::variant<::grpc::Status, dispatch_t>;
};
#endif // INCLUDED_SRC_BUILD_SERVE_API_SERVE_SERVICE_TARGET_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-13 18:10:02 +0000