From cc393dfca1bfd0fdc3fe032df5ea54e12f50cb9d Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Fri, 28 Feb 2025 12:11:32 +0100 Subject: Profiling: start collecting informations per actions ... so far, which actions where considered, and which of those were cached. --- src/buildtool/computed_roots/evaluate.cpp | 3 +- src/buildtool/execution_engine/executor/TARGETS | 2 ++ .../execution_engine/executor/context.hpp | 4 +++ .../execution_engine/executor/executor.hpp | 34 +++++++++++++++------- src/buildtool/main/main.cpp | 3 +- src/buildtool/profile/TARGETS | 5 +++- src/buildtool/profile/profile.cpp | 17 +++++++++++ src/buildtool/profile/profile.hpp | 14 ++++++++- src/buildtool/serve_api/serve_service/target.cpp | 3 +- 9 files changed, 70 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/buildtool/computed_roots/evaluate.cpp b/src/buildtool/computed_roots/evaluate.cpp index 839f9f98..507f13a9 100644 --- a/src/buildtool/computed_roots/evaluate.cpp +++ b/src/buildtool/computed_roots/evaluate.cpp @@ -205,7 +205,8 @@ void ComputeAndFill( context->apis, context->remote_context, &statistics, - &progress}; + &progress, + std::nullopt}; auto cache_lookup = expected, std::monostate>(std::nullopt); diff --git a/src/buildtool/execution_engine/executor/TARGETS b/src/buildtool/execution_engine/executor/TARGETS index 4e3d1abd..049efe44 100644 --- a/src/buildtool/execution_engine/executor/TARGETS +++ b/src/buildtool/execution_engine/executor/TARGETS @@ -30,6 +30,7 @@ , ["src/buildtool/file_system", "object_type"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] + , ["src/buildtool/profile", "profile"] , ["src/buildtool/progress_reporting", "progress"] , ["src/buildtool/progress_reporting", "task_tracker"] , ["src/utils/cpp", "back_map"] @@ -50,6 +51,7 @@ , ["src/buildtool/common", "statistics"] , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/execution_api/remote", "context"] + , ["src/buildtool/profile", "profile"] , ["src/buildtool/progress_reporting", "progress"] ] , "stage": ["src", "buildtool", "execution_engine", "executor"] diff --git a/src/buildtool/execution_engine/executor/context.hpp b/src/buildtool/execution_engine/executor/context.hpp index 315ab805..cbc2af15 100644 --- a/src/buildtool/execution_engine/executor/context.hpp +++ b/src/buildtool/execution_engine/executor/context.hpp @@ -15,11 +15,14 @@ #ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_CONTEXT_HPP #define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_CONTEXT_HPP +#include + #include "gsl/gsl" #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/remote/context.hpp" +#include "src/buildtool/profile/profile.hpp" #include "src/buildtool/progress_reporting/progress.hpp" /// \brief Aggregate to be passed to graph traverser. @@ -30,6 +33,7 @@ struct ExecutionContext final { gsl::not_null const remote_context; gsl::not_null const statistics; gsl::not_null const progress; + std::optional> const profile; }; #endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_CONTEXT_HPP diff --git a/src/buildtool/execution_engine/executor/executor.hpp b/src/buildtool/execution_engine/executor/executor.hpp index 93cb9655..e030d2d6 100644 --- a/src/buildtool/execution_engine/executor/executor.hpp +++ b/src/buildtool/execution_engine/executor/executor.hpp @@ -64,6 +64,7 @@ #include "src/buildtool/file_system/object_type.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/profile/profile.hpp" #include "src/buildtool/progress_reporting/progress.hpp" #include "src/buildtool/progress_reporting/task_tracker.hpp" #include "src/utils/cpp/back_map.hpp" @@ -828,11 +829,19 @@ class Executor { context_.statistics, context_.progress); // check response and save digests of results - return not response or Impl::ParseResponse(*logger_, - *response, - action, - context_.statistics, - context_.progress); + if (not response) { + return true; + } + auto result = Impl::ParseResponse(*logger_, + *response, + action, + context_.statistics, + context_.progress); + if (context_.profile) { + (*context_.profile) + ->NoteActionCompleted(action->Content().Id(), *response); + } + return result; } Logger logger("action:" + action->Content().Id()); @@ -851,11 +860,16 @@ class Executor { context_.progress); // check response and save digests of results - return not response or Impl::ParseResponse(logger, - *response, - action, - context_.statistics, - context_.progress); + if (not response) { + return true; + } + auto result = Impl::ParseResponse( + logger, *response, action, context_.statistics, context_.progress); + if (context_.profile) { + (*context_.profile) + ->NoteActionCompleted(action->Content().Id(), *response); + } + return result; } /// \brief Check artifact is available to the CAS or upload it. diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp index 2f41ca72..108b9798 100644 --- a/src/buildtool/main/main.cpp +++ b/src/buildtool/main/main.cpp @@ -999,7 +999,8 @@ auto main(int argc, char* argv[]) -> int { .apis = &main_apis, .remote_context = &remote_context, .statistics = &stats, - .progress = &progress}; + .progress = &progress, + .profile = profile}; const GraphTraverser::CommandLineArguments traverse_args{ jobs, std::move(arguments.build), diff --git a/src/buildtool/profile/TARGETS b/src/buildtool/profile/TARGETS index c2bde05e..657758ba 100644 --- a/src/buildtool/profile/TARGETS +++ b/src/buildtool/profile/TARGETS @@ -3,7 +3,10 @@ , "name": ["profile"] , "hdrs": ["profile.hpp"] , "srcs": ["profile.cpp"] - , "deps": [["@", "json", "", "json"]] + , "deps": + [ ["@", "json", "", "json"] + , ["src/buildtool/execution_api/common", "common"] + ] , "stage": ["src", "buildtool", "profile"] } } diff --git a/src/buildtool/profile/profile.cpp b/src/buildtool/profile/profile.cpp index 543cd681..3ee7f2a6 100644 --- a/src/buildtool/profile/profile.cpp +++ b/src/buildtool/profile/profile.cpp @@ -20,6 +20,17 @@ void Profile::Write(int exit_code) { if (not output_file_) { return; } + + if (not actions_.empty()) { + auto actions = nlohmann::json::object(); + for (auto const& [k, v] : actions_) { + auto entry = nlohmann::json::object(); + entry["cached"] = v.cached; + actions[k] = entry; + } + profile_["actions"] = actions; + } + profile_["exit code"] = exit_code; std::ofstream os(*output_file_); @@ -33,3 +44,9 @@ void Profile::SetTarget(nlohmann::json target) { void Profile::SetConfiguration(nlohmann::json configuration) { profile_["configuration"] = std::move(configuration); } + +void Profile::NoteActionCompleted(std::string const& id, + IExecutionResponse::Ptr const& response) { + std::unique_lock lock{mutex_}; + actions_[id] = ActionData{.cached = response->IsCached()}; +} diff --git a/src/buildtool/profile/profile.hpp b/src/buildtool/profile/profile.hpp index 0d4e14b6..6cf222e1 100644 --- a/src/buildtool/profile/profile.hpp +++ b/src/buildtool/profile/profile.hpp @@ -15,26 +15,38 @@ #ifndef INCLUDED_SRC_BUILDTOOL_PROFILE_PROFILE_HPP #define INCLUDED_SRC_BUILDTOOL_PROFILE_PROFILE_HPP +#include +#include #include #include +#include #include #include "nlohmann/json.hpp" +#include "src/buildtool/execution_api/common/execution_response.hpp" class Profile { public: explicit Profile(std::optional output_file) - : output_file_{std::move(output_file)} { + : output_file_{std::move(output_file)}, actions_{}, mutex_{} { profile_ = nlohmann::json::object(); } void Write(int exit_code); void SetTarget(nlohmann::json target); void SetConfiguration(nlohmann::json configuration); + void NoteActionCompleted(std::string const& id, + IExecutionResponse::Ptr const& response); private: + struct ActionData { + bool cached; + }; + std::optional output_file_; nlohmann::json profile_; + std::unordered_map actions_; + std::mutex mutex_; }; #endif diff --git a/src/buildtool/serve_api/serve_service/target.cpp b/src/buildtool/serve_api/serve_service/target.cpp index 6b46157c..e8a0714e 100644 --- a/src/buildtool/serve_api/serve_service/target.cpp +++ b/src/buildtool/serve_api/serve_service/target.cpp @@ -558,7 +558,8 @@ auto TargetService::ServeTarget( .apis = &local_apis, .remote_context = &dispatch_context, .statistics = &stats, - .progress = &progress}; + .progress = &progress, + .profile = std::nullopt}; GraphTraverser const traverser{ std::move(traverser_args), -- cgit v1.2.3