summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorKlaus Aehlig <klaus.aehlig@huawei.com>2022-02-22 17:03:21 +0100
committerKlaus Aehlig <klaus.aehlig@huawei.com>2022-02-22 17:03:21 +0100
commit619def44c1cca9f3cdf63544d5f24f2c7a7d9b77 (patch)
tree01868de723cb82c86842f33743fa7b14e24c1fa3 /src
downloadjustbuild-619def44c1cca9f3cdf63544d5f24f2c7a7d9b77.tar.gz
Initial self-hosting commit
This is the initial version of our tool that is able to build itself. In can be bootstrapped by ./bin/bootstrap.py Co-authored-by: Oliver Reiche <oliver.reiche@huawei.com> Co-authored-by: Victor Moreno <victor.moreno1@huawei.com>
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/TARGETS1
-rw-r--r--src/buildtool/build_engine/analysed_target/TARGETS12
-rw-r--r--src/buildtool/build_engine/analysed_target/analysed_target.hpp101
-rw-r--r--src/buildtool/build_engine/base_maps/TARGETS162
-rw-r--r--src/buildtool/build_engine/base_maps/directory_map.cpp35
-rw-r--r--src/buildtool/build_engine/base_maps/directory_map.hpp22
-rw-r--r--src/buildtool/build_engine/base_maps/entity_name.hpp206
-rw-r--r--src/buildtool/build_engine/base_maps/entity_name_data.hpp129
-rw-r--r--src/buildtool/build_engine/base_maps/expression_function.hpp103
-rw-r--r--src/buildtool/build_engine/base_maps/expression_map.cpp90
-rw-r--r--src/buildtool/build_engine/base_maps/expression_map.hpp32
-rw-r--r--src/buildtool/build_engine/base_maps/field_reader.hpp231
-rw-r--r--src/buildtool/build_engine/base_maps/json_file_map.hpp93
-rw-r--r--src/buildtool/build_engine/base_maps/module_name.hpp36
-rw-r--r--src/buildtool/build_engine/base_maps/rule_map.cpp371
-rw-r--r--src/buildtool/build_engine/base_maps/rule_map.hpp33
-rw-r--r--src/buildtool/build_engine/base_maps/source_map.cpp88
-rw-r--r--src/buildtool/build_engine/base_maps/source_map.hpp24
-rw-r--r--src/buildtool/build_engine/base_maps/targets_file_map.hpp23
-rw-r--r--src/buildtool/build_engine/base_maps/user_rule.hpp404
-rw-r--r--src/buildtool/build_engine/expression/TARGETS46
-rw-r--r--src/buildtool/build_engine/expression/configuration.hpp154
-rw-r--r--src/buildtool/build_engine/expression/evaluator.cpp936
-rw-r--r--src/buildtool/build_engine/expression/evaluator.hpp76
-rw-r--r--src/buildtool/build_engine/expression/expression.cpp249
-rw-r--r--src/buildtool/build_engine/expression/expression.hpp380
-rw-r--r--src/buildtool/build_engine/expression/expression_ptr.cpp89
-rw-r--r--src/buildtool/build_engine/expression/expression_ptr.hpp95
-rw-r--r--src/buildtool/build_engine/expression/function_map.hpp23
-rw-r--r--src/buildtool/build_engine/expression/linked_map.hpp414
-rw-r--r--src/buildtool/build_engine/expression/target_node.cpp20
-rw-r--r--src/buildtool/build_engine/expression/target_node.hpp83
-rw-r--r--src/buildtool/build_engine/expression/target_result.hpp33
-rw-r--r--src/buildtool/build_engine/target_map/TARGETS50
-rw-r--r--src/buildtool/build_engine/target_map/built_in_rules.cpp857
-rw-r--r--src/buildtool/build_engine/target_map/built_in_rules.hpp21
-rw-r--r--src/buildtool/build_engine/target_map/configured_target.hpp41
-rw-r--r--src/buildtool/build_engine/target_map/export.cpp126
-rw-r--r--src/buildtool/build_engine/target_map/export.hpp17
-rw-r--r--src/buildtool/build_engine/target_map/result_map.hpp291
-rw-r--r--src/buildtool/build_engine/target_map/target_map.cpp1338
-rw-r--r--src/buildtool/build_engine/target_map/target_map.hpp27
-rw-r--r--src/buildtool/build_engine/target_map/utils.cpp197
-rw-r--r--src/buildtool/build_engine/target_map/utils.hpp55
-rw-r--r--src/buildtool/common/TARGETS101
-rw-r--r--src/buildtool/common/action.hpp78
-rw-r--r--src/buildtool/common/action_description.hpp200
-rw-r--r--src/buildtool/common/artifact.hpp214
-rw-r--r--src/buildtool/common/artifact_description.hpp316
-rw-r--r--src/buildtool/common/artifact_digest.hpp74
-rw-r--r--src/buildtool/common/artifact_factory.hpp91
-rw-r--r--src/buildtool/common/bazel_types.hpp86
-rw-r--r--src/buildtool/common/cli.hpp365
-rw-r--r--src/buildtool/common/identifier.hpp25
-rw-r--r--src/buildtool/common/repository_config.hpp133
-rw-r--r--src/buildtool/common/statistics.hpp61
-rw-r--r--src/buildtool/common/tree.hpp72
-rw-r--r--src/buildtool/crypto/TARGETS31
-rw-r--r--src/buildtool/crypto/hash_generator.hpp130
-rw-r--r--src/buildtool/crypto/hash_impl.hpp40
-rw-r--r--src/buildtool/crypto/hash_impl_git.cpp42
-rw-r--r--src/buildtool/crypto/hash_impl_git.hpp10
-rw-r--r--src/buildtool/crypto/hash_impl_md5.cpp50
-rw-r--r--src/buildtool/crypto/hash_impl_md5.hpp10
-rw-r--r--src/buildtool/crypto/hash_impl_sha1.cpp50
-rw-r--r--src/buildtool/crypto/hash_impl_sha1.hpp10
-rw-r--r--src/buildtool/crypto/hash_impl_sha256.cpp50
-rw-r--r--src/buildtool/crypto/hash_impl_sha256.hpp10
-rw-r--r--src/buildtool/execution_api/TARGETS1
-rw-r--r--src/buildtool/execution_api/bazel_msg/TARGETS31
-rw-r--r--src/buildtool/execution_api/bazel_msg/bazel_blob.hpp31
-rw-r--r--src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp264
-rw-r--r--src/buildtool/execution_api/bazel_msg/bazel_common.hpp21
-rw-r--r--src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp590
-rw-r--r--src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp128
-rw-r--r--src/buildtool/execution_api/common/TARGETS22
-rw-r--r--src/buildtool/execution_api/common/execution_action.hpp58
-rw-r--r--src/buildtool/execution_api/common/execution_api.hpp78
-rw-r--r--src/buildtool/execution_api/common/execution_common.hpp109
-rw-r--r--src/buildtool/execution_api/common/execution_response.hpp48
-rw-r--r--src/buildtool/execution_api/common/local_tree_map.hpp140
-rw-r--r--src/buildtool/execution_api/local/TARGETS36
-rw-r--r--src/buildtool/execution_api/local/config.hpp137
-rw-r--r--src/buildtool/execution_api/local/file_storage.hpp107
-rw-r--r--src/buildtool/execution_api/local/local_ac.hpp82
-rw-r--r--src/buildtool/execution_api/local/local_action.cpp295
-rw-r--r--src/buildtool/execution_api/local/local_action.hpp122
-rw-r--r--src/buildtool/execution_api/local/local_api.hpp157
-rw-r--r--src/buildtool/execution_api/local/local_cas.hpp103
-rw-r--r--src/buildtool/execution_api/local/local_response.hpp101
-rw-r--r--src/buildtool/execution_api/local/local_storage.cpp125
-rw-r--r--src/buildtool/execution_api/local/local_storage.hpp109
-rw-r--r--src/buildtool/execution_api/remote/TARGETS59
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_ac_client.cpp75
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp41
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_action.cpp94
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_action.hpp54
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.cpp177
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_api.hpp65
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp354
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp169
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp54
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_execution_client.cpp129
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp66
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_network.cpp327
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_network.hpp118
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_response.cpp125
-rw-r--r--src/buildtool/execution_api/remote/bazel/bazel_response.hpp77
-rw-r--r--src/buildtool/execution_api/remote/bazel/bytestream_client.hpp185
-rw-r--r--src/buildtool/execution_api/remote/config.hpp72
-rw-r--r--src/buildtool/execution_engine/TARGETS1
-rw-r--r--src/buildtool/execution_engine/dag/TARGETS17
-rw-r--r--src/buildtool/execution_engine/dag/dag.cpp263
-rw-r--r--src/buildtool/execution_engine/dag/dag.hpp613
-rw-r--r--src/buildtool/execution_engine/executor/TARGETS16
-rw-r--r--src/buildtool/execution_engine/executor/executor.hpp532
-rw-r--r--src/buildtool/execution_engine/traverser/TARGETS14
-rw-r--r--src/buildtool/execution_engine/traverser/traverser.hpp187
-rw-r--r--src/buildtool/file_system/TARGETS79
-rw-r--r--src/buildtool/file_system/file_root.hpp239
-rw-r--r--src/buildtool/file_system/file_system_manager.hpp565
-rw-r--r--src/buildtool/file_system/git_cas.cpp180
-rw-r--r--src/buildtool/file_system/git_cas.hpp60
-rw-r--r--src/buildtool/file_system/git_tree.cpp178
-rw-r--r--src/buildtool/file_system/git_tree.hpp87
-rw-r--r--src/buildtool/file_system/jsonfs.hpp47
-rw-r--r--src/buildtool/file_system/object_type.hpp44
-rw-r--r--src/buildtool/file_system/system_command.hpp202
-rw-r--r--src/buildtool/graph_traverser/TARGETS22
-rw-r--r--src/buildtool/graph_traverser/graph_traverser.hpp569
-rw-r--r--src/buildtool/logging/TARGETS20
-rw-r--r--src/buildtool/logging/log_config.hpp69
-rw-r--r--src/buildtool/logging/log_level.hpp41
-rw-r--r--src/buildtool/logging/log_sink.hpp41
-rw-r--r--src/buildtool/logging/log_sink_cmdline.hpp93
-rw-r--r--src/buildtool/logging/log_sink_file.hpp129
-rw-r--r--src/buildtool/logging/logger.hpp123
-rw-r--r--src/buildtool/main/TARGETS21
-rw-r--r--src/buildtool/main/main.cpp1292
-rw-r--r--src/buildtool/main/main.hpp10
-rw-r--r--src/buildtool/multithreading/TARGETS54
-rw-r--r--src/buildtool/multithreading/async_map.hpp109
-rw-r--r--src/buildtool/multithreading/async_map_consumer.hpp331
-rw-r--r--src/buildtool/multithreading/async_map_node.hpp173
-rw-r--r--src/buildtool/multithreading/notification_queue.hpp188
-rw-r--r--src/buildtool/multithreading/task.hpp38
-rw-r--r--src/buildtool/multithreading/task_system.cpp56
-rw-r--r--src/buildtool/multithreading/task_system.hpp65
-rw-r--r--src/utils/TARGETS1
-rw-r--r--src/utils/cpp/TARGETS40
-rw-r--r--src/utils/cpp/atomic.hpp119
-rw-r--r--src/utils/cpp/concepts.hpp55
-rw-r--r--src/utils/cpp/hash_combine.hpp15
-rw-r--r--src/utils/cpp/hex_string.hpp19
-rw-r--r--src/utils/cpp/json.hpp83
-rw-r--r--src/utils/cpp/type_safe_arithmetic.hpp197
156 files changed, 22496 insertions, 0 deletions
diff --git a/src/buildtool/TARGETS b/src/buildtool/TARGETS
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/buildtool/TARGETS
@@ -0,0 +1 @@
+{} \ No newline at end of file
diff --git a/src/buildtool/build_engine/analysed_target/TARGETS b/src/buildtool/build_engine/analysed_target/TARGETS
new file mode 100644
index 00000000..4884ec46
--- /dev/null
+++ b/src/buildtool/build_engine/analysed_target/TARGETS
@@ -0,0 +1,12 @@
+{ "target":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["target"]
+ , "hdrs": ["analysed_target.hpp"]
+ , "deps":
+ [ ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/common", "action_description"]
+ , ["src/buildtool/common", "tree"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "analysed_target"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/build_engine/analysed_target/analysed_target.hpp b/src/buildtool/build_engine/analysed_target/analysed_target.hpp
new file mode 100644
index 00000000..af92b0bc
--- /dev/null
+++ b/src/buildtool/build_engine/analysed_target/analysed_target.hpp
@@ -0,0 +1,101 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_ANALYSED_TARGET_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_ANALYSED_TARGET_HPP
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
+#include "src/buildtool/build_engine/expression/target_result.hpp"
+#include "src/buildtool/common/action_description.hpp"
+#include "src/buildtool/common/tree.hpp"
+
+class AnalysedTarget {
+ public:
+ AnalysedTarget(TargetResult result,
+ std::vector<ActionDescription> actions,
+ std::vector<std::string> blobs,
+ std::vector<Tree> trees,
+ std::unordered_set<std::string> vars,
+ std::set<std::string> tainted)
+ : result_{std::move(result)},
+ actions_{std::move(actions)},
+ blobs_{std::move(blobs)},
+ trees_{std::move(trees)},
+ vars_{std::move(vars)},
+ tainted_{std::move(tainted)} {}
+
+ [[nodiscard]] auto Actions() const& noexcept
+ -> std::vector<ActionDescription> const& {
+ return actions_;
+ }
+ [[nodiscard]] auto Actions() && noexcept -> std::vector<ActionDescription> {
+ return std::move(actions_);
+ }
+ [[nodiscard]] auto Artifacts() const& noexcept -> ExpressionPtr const& {
+ return result_.artifact_stage;
+ }
+ [[nodiscard]] auto Artifacts() && noexcept -> ExpressionPtr {
+ return std::move(result_.artifact_stage);
+ }
+ [[nodiscard]] auto RunFiles() const& noexcept -> ExpressionPtr const& {
+ return result_.runfiles;
+ }
+ [[nodiscard]] auto RunFiles() && noexcept -> ExpressionPtr {
+ return std::move(result_.runfiles);
+ }
+ [[nodiscard]] auto Provides() const& noexcept -> ExpressionPtr const& {
+ return result_.provides;
+ }
+ [[nodiscard]] auto Provides() && noexcept -> ExpressionPtr {
+ return std::move(result_.provides);
+ }
+ [[nodiscard]] auto Blobs() const& noexcept
+ -> std::vector<std::string> const& {
+ return blobs_;
+ }
+ [[nodiscard]] auto Trees() && noexcept -> std::vector<Tree> {
+ return std::move(trees_);
+ }
+ [[nodiscard]] auto Trees() const& noexcept -> std::vector<Tree> const& {
+ return trees_;
+ }
+ [[nodiscard]] auto Blobs() && noexcept -> std::vector<std::string> {
+ return std::move(blobs_);
+ }
+ [[nodiscard]] auto Vars() const& noexcept
+ -> std::unordered_set<std::string> const& {
+ return vars_;
+ }
+ [[nodiscard]] auto Vars() && noexcept -> std::unordered_set<std::string> {
+ return std::move(vars_);
+ }
+ [[nodiscard]] auto Tainted() const& noexcept
+ -> std::set<std::string> const& {
+ return tainted_;
+ }
+ [[nodiscard]] auto Tainted() && noexcept -> std::set<std::string> {
+ return std::move(tainted_);
+ }
+ [[nodiscard]] auto Result() const& noexcept -> TargetResult const& {
+ return result_;
+ }
+ [[nodiscard]] auto Result() && noexcept -> TargetResult {
+ return std::move(result_);
+ }
+
+ private:
+ TargetResult result_;
+ std::vector<ActionDescription> actions_;
+ std::vector<std::string> blobs_;
+ std::vector<Tree> trees_;
+ std::unordered_set<std::string> vars_;
+ std::set<std::string> tainted_;
+};
+
+using AnalysedTargetPtr = std::shared_ptr<AnalysedTarget>;
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILDENGINE_ANALYSED_TARGET_ANALYSED_TARGET_HPP
diff --git a/src/buildtool/build_engine/base_maps/TARGETS b/src/buildtool/build_engine/base_maps/TARGETS
new file mode 100644
index 00000000..7ba580ca
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/TARGETS
@@ -0,0 +1,162 @@
+{ "module_name":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["module_name"]
+ , "hdrs": ["module_name.hpp"]
+ , "deps": [["src/utils/cpp", "hash_combine"]]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "directory_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["directory_map"]
+ , "hdrs": ["directory_map.hpp"]
+ , "srcs": ["directory_map.cpp"]
+ , "deps":
+ [ ["src/buildtool/common", "config"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ , "module_name"
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "json_file_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["json_file_map"]
+ , "hdrs": ["json_file_map.hpp"]
+ , "deps":
+ [ ["@", "fmt", "", "fmt"]
+ , ["@", "json", "", "json"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["src/buildtool/common", "config"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ , "module_name"
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "targets_file_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["targets_file_map"]
+ , "hdrs": ["targets_file_map.hpp"]
+ , "deps":
+ [ "json_file_map"
+ , ["@", "json", "", "json"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "entity_name_data":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["entity_name_data"]
+ , "hdrs": ["entity_name_data.hpp"]
+ , "deps":
+ [ ["@", "json", "", "json"]
+ , ["src/utils/cpp", "hash_combine"]
+ , "module_name"
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "entity_name":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["entity_name"]
+ , "hdrs": ["entity_name.hpp"]
+ , "deps":
+ [ "entity_name_data"
+ , ["@", "json", "", "json"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["src/buildtool/common", "config"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "source_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["source_map"]
+ , "hdrs": ["source_map.hpp"]
+ , "srcs": ["source_map.cpp"]
+ , "deps":
+ [ "directory_map"
+ , "entity_name"
+ , ["@", "json", "", "json"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["src/buildtool/build_engine/analysed_target", "target"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ , ["src/utils/cpp", "json"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "field_reader":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["field_reader"]
+ , "hdrs": ["field_reader.hpp"]
+ , "deps":
+ [ "entity_name"
+ , ["@", "fmt", "", "fmt"]
+ , ["@", "json", "", "json"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "expression_function":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["expression_function"]
+ , "hdrs": ["expression_function.hpp"]
+ , "deps":
+ [ ["src/utils/cpp", "hash_combine"]
+ , ["src/buildtool/logging", "logging"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "expression_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["expression_map"]
+ , "hdrs": ["expression_map.hpp"]
+ , "srcs": ["expression_map.cpp"]
+ , "deps":
+ [ "json_file_map"
+ , "entity_name"
+ , "expression_function"
+ , "field_reader"
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["@", "fmt", "", "fmt"]
+ , ["@", "json", "", "json"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "user_rule":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["user_rule"]
+ , "hdrs": ["user_rule.hpp"]
+ , "deps":
+ [ "entity_name"
+ , "expression_function"
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["@", "fmt", "", "fmt"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/utils/cpp", "concepts"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+, "rule_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["rule_map"]
+ , "hdrs": ["rule_map.hpp"]
+ , "srcs": ["rule_map.cpp"]
+ , "deps":
+ [ "json_file_map"
+ , "entity_name"
+ , "user_rule"
+ , "field_reader"
+ , "expression_map"
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["@", "fmt", "", "fmt"]
+ , ["@", "json", "", "json"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "base_maps"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/build_engine/base_maps/directory_map.cpp b/src/buildtool/build_engine/base_maps/directory_map.cpp
new file mode 100644
index 00000000..1b862386
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/directory_map.cpp
@@ -0,0 +1,35 @@
+#include "src/buildtool/build_engine/base_maps/directory_map.hpp"
+
+#include <filesystem>
+#include <unordered_set>
+
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+auto BuildMaps::Base::CreateDirectoryEntriesMap(std::size_t jobs)
+ -> DirectoryEntriesMap {
+ auto directory_reader = [](auto /* unused*/,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
+ auto const* ws_root =
+ RepositoryConfig::Instance().WorkspaceRoot(key.repository);
+ if (ws_root == nullptr) {
+ (*logger)(
+ fmt::format("Cannot determine workspace root for repository {}",
+ key.repository),
+ true);
+ return;
+ }
+ if (not ws_root->IsDirectory(key.module)) {
+ // Missing directory is fine (source tree might be incomplete),
+ // contains no entries.
+ (*setter)(FileRoot::DirectoryEntries{});
+ return;
+ }
+ (*setter)(ws_root->ReadDirectory(key.module));
+ };
+ return AsyncMapConsumer<BuildMaps::Base::ModuleName,
+ FileRoot::DirectoryEntries>{directory_reader, jobs};
+}
diff --git a/src/buildtool/build_engine/base_maps/directory_map.hpp b/src/buildtool/build_engine/base_maps/directory_map.hpp
new file mode 100644
index 00000000..fb675997
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/directory_map.hpp
@@ -0,0 +1,22 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_DIRECTORY_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_DIRECTORY_MAP_HPP
+
+#include <filesystem>
+#include <map>
+#include <unordered_set>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/base_maps/module_name.hpp"
+#include "src/buildtool/file_system/file_root.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+namespace BuildMaps::Base {
+
+using DirectoryEntriesMap =
+ AsyncMapConsumer<ModuleName, FileRoot::DirectoryEntries>;
+
+auto CreateDirectoryEntriesMap(std::size_t jobs = 0) -> DirectoryEntriesMap;
+
+} // namespace BuildMaps::Base
+
+#endif
diff --git a/src/buildtool/build_engine/base_maps/entity_name.hpp b/src/buildtool/build_engine/base_maps/entity_name.hpp
new file mode 100644
index 00000000..12fbd6ee
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/entity_name.hpp
@@ -0,0 +1,206 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_HPP
+
+#include <filesystem>
+#include <optional>
+#include <utility>
+
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name_data.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+
+namespace BuildMaps::Base {
+
+[[nodiscard]] inline auto ParseEntityNameFromJson(
+ nlohmann::json const& json,
+ EntityName const& current,
+ std::optional<std::function<void(std::string const&)>> logger =
+ std::nullopt) noexcept -> std::optional<EntityName> {
+ try {
+ if (json.is_string()) {
+ return EntityName{current.repository,
+ current.module,
+ json.template get<std::string>()};
+ }
+ if (json.is_array() and json.size() == 2 and json[0].is_string() and
+ json[1].is_string()) {
+ return EntityName{current.repository,
+ json[0].template get<std::string>(),
+ json[1].template get<std::string>()};
+ }
+ if (json.is_array() and json.size() == 3 and json[0].is_string() and
+ json[0].template get<std::string>() ==
+ EntityName::kFileLocationMarker and
+ json[2].is_string()) {
+ auto name = json[2].template get<std::string>();
+ if (json[1].is_null()) {
+ return EntityName{
+ current.repository, current.module, name, true};
+ }
+ if (json[1].is_string()) {
+ auto middle = json[1].template get<std::string>();
+ if (middle == "." or middle == current.module) {
+ return EntityName{
+ current.repository, current.module, name, true};
+ }
+ }
+ if (logger) {
+ (*logger)(
+ fmt::format("Invalid module name {} for file reference",
+ json[1].dump()));
+ }
+ }
+ else if (json.is_array() and json.size() == 3 and
+ json[0].is_string() and
+ json[0].template get<std::string>() ==
+ EntityName::kRelativeLocationMarker and
+ json[1].is_string() and json[2].is_string()) {
+ auto relmodule = json[1].template get<std::string>();
+ auto name = json[2].template get<std::string>();
+
+ std::filesystem::path m{current.module};
+ auto module = (m / relmodule).lexically_normal().string();
+ if (module.compare(0, 3, "../") != 0) {
+ return EntityName{current.repository, module, name};
+ }
+ if (logger) {
+ (*logger)(fmt::format(
+ "Relative module name {} is outside of workspace",
+ relmodule));
+ }
+ }
+ else if (json.is_array() and json.size() == 3 and
+ json[0].is_string() and
+ json[0].template get<std::string>() ==
+ EntityName::kAnonymousMarker) {
+ if (logger) {
+ (*logger)(fmt::format(
+ "Parsing anonymous target from JSON is not supported."));
+ }
+ }
+ else if (json.is_array() and json.size() == 4 and
+ json[0].is_string() and
+ json[0].template get<std::string>() ==
+ EntityName::kLocationMarker and
+ json[1].is_string() and json[2].is_string() and
+ json[3].is_string()) {
+ auto local_repo_name = json[1].template get<std::string>();
+ auto module = json[2].template get<std::string>();
+ auto target = json[3].template get<std::string>();
+ auto const* repo_name = RepositoryConfig::Instance().GlobalName(
+ current.repository, local_repo_name);
+ if (repo_name != nullptr) {
+ return EntityName{*repo_name, module, target};
+ }
+ if (logger) {
+ (*logger)(fmt::format("Cannot resolve repository name {}",
+ local_repo_name));
+ }
+ }
+ else if (logger) {
+ (*logger)(fmt::format("Syntactically invalid entity name: {}.",
+ json.dump()));
+ }
+ } catch (...) {
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] inline auto ParseEntityNameFromExpression(
+ ExpressionPtr const& expr,
+ EntityName const& current,
+ std::optional<std::function<void(std::string const&)>> logger =
+ std::nullopt) noexcept -> std::optional<EntityName> {
+ try {
+ if (expr) {
+ if (expr->IsString()) {
+ return EntityName{current.repository,
+ current.module,
+ expr->Value<std::string>()->get()};
+ }
+ if (expr->IsList()) {
+ auto const& list = expr->Value<Expression::list_t>()->get();
+ if (list.size() == 2 and list[0]->IsString() and
+ list[1]->IsString()) {
+ return EntityName{current.repository,
+ list[0]->Value<std::string>()->get(),
+ list[1]->Value<std::string>()->get()};
+ }
+ if (list.size() == 3 and list[0]->IsString() and
+ list[0]->String() == EntityName::kFileLocationMarker and
+ list[2]->IsString()) {
+ auto name = list[2]->Value<std::string>()->get();
+ if (list[1]->IsNone()) {
+ return EntityName{
+ current.repository, current.module, name, true};
+ }
+ if (list[1]->IsString() and
+ (list[1]->String() == "." or
+ list[1]->String() == current.module)) {
+ return EntityName{
+ current.repository, current.module, name, true};
+ }
+ if (logger) {
+ (*logger)(fmt::format(
+ "Invalid module name {} for file reference",
+ list[1]->ToString()));
+ }
+ }
+ else if (list.size() == 3 and list[0]->IsString() and
+ list[0]->String() ==
+ EntityName::kRelativeLocationMarker and
+ list[1]->IsString() and list[2]->IsString()) {
+ std::filesystem::path m{current.module};
+ auto module =
+ (m / (list[1]->String())).lexically_normal().string();
+ if (module.compare(0, 3, "../") != 0) {
+ return EntityName{
+ current.repository, module, list[2]->String()};
+ }
+ if (logger) {
+ (*logger)(fmt::format(
+ "Relative module name {} is outside of workspace",
+ list[1]->String()));
+ }
+ }
+ else if (list.size() == 3 and list[0]->IsString() and
+ list[0]->String() ==
+ EntityName::kRelativeLocationMarker and
+ list[1]->IsMap() and list[2]->IsNode()) {
+ return EntityName{AnonymousTarget{list[1], list[2]}};
+ }
+ else if (list.size() == 4 and list[0]->IsString() and
+ list[0]->String() == EntityName::kLocationMarker and
+ list[1]->IsString() and list[2]->IsString() and
+ list[3]->IsString()) {
+ auto const* repo_name =
+ RepositoryConfig::Instance().GlobalName(
+ current.repository, list[1]->String());
+ if (repo_name != nullptr) {
+ return EntityName{
+ *repo_name, list[2]->String(), list[3]->String()};
+ }
+ if (logger) {
+ (*logger)(
+ fmt::format("Cannot resolve repository name {}",
+ list[1]->String()));
+ }
+ }
+ else if (logger) {
+ (*logger)(
+ fmt::format("Syntactically invalid entity name: {}.",
+ expr->ToString()));
+ }
+ }
+ }
+ } catch (...) {
+ }
+ return std::nullopt;
+}
+
+} // namespace BuildMaps::Base
+
+#endif
diff --git a/src/buildtool/build_engine/base_maps/entity_name_data.hpp b/src/buildtool/build_engine/base_maps/entity_name_data.hpp
new file mode 100644
index 00000000..217ccfb5
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/entity_name_data.hpp
@@ -0,0 +1,129 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_DATA_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_ENTITY_NAME_DATA_HPP
+
+#include <filesystem>
+#include <optional>
+#include <utility>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/build_engine/base_maps/module_name.hpp"
+#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+namespace BuildMaps::Base {
+
+struct AnonymousTarget {
+ ExpressionPtr rule_map;
+ ExpressionPtr target_node;
+
+ [[nodiscard]] auto operator==(AnonymousTarget const& other) const noexcept
+ -> bool {
+ return rule_map == other.rule_map && target_node == other.target_node;
+ }
+};
+
+struct EntityName {
+ static constexpr auto kLocationMarker = "@";
+ static constexpr auto kFileLocationMarker = "FILE";
+ static constexpr auto kRelativeLocationMarker = "./";
+ static constexpr auto kAnonymousMarker = "#";
+
+ std::string repository{};
+ std::string module{};
+ std::string name{};
+ std::optional<AnonymousTarget> anonymous{};
+ bool explicit_file_reference{};
+
+ EntityName() = default;
+ EntityName(std::string repository,
+ const std::string& module,
+ std::string name)
+ : repository{std::move(repository)},
+ module{normal_module_name(module)},
+ name{std::move(name)} {}
+ explicit EntityName(AnonymousTarget anonymous)
+ : anonymous{std::move(anonymous)} {}
+
+ static auto normal_module_name(const std::string& module) -> std::string {
+ return std::filesystem::path("/" + module + "/")
+ .lexically_normal()
+ .lexically_relative("/")
+ .parent_path()
+ .string();
+ }
+
+ [[nodiscard]] auto operator==(
+ BuildMaps::Base::EntityName const& other) const noexcept -> bool {
+ return module == other.module && name == other.name &&
+ repository == other.repository && anonymous == other.anonymous &&
+ explicit_file_reference == other.explicit_file_reference;
+ }
+
+ [[nodiscard]] auto ToJson() const -> nlohmann::json {
+ nlohmann::json j;
+ if (IsAnonymousTarget()) {
+ j.push_back(kAnonymousMarker);
+ j.push_back(anonymous->rule_map.ToIdentifier());
+ j.push_back(anonymous->target_node.ToIdentifier());
+ }
+ else {
+ j.push_back(kLocationMarker);
+ j.push_back(repository);
+ if (explicit_file_reference) {
+ j.push_back(kFileLocationMarker);
+ }
+ j.push_back(module);
+ j.push_back(name);
+ }
+ return j;
+ }
+
+ [[nodiscard]] auto ToString() const -> std::string {
+ return ToJson().dump();
+ }
+
+ [[nodiscard]] auto ToModule() const -> ModuleName {
+ return ModuleName{repository, module};
+ }
+
+ [[nodiscard]] auto IsDefinitionName() const -> bool {
+ return (not explicit_file_reference);
+ }
+
+ [[nodiscard]] auto IsAnonymousTarget() const -> bool {
+ return static_cast<bool>(anonymous);
+ }
+
+ EntityName(std::string repository,
+ const std::string& module,
+ std::string name,
+ bool explicit_file_reference)
+ : repository{std::move(repository)},
+ module{normal_module_name(module)},
+ name{std::move(name)},
+ explicit_file_reference{explicit_file_reference} {}
+};
+} // namespace BuildMaps::Base
+
+namespace std {
+template <>
+struct hash<BuildMaps::Base::EntityName> {
+ [[nodiscard]] auto operator()(
+ const BuildMaps::Base::EntityName& t) const noexcept -> std::size_t {
+ size_t seed{};
+ hash_combine<std::string>(&seed, t.repository);
+ hash_combine<std::string>(&seed, t.module);
+ hash_combine<std::string>(&seed, t.name);
+ auto anonymous =
+ t.anonymous.value_or(BuildMaps::Base::AnonymousTarget{});
+ hash_combine<ExpressionPtr>(&seed, anonymous.rule_map);
+ hash_combine<ExpressionPtr>(&seed, anonymous.target_node);
+ hash_combine<bool>(&seed, t.explicit_file_reference);
+ return seed;
+ }
+};
+
+} // namespace std
+
+#endif
diff --git a/src/buildtool/build_engine/base_maps/expression_function.hpp b/src/buildtool/build_engine/base_maps/expression_function.hpp
new file mode 100644
index 00000000..0eb07a88
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/expression_function.hpp
@@ -0,0 +1,103 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_FUNCTION_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_FUNCTION_HPP
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "fmt/core.h"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/build_engine/expression/evaluator.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+namespace BuildMaps::Base {
+
+class ExpressionFunction {
+ public:
+ using Ptr = std::shared_ptr<ExpressionFunction>;
+ using imports_t = std::unordered_map<std::string, gsl::not_null<Ptr>>;
+
+ ExpressionFunction(std::vector<std::string> vars,
+ imports_t imports,
+ ExpressionPtr expr) noexcept
+ : vars_{std::move(vars)},
+ imports_{std::move(imports)},
+ expr_{std::move(expr)} {}
+
+ [[nodiscard]] auto Evaluate(
+ Configuration const& env,
+ FunctionMapPtr const& functions,
+ std::function<void(std::string const&)> const& logger =
+ [](std::string const& error) noexcept -> void {
+ Logger::Log(LogLevel::Error, error);
+ },
+ std::function<void(void)> const& note_user_context =
+ []() noexcept -> void {}) const noexcept -> ExpressionPtr {
+ try { // try-catch to silence clang-tidy's bugprone-exception-escape,
+ // only imports_caller can throw but it is not called here.
+ auto imports_caller = [this, &functions](
+ SubExprEvaluator&& /*eval*/,
+ ExpressionPtr const& expr,
+ Configuration const& env) {
+ auto name_expr = expr["name"];
+ auto const& name = name_expr->String();
+ auto it = imports_.find(name);
+ if (it != imports_.end()) {
+ std::stringstream ss{};
+ bool user_context = false;
+ auto result = it->second->Evaluate(
+ env,
+ functions,
+ [&ss](auto const& msg) { ss << msg; },
+ [&user_context]() { user_context = true; }
+
+ );
+ if (result) {
+ return result;
+ }
+ if (user_context) {
+ throw Evaluator::EvaluationError(ss.str(), true, true);
+ }
+ throw Evaluator::EvaluationError(
+ fmt::format(
+ "This call to {} failed in the following way:\n{}",
+ name_expr->ToString(),
+ ss.str()),
+ true);
+ }
+ throw Evaluator::EvaluationError(
+ fmt::format("Unknown expression '{}'.", name));
+ };
+ auto newenv = env.Prune(vars_);
+ return expr_.Evaluate(
+ newenv,
+ FunctionMap::MakePtr(
+ functions, "CALL_EXPRESSION", imports_caller),
+ logger,
+ note_user_context);
+ } catch (...) {
+ gsl_EnsuresAudit(false); // ensure that the try-block never throws
+ return ExpressionPtr{nullptr};
+ }
+ }
+
+ inline static Ptr const kEmptyTransition =
+ std::make_shared<ExpressionFunction>(
+ std::vector<std::string>{},
+ ExpressionFunction::imports_t{},
+ Expression::FromJson(R"([{"type": "empty_map"}])"_json));
+
+ private:
+ std::vector<std::string> vars_{};
+ imports_t imports_{};
+ ExpressionPtr expr_{};
+};
+
+using ExpressionFunctionPtr = ExpressionFunction::Ptr;
+
+} // namespace BuildMaps::Base
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_FUNCTION_HPP
diff --git a/src/buildtool/build_engine/base_maps/expression_map.cpp b/src/buildtool/build_engine/base_maps/expression_map.cpp
new file mode 100644
index 00000000..df464b15
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/expression_map.cpp
@@ -0,0 +1,90 @@
+
+#include "src/buildtool/build_engine/base_maps/expression_map.hpp"
+
+#include <optional>
+#include <string>
+
+#include "fmt/core.h"
+#include "src/buildtool/build_engine/base_maps/field_reader.hpp"
+
+namespace BuildMaps::Base {
+
+auto CreateExpressionMap(gsl::not_null<ExpressionFileMap*> const& expr_file_map,
+ std::size_t jobs) -> ExpressionFunctionMap {
+ auto expr_func_creator = [expr_file_map](auto ts,
+ auto setter,
+ auto logger,
+ auto subcaller,
+ auto const& id) {
+ if (not id.IsDefinitionName()) {
+ (*logger)(
+ fmt::format("{} cannot name an expression", id.ToString()),
+ true);
+ return;
+ }
+ expr_file_map->ConsumeAfterKeysReady(
+ ts,
+ {id.ToModule()},
+ [setter = std::move(setter),
+ logger,
+ subcaller = std::move(subcaller),
+ id](auto json_values) {
+ auto func_it = json_values[0]->find(id.name);
+ if (func_it == json_values[0]->end()) {
+ (*logger)(fmt::format("Cannot find expression {} in {}",
+ id.name,
+ id.module),
+ true);
+ return;
+ }
+
+ auto reader = FieldReader::Create(
+ func_it.value(), id, "expression", logger);
+ if (not reader) {
+ return;
+ }
+
+ auto expr = reader->ReadExpression("expression");
+ if (not expr) {
+ return;
+ }
+
+ auto vars = reader->ReadStringList("vars");
+ if (not vars) {
+ return;
+ }
+
+ auto import_aliases =
+ reader->ReadEntityAliasesObject("imports");
+ if (not import_aliases) {
+ return;
+ }
+ auto [names, ids] = std::move(*import_aliases).Obtain();
+
+ (*subcaller)(
+ std::move(ids),
+ [setter = std::move(setter),
+ vars = std::move(*vars),
+ names = std::move(names),
+ expr = std::move(expr)](auto const& expr_funcs) {
+ auto imports = ExpressionFunction::imports_t{};
+ imports.reserve(expr_funcs.size());
+ for (std::size_t i{}; i < expr_funcs.size(); ++i) {
+ imports.emplace(names[i], *expr_funcs[i]);
+ }
+ (*setter)(std::make_shared<ExpressionFunction>(
+ vars, imports, expr));
+ },
+ std::move(logger));
+ },
+ [logger, id](auto msg, auto fatal) {
+ (*logger)(fmt::format("While reading expression file in {}: {}",
+ id.module,
+ msg),
+ fatal);
+ });
+ };
+ return ExpressionFunctionMap{expr_func_creator, jobs};
+}
+
+} // namespace BuildMaps::Base
diff --git a/src/buildtool/build_engine/base_maps/expression_map.hpp b/src/buildtool/build_engine/base_maps/expression_map.hpp
new file mode 100644
index 00000000..21ea8ec8
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/expression_map.hpp
@@ -0,0 +1,32 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_MAP_HPP
+
+#include <memory>
+#include <string>
+
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/base_maps/expression_function.hpp"
+#include "src/buildtool/build_engine/base_maps/json_file_map.hpp"
+#include "src/buildtool/build_engine/base_maps/module_name.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+namespace BuildMaps::Base {
+
+using ExpressionFileMap = AsyncMapConsumer<ModuleName, nlohmann::json>;
+
+constexpr auto CreateExpressionFileMap =
+ CreateJsonFileMap<&RepositoryConfig::ExpressionRoot,
+ &RepositoryConfig::ExpressionFileName,
+ /*kMandatory=*/true>;
+
+using ExpressionFunctionMap =
+ AsyncMapConsumer<EntityName, ExpressionFunctionPtr>;
+
+auto CreateExpressionMap(gsl::not_null<ExpressionFileMap*> const& expr_file_map,
+ std::size_t jobs = 0) -> ExpressionFunctionMap;
+
+} // namespace BuildMaps::Base
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_EXPRESSION_MAP_HPP
diff --git a/src/buildtool/build_engine/base_maps/field_reader.hpp b/src/buildtool/build_engine/base_maps/field_reader.hpp
new file mode 100644
index 00000000..30e0fe00
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/field_reader.hpp
@@ -0,0 +1,231 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_FIELD_READER_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_FIELD_READER_HPP
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "fmt/core.h"
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+namespace BuildMaps::Base {
+
+[[nodiscard]] static inline auto GetOrDefault(nlohmann::json const& json,
+ std::string const& key,
+ nlohmann::json&& default_value)
+ -> nlohmann::json {
+ auto value = json.find(key);
+ if (value != json.end()) {
+ return value.value();
+ }
+ return std::move(default_value);
+}
+
+class FieldReader {
+ public:
+ using Ptr = std::shared_ptr<FieldReader>;
+
+ class EntityAliases {
+ public:
+ [[nodiscard]] auto Obtain() && -> std::pair<std::vector<std::string>,
+ std::vector<EntityName>> {
+ return std::make_pair(std::move(names_), std::move(ids_));
+ }
+ auto reserve(std::size_t size) -> void {
+ names_.reserve(size);
+ ids_.reserve(size);
+ }
+ template <class T_Name, class T_Id>
+ auto emplace_back(T_Name&& name, T_Id&& id) -> void {
+ names_.emplace_back(std::forward<T_Name>(name));
+ ids_.emplace_back(std::forward<T_Id>(id));
+ }
+
+ private:
+ std::vector<std::string> names_;
+ std::vector<EntityName> ids_;
+ };
+
+ [[nodiscard]] static auto Create(nlohmann::json const& json,
+ EntityName const& id,
+ std::string const& entity_type,
+ AsyncMapConsumerLoggerPtr const& logger)
+ -> std::optional<FieldReader> {
+ if (not json.is_object()) {
+ (*logger)(
+ fmt::format(
+ "{} definition {} is not an object.", entity_type, id.name),
+ true);
+ return std::nullopt;
+ }
+ return FieldReader(json, id, entity_type, logger);
+ }
+
+ [[nodiscard]] static auto CreatePtr(nlohmann::json const& json,
+ EntityName const& id,
+ std::string const& entity_type,
+ AsyncMapConsumerLoggerPtr const& logger)
+ -> Ptr {
+ if (not json.is_object()) {
+ (*logger)(
+ fmt::format(
+ "{} definition {} is not an object.", entity_type, id.name),
+ true);
+ return nullptr;
+ }
+ return std::make_shared<FieldReader>(json, id, entity_type, logger);
+ }
+
+ [[nodiscard]] auto ReadExpression(std::string const& field_name) const
+ -> ExpressionPtr {
+ auto expr_it = json_.find(field_name);
+ if (expr_it == json_.end()) {
+ (*logger_)(fmt::format("Missing mandatory field {} in {} {}.",
+ field_name,
+ entity_type_,
+ id_.name),
+ true);
+ return ExpressionPtr{nullptr};
+ }
+
+ auto expr = Expression::FromJson(expr_it.value());
+ if (not expr) {
+ (*logger_)(
+ fmt::format("Failed to create expression from JSON:\n {}",
+ json_.dump()),
+ true);
+ }
+ return expr;
+ }
+
+ [[nodiscard]] auto ReadOptionalExpression(
+ std::string const& field_name,
+ ExpressionPtr const& default_value) const -> ExpressionPtr {
+ auto expr_it = json_.find(field_name);
+ if (expr_it == json_.end()) {
+ return default_value;
+ }
+
+ auto expr = Expression::FromJson(expr_it.value());
+ if (not expr) {
+ (*logger_)(
+ fmt::format("Failed to create expression from JSON:\n {}",
+ json_.dump()),
+ true);
+ }
+ return expr;
+ }
+
+ [[nodiscard]] auto ReadStringList(std::string const& field_name) const
+ -> std::optional<std::vector<std::string>> {
+ auto const& list =
+ GetOrDefault(json_, field_name, nlohmann::json::array());
+ if (not list.is_array()) {
+ (*logger_)(fmt::format("Field {} in {} {} is not a list",
+ field_name,
+ entity_type_,
+ id_.name),
+ true);
+ return std::nullopt;
+ }
+
+ auto vars = std::vector<std::string>{};
+ vars.reserve(list.size());
+
+ try {
+ std::transform(
+ list.begin(),
+ list.end(),
+ std::back_inserter(vars),
+ [](auto const& j) { return j.template get<std::string>(); });
+ } catch (...) {
+ (*logger_)(fmt::format("List entry in {} of {} {} is not a string",
+ field_name,
+ entity_type_,
+ id_.name),
+ true);
+ return std::nullopt;
+ }
+
+ return vars;
+ }
+
+ [[nodiscard]] auto ReadEntityAliasesObject(
+ std::string const& field_name) const -> std::optional<EntityAliases> {
+ auto const& map =
+ GetOrDefault(json_, field_name, nlohmann::json::object());
+ if (not map.is_object()) {
+ (*logger_)(fmt::format("Field {} in {} {} is not an object",
+ field_name,
+ entity_type_,
+ id_.name),
+ true);
+ return std::nullopt;
+ }
+
+ auto imports = EntityAliases{};
+ imports.reserve(map.size());
+
+ for (auto const& [key, val] : map.items()) {
+ auto expr_id = ParseEntityNameFromJson(
+ val,
+ id_,
+ [this, &field_name, entry = val.dump()](
+ std::string const& parse_err) {
+ (*logger_)(fmt::format("Parsing entry {} in field {} of {} "
+ "{} failed with:\n{}",
+ entry,
+ field_name,
+ entity_type_,
+ id_.name,
+ parse_err),
+ true);
+ });
+ if (not expr_id) {
+ return std::nullopt;
+ }
+ imports.emplace_back(key, *expr_id);
+ }
+ return imports;
+ }
+
+ void ExpectFields(std::unordered_set<std::string> const& expected) {
+ auto unexpected = nlohmann::json::array();
+ for (auto const& [key, value] : json_.items()) {
+ if (not expected.contains(key)) {
+ unexpected.push_back(key);
+ }
+ }
+ if (not unexpected.empty()) {
+ (*logger_)(fmt::format("{} {} has unexpected parameters {}",
+ entity_type_,
+ id_.ToString(),
+ unexpected.dump()),
+ false);
+ }
+ }
+
+ FieldReader(nlohmann::json json,
+ EntityName id,
+ std::string entity_type,
+ AsyncMapConsumerLoggerPtr logger) noexcept
+ : json_{std::move(json)},
+ id_{std::move(id)},
+ entity_type_{std::move(entity_type)},
+ logger_{std::move(logger)} {}
+
+ private:
+ nlohmann::json json_;
+ EntityName id_;
+ std::string entity_type_;
+ AsyncMapConsumerLoggerPtr logger_;
+};
+
+} // namespace BuildMaps::Base
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_FIELD_READER_HPP
diff --git a/src/buildtool/build_engine/base_maps/json_file_map.hpp b/src/buildtool/build_engine/base_maps/json_file_map.hpp
new file mode 100644
index 00000000..bf7495d7
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/json_file_map.hpp
@@ -0,0 +1,93 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_JSON_FILE_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_JSON_FILE_MAP_HPP
+
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+#include "fmt/core.h"
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/base_maps/module_name.hpp"
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+namespace BuildMaps::Base {
+
+using JsonFileMap = AsyncMapConsumer<ModuleName, nlohmann::json>;
+
+// function pointer type for specifying which root to get from global config
+using RootGetter = auto (RepositoryConfig::*)(std::string const&) const
+ -> FileRoot const*;
+
+// function pointer type for specifying the file name from the global config
+using FileNameGetter = auto (RepositoryConfig::*)(std::string const&) const
+ -> std::string const*;
+
+template <RootGetter get_root, FileNameGetter get_name, bool kMandatory = true>
+auto CreateJsonFileMap(std::size_t jobs) -> JsonFileMap {
+ auto json_file_reader = [](auto /* unused */,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
+ auto const& config = RepositoryConfig::Instance();
+ auto const* root = (config.*get_root)(key.repository);
+ auto const* json_file_name = (config.*get_name)(key.repository);
+ if (root == nullptr or json_file_name == nullptr) {
+ (*logger)(fmt::format("Cannot determine root or JSON file name for "
+ "repository {}.",
+ key.repository),
+ true);
+ return;
+ }
+ auto module = std::filesystem::path{key.module}.lexically_normal();
+ if (module.is_absolute() or *module.begin() == "..") {
+ (*logger)(fmt::format("Modules have to live inside their "
+ "repository, but found {}.",
+ key.module),
+ true);
+ return;
+ }
+ auto json_file_path = module / *json_file_name;
+
+ if (not root->IsFile(json_file_path)) {
+ if constexpr (kMandatory) {
+ (*logger)(fmt::format("JSON file {} does not exist.",
+ json_file_path.string()),
+ true);
+ }
+ else {
+ (*setter)(nlohmann::json::object());
+ }
+ return;
+ }
+
+ auto const file_content = root->ReadFile(json_file_path);
+ if (not file_content) {
+ (*logger)(fmt::format("cannot read JSON file {}.",
+ json_file_path.string()),
+ true);
+ return;
+ }
+ auto json = nlohmann::json::parse(*file_content, nullptr, false);
+ if (json.is_discarded()) {
+ (*logger)(fmt::format("JSON file {} does not contain valid JSON.",
+ json_file_path.string()),
+ true);
+ return;
+ }
+ if (!json.is_object()) {
+ (*logger)(fmt::format("JSON in {} is not an object.",
+ json_file_path.string()),
+ true);
+ return;
+ }
+ (*setter)(std::move(json));
+ };
+ return AsyncMapConsumer<ModuleName, nlohmann::json>{json_file_reader, jobs};
+}
+
+} // namespace BuildMaps::Base
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_JSON_FILE_MAP_HPP
diff --git a/src/buildtool/build_engine/base_maps/module_name.hpp b/src/buildtool/build_engine/base_maps/module_name.hpp
new file mode 100644
index 00000000..26465bf6
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/module_name.hpp
@@ -0,0 +1,36 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_MODULE_NAME_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_MODULE_NAME_HPP
+
+#include "src/utils/cpp/hash_combine.hpp"
+
+namespace BuildMaps::Base {
+
+struct ModuleName {
+ std::string repository{};
+ std::string module{};
+
+ ModuleName(std::string repository, std::string module)
+ : repository{std::move(repository)}, module{std::move(module)} {}
+
+ [[nodiscard]] auto operator==(ModuleName const& other) const noexcept
+ -> bool {
+ return module == other.module && repository == other.repository;
+ }
+};
+} // namespace BuildMaps::Base
+
+namespace std {
+template <>
+struct hash<BuildMaps::Base::ModuleName> {
+ [[nodiscard]] auto operator()(
+ const BuildMaps::Base::ModuleName& t) const noexcept -> std::size_t {
+ size_t seed{};
+ hash_combine<std::string>(&seed, t.repository);
+ hash_combine<std::string>(&seed, t.module);
+ return seed;
+ }
+};
+
+} // namespace std
+
+#endif
diff --git a/src/buildtool/build_engine/base_maps/rule_map.cpp b/src/buildtool/build_engine/base_maps/rule_map.cpp
new file mode 100644
index 00000000..7bdc14e4
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/rule_map.cpp
@@ -0,0 +1,371 @@
+
+#include "src/buildtool/build_engine/base_maps/rule_map.hpp"
+
+#include <optional>
+#include <string>
+#include <unordered_set>
+
+#include "fmt/core.h"
+#include "src/buildtool/build_engine/base_maps/field_reader.hpp"
+
+namespace BuildMaps::Base {
+
+namespace {
+
+auto rule_fields = std::unordered_set<std::string>{"anonymous",
+ "config_doc",
+ "config_fields",
+ "config_transitions",
+ "config_vars",
+ "doc",
+ "expression",
+ "field_doc",
+ "implicit",
+ "imports",
+ "import",
+ "string_fields",
+ "tainted",
+ "target_fields"};
+
+[[nodiscard]] auto ReadAnonymousObject(EntityName const& id,
+ nlohmann::json const& json,
+ AsyncMapConsumerLoggerPtr const& logger)
+ -> std::optional<UserRule::anonymous_defs_t> {
+ auto obj = GetOrDefault(json, "anonymous", nlohmann::json::object());
+ if (not obj.is_object()) {
+ (*logger)(
+ fmt::format("Field anonymous in rule {} is not an object", id.name),
+ true);
+ return std::nullopt;
+ }
+
+ UserRule::anonymous_defs_t anon_defs{};
+ anon_defs.reserve(obj.size());
+ for (auto const& [name, def] : obj.items()) {
+ if (not def.is_object()) {
+ (*logger)(fmt::format("Entry {} in field anonymous in rule {} is "
+ "not an object",
+ name,
+ id.name),
+ true);
+ return std::nullopt;
+ }
+
+ auto target = def.find("target");
+ if (target == def.end()) {
+ (*logger)(fmt::format("Entry target for {} in field anonymous in "
+ "rule {} is missing",
+ name,
+ id.name),
+ true);
+ return std::nullopt;
+ }
+ if (not target->is_string()) {
+ (*logger)(fmt::format("Entry target for {} in field anonymous in "
+ "rule {} is not a string",
+ name,
+ id.name),
+ true);
+ return std::nullopt;
+ }
+
+ auto provider = def.find("provider");
+ if (provider == def.end()) {
+ (*logger)(fmt::format("Entry provider for {} in field anonymous in "
+ "rule {} is missing",
+ name,
+ id.name),
+ true);
+ return std::nullopt;
+ }
+ if (not provider->is_string()) {
+ (*logger)(fmt::format("Entry provider for {} in field anonymous in "
+ "rule {} is not a string",
+ name,
+ id.name),
+ true);
+ return std::nullopt;
+ }
+
+ auto rule_map = def.find("rule_map");
+ if (rule_map == def.end()) {
+ (*logger)(fmt::format("Entry rule_map for {} in field anonymous in "
+ "rule {} is missing",
+ name,
+ id.name),
+ true);
+ return std::nullopt;
+ }
+ if (not rule_map->is_object()) {
+ (*logger)(fmt::format("Entry rule_map for {} in field anonymous in "
+ "rule {} is not an object",
+ name,
+ id.name),
+ true);
+ return std::nullopt;
+ }
+
+ Expression::map_t::underlying_map_t rule_mapping{};
+ for (auto const& [key, val] : rule_map->items()) {
+ auto rule_name = ParseEntityNameFromJson(
+ val, id, [&logger, &id, &name = name](auto msg) {
+ (*logger)(
+ fmt::format("Parsing rule name for entry {} in field "
+ "anonymous in rule {} failed with:\n{}",
+ name,
+ id.name,
+ msg),
+ true);
+ });
+ if (not rule_name) {
+ return std::nullopt;
+ }
+ rule_mapping.emplace(key, ExpressionPtr{std::move(*rule_name)});
+ }
+
+ anon_defs.emplace(
+ name,
+ UserRule::AnonymousDefinition{
+ target->get<std::string>(),
+ provider->get<std::string>(),
+ ExpressionPtr{Expression::map_t{std::move(rule_mapping)}}});
+ }
+ return anon_defs;
+}
+
+[[nodiscard]] auto ReadImplicitObject(EntityName const& id,
+ nlohmann::json const& json,
+ AsyncMapConsumerLoggerPtr const& logger)
+ -> std::optional<UserRule::implicit_t> {
+ auto map = GetOrDefault(json, "implicit", nlohmann::json::object());
+ if (not map.is_object()) {
+ (*logger)(
+ fmt::format("Field implicit in rule {} is not an object", id.name),
+ true);
+ return std::nullopt;
+ }
+
+ auto implicit_targets = UserRule::implicit_t{};
+ implicit_targets.reserve(map.size());
+
+ for (auto const& [key, val] : map.items()) {
+ if (not val.is_array()) {
+ (*logger)(fmt::format("Entry in implicit field of rule {} is not a "
+ "list.",
+ id.name),
+ true);
+ return std::nullopt;
+ }
+ auto targets = typename UserRule::implicit_t::mapped_type{};
+ targets.reserve(val.size());
+ for (auto const& item : val) {
+ auto expr_id = ParseEntityNameFromJson(
+ item, id, [&logger, &item, &id](std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing entry {} in implicit field "
+ "of rule {} failed with:\n{}",
+ item.dump(),
+ id.name,
+ parse_err),
+ true);
+ });
+ if (not expr_id) {
+ return std::nullopt;
+ }
+ targets.emplace_back(*expr_id);
+ }
+ implicit_targets.emplace(key, targets);
+ }
+ return implicit_targets;
+}
+
+[[nodiscard]] auto ReadConfigTransitionsObject(
+ EntityName const& id,
+ nlohmann::json const& json,
+ std::vector<std::string> const& config_vars,
+ ExpressionFunction::imports_t const& imports,
+ AsyncMapConsumerLoggerPtr const& logger)
+ -> std::optional<UserRule::config_trans_t> {
+ auto map =
+ GetOrDefault(json, "config_transitions", nlohmann::json::object());
+ if (not map.is_object()) {
+ (*logger)(
+ fmt::format("Field config_transitions in rule {} is not an object",
+ id.name),
+ true);
+ return std::nullopt;
+ }
+
+ auto config_transitions = UserRule::config_trans_t{};
+ config_transitions.reserve(map.size());
+
+ for (auto const& [key, val] : map.items()) {
+ auto expr = Expression::FromJson(val);
+ if (not expr) {
+ (*logger)(fmt::format("Failed to create expression for entry {} in "
+ "config_transitions list of rule {}.",
+ key,
+ id.name),
+ true);
+ return std::nullopt;
+ }
+ config_transitions.emplace(
+ key,
+ std::make_shared<ExpressionFunction>(config_vars, imports, expr));
+ }
+ return config_transitions;
+}
+
+} // namespace
+
+auto CreateRuleMap(gsl::not_null<RuleFileMap*> const& rule_file_map,
+ gsl::not_null<ExpressionFunctionMap*> const& expr_map,
+ std::size_t jobs) -> UserRuleMap {
+ auto user_rule_creator = [rule_file_map, expr_map](auto ts,
+ auto setter,
+ auto logger,
+ auto /*subcaller*/,
+ auto const& id) {
+ if (not id.IsDefinitionName()) {
+ (*logger)(fmt::format("{} cannot name a rule", id.ToString()),
+ true);
+ return;
+ }
+ rule_file_map->ConsumeAfterKeysReady(
+ ts,
+ {id.ToModule()},
+ [ts, expr_map, setter = std::move(setter), logger, id](
+ auto json_values) {
+ auto rule_it = json_values[0]->find(id.name);
+ if (rule_it == json_values[0]->end()) {
+ (*logger)(
+ fmt::format(
+ "Cannot find rule {} in {}", id.name, id.module),
+ true);
+ return;
+ }
+
+ auto reader =
+ FieldReader::Create(rule_it.value(), id, "rule", logger);
+ if (not reader) {
+ return;
+ }
+ reader->ExpectFields(rule_fields);
+
+ auto expr = reader->ReadExpression("expression");
+ if (not expr) {
+ return;
+ }
+
+ auto target_fields = reader->ReadStringList("target_fields");
+ if (not target_fields) {
+ return;
+ }
+
+ auto string_fields = reader->ReadStringList("string_fields");
+ if (not string_fields) {
+ return;
+ }
+
+ auto config_fields = reader->ReadStringList("config_fields");
+ if (not config_fields) {
+ return;
+ }
+
+ auto implicit_targets =
+ ReadImplicitObject(id, rule_it.value(), logger);
+ if (not implicit_targets) {
+ return;
+ }
+
+ auto anonymous_defs =
+ ReadAnonymousObject(id, rule_it.value(), logger);
+ if (not anonymous_defs) {
+ return;
+ }
+
+ auto config_vars = reader->ReadStringList("config_vars");
+ if (not config_vars) {
+ return;
+ }
+
+ auto tainted = reader->ReadStringList("tainted");
+ if (not tainted) {
+ return;
+ }
+
+ auto import_aliases =
+ reader->ReadEntityAliasesObject("imports");
+ if (not import_aliases) {
+ return;
+ }
+ auto [names, ids] = std::move(*import_aliases).Obtain();
+
+ expr_map->ConsumeAfterKeysReady(
+ ts,
+ std::move(ids),
+ [ts,
+ id,
+ json = rule_it.value(),
+ expr = std::move(expr),
+ target_fields = std::move(*target_fields),
+ string_fields = std::move(*string_fields),
+ config_fields = std::move(*config_fields),
+ implicit_targets = std::move(*implicit_targets),
+ anonymous_defs = std::move(*anonymous_defs),
+ config_vars = std::move(*config_vars),
+ tainted = std::move(*tainted),
+ names = std::move(names),
+ setter = std::move(setter),
+ logger](auto expr_funcs) {
+ auto imports = ExpressionFunction::imports_t{};
+ imports.reserve(expr_funcs.size());
+ for (std::size_t i{}; i < expr_funcs.size(); ++i) {
+ imports.emplace(names[i], *expr_funcs[i]);
+ }
+
+ auto config_transitions = ReadConfigTransitionsObject(
+ id, json, config_vars, imports, logger);
+ if (not config_transitions) {
+ return;
+ }
+
+ auto rule = UserRule::Create(
+ target_fields,
+ string_fields,
+ config_fields,
+ implicit_targets,
+ anonymous_defs,
+ config_vars,
+ tainted,
+ std::move(*config_transitions),
+ std::make_shared<ExpressionFunction>(
+ std::move(config_vars),
+ std::move(imports),
+ std::move(expr)),
+ [&logger](auto const& msg) {
+ (*logger)(msg, true);
+ });
+ if (rule) {
+ (*setter)(std::move(rule));
+ }
+ },
+ [logger, id](auto msg, auto fatal) {
+ (*logger)(fmt::format("While reading expression map "
+ "for rule {} in {}: {}",
+ id.name,
+ id.module,
+ msg),
+ fatal);
+ });
+ },
+ [logger, id](auto msg, auto fatal) {
+ (*logger)(
+ fmt::format(
+ "While reading rule file in {}: {}", id.module, msg),
+ fatal);
+ });
+ };
+ return UserRuleMap{user_rule_creator, jobs};
+}
+
+} // namespace BuildMaps::Base
diff --git a/src/buildtool/build_engine/base_maps/rule_map.hpp b/src/buildtool/build_engine/base_maps/rule_map.hpp
new file mode 100644
index 00000000..1547e720
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/rule_map.hpp
@@ -0,0 +1,33 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_RULE_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_RULE_MAP_HPP
+
+#include <memory>
+#include <string>
+
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/base_maps/expression_map.hpp"
+#include "src/buildtool/build_engine/base_maps/json_file_map.hpp"
+#include "src/buildtool/build_engine/base_maps/module_name.hpp"
+#include "src/buildtool/build_engine/base_maps/user_rule.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+namespace BuildMaps::Base {
+
+using RuleFileMap = AsyncMapConsumer<ModuleName, nlohmann::json>;
+
+constexpr auto CreateRuleFileMap =
+ CreateJsonFileMap<&RepositoryConfig::RuleRoot,
+ &RepositoryConfig::RuleFileName,
+ /*kMandatory=*/true>;
+
+using UserRuleMap = AsyncMapConsumer<EntityName, UserRulePtr>;
+
+auto CreateRuleMap(gsl::not_null<RuleFileMap*> const& rule_file_map,
+ gsl::not_null<ExpressionFunctionMap*> const& expr_map,
+ std::size_t jobs = 0) -> UserRuleMap;
+
+} // namespace BuildMaps::Base
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_RULE_MAP_HPP
diff --git a/src/buildtool/build_engine/base_maps/source_map.cpp b/src/buildtool/build_engine/base_maps/source_map.cpp
new file mode 100644
index 00000000..5c2a79d0
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/source_map.cpp
@@ -0,0 +1,88 @@
+#include "src/buildtool/build_engine/base_maps/source_map.hpp"
+
+#include <filesystem>
+
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+#include "src/utils/cpp/json.hpp"
+
+namespace BuildMaps::Base {
+
+namespace {
+
+auto as_target(const BuildMaps::Base::EntityName& key, ExpressionPtr artifact)
+ -> AnalysedTargetPtr {
+ auto stage =
+ ExpressionPtr{Expression::map_t{key.name, std::move(artifact)}};
+ return std::make_shared<AnalysedTarget>(
+ TargetResult{stage, Expression::kEmptyMap, stage},
+ std::vector<ActionDescription>{},
+ std::vector<std::string>{},
+ std::vector<Tree>{},
+ std::unordered_set<std::string>{},
+ std::set<std::string>{});
+}
+
+} // namespace
+
+auto CreateSourceTargetMap(const gsl::not_null<DirectoryEntriesMap*>& dirs,
+ std::size_t jobs) -> SourceTargetMap {
+ auto src_target_reader = [dirs](auto ts,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
+ using std::filesystem::path;
+ auto name = path(key.name).lexically_normal();
+ if (name.is_absolute() or *name.begin() == "..") {
+ (*logger)(
+ fmt::format("Source file reference outside current module: {}",
+ key.name),
+ true);
+ return;
+ }
+ auto dir = (path(key.module) / name).parent_path();
+ auto const* ws_root =
+ RepositoryConfig::Instance().WorkspaceRoot(key.repository);
+
+ auto src_file_reader = [ts, key, name, setter, logger, dir, ws_root](
+ bool exists_in_ws_root) {
+ if (ws_root != nullptr and exists_in_ws_root) {
+ if (auto desc = ws_root->ToArtifactDescription(
+ path(key.module) / name, key.repository)) {
+ (*setter)(as_target(key, ExpressionPtr{std::move(*desc)}));
+ return;
+ }
+ }
+ (*logger)(fmt::format("Cannot determine source file {}",
+ path(key.name).filename().string()),
+ true);
+ };
+
+ if (ws_root != nullptr and ws_root->HasFastDirectoryLookup()) {
+ // by-pass directory map and directly attempt to read from ws_root
+ src_file_reader(ws_root->IsFile(path(key.module) / name));
+ return;
+ }
+ dirs->ConsumeAfterKeysReady(
+ ts,
+ {ModuleName{key.repository, dir.string()}},
+ [key, src_file_reader](auto values) {
+ src_file_reader(
+ values[0]->Contains(path(key.name).filename().string()));
+ },
+ [logger, dir](auto msg, auto fatal) {
+ (*logger)(
+ fmt::format(
+ "While reading contents of {}: {}", dir.string(), msg),
+ fatal);
+ }
+
+ );
+ };
+ return AsyncMapConsumer<EntityName, AnalysedTargetPtr>(src_target_reader,
+ jobs);
+}
+
+}; // namespace BuildMaps::Base
diff --git a/src/buildtool/build_engine/base_maps/source_map.hpp b/src/buildtool/build_engine/base_maps/source_map.hpp
new file mode 100644
index 00000000..a8e9fd9b
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/source_map.hpp
@@ -0,0 +1,24 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_SOURCE_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_SOURCE_MAP_HPP
+
+#include <unordered_set>
+
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
+#include "src/buildtool/build_engine/base_maps/directory_map.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+#include "src/buildtool/multithreading/task_system.hpp"
+
+namespace BuildMaps::Base {
+
+using SourceTargetMap = AsyncMapConsumer<EntityName, AnalysedTargetPtr>;
+
+auto CreateSourceTargetMap(const gsl::not_null<DirectoryEntriesMap*>& dirs,
+ std::size_t jobs = 0) -> SourceTargetMap;
+
+} // namespace BuildMaps::Base
+
+#endif
diff --git a/src/buildtool/build_engine/base_maps/targets_file_map.hpp b/src/buildtool/build_engine/base_maps/targets_file_map.hpp
new file mode 100644
index 00000000..23db052e
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/targets_file_map.hpp
@@ -0,0 +1,23 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_TARGETS_FILE_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_TARGETS_FILE_MAP_HPP
+
+#include <filesystem>
+#include <string>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/build_engine/base_maps/json_file_map.hpp"
+#include "src/buildtool/build_engine/base_maps/module_name.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+namespace BuildMaps::Base {
+
+using TargetsFileMap = AsyncMapConsumer<ModuleName, nlohmann::json>;
+
+constexpr auto CreateTargetsFileMap =
+ CreateJsonFileMap<&RepositoryConfig::TargetRoot,
+ &RepositoryConfig::TargetFileName,
+ /*kMandatory=*/true>;
+
+} // namespace BuildMaps::Base
+
+#endif
diff --git a/src/buildtool/build_engine/base_maps/user_rule.hpp b/src/buildtool/build_engine/base_maps/user_rule.hpp
new file mode 100644
index 00000000..807e3478
--- /dev/null
+++ b/src/buildtool/build_engine/base_maps/user_rule.hpp
@@ -0,0 +1,404 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_USER_RULE_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_USER_RULE_HPP
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "fmt/core.h"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/base_maps/expression_function.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/utils/cpp/concepts.hpp"
+
+namespace BuildMaps::Base {
+
+// Get duplicates from containers.
+// NOTE: Requires all input containers to be sorted!
+// kTriangular=true Performs triangular compare, everyone with everyone.
+// kTriangular=false Performs linear compare, first with each of the rest.
+template <bool kTriangular,
+ InputIterableContainer T_Container,
+ InputIterableContainer... T_Rest,
+ OutputIterableContainer T_Result =
+ std::unordered_set<typename T_Container::value_type>>
+[[nodiscard]] static inline auto GetDuplicates(T_Container const& first,
+ T_Rest const&... rest)
+ -> T_Result;
+
+template <InputIterableStringContainer T_Container>
+[[nodiscard]] static inline auto JoinContainer(T_Container const& c,
+ std::string const& sep)
+ -> std::string;
+
+class UserRule {
+ public:
+ using Ptr = std::shared_ptr<UserRule>;
+ using implicit_t = std::unordered_map<std::string, std::vector<EntityName>>;
+ using implicit_exp_t = std::unordered_map<std::string, ExpressionPtr>;
+ using config_trans_t =
+ std::unordered_map<std::string, ExpressionFunctionPtr>;
+
+ struct AnonymousDefinition {
+ std::string target;
+ std::string provider;
+ ExpressionPtr rule_map;
+ };
+ using anonymous_defs_t =
+ std::unordered_map<std::string, AnonymousDefinition>;
+
+ [[nodiscard]] static auto Create(
+ std::vector<std::string> target_fields,
+ std::vector<std::string> string_fields,
+ std::vector<std::string> config_fields,
+ implicit_t const& implicit_targets,
+ anonymous_defs_t anonymous_defs,
+ std::vector<std::string> const& config_vars,
+ std::vector<std::string> const& tainted,
+ config_trans_t config_transitions,
+ ExpressionFunctionPtr const& expr,
+ std::function<void(std::string const&)> const& logger) -> Ptr {
+
+ auto implicit_fields = std::vector<std::string>{};
+ implicit_fields.reserve(implicit_targets.size());
+ std::transform(implicit_targets.begin(),
+ implicit_targets.end(),
+ std::back_inserter(implicit_fields),
+ [](auto const& el) { return el.first; });
+ std::sort(implicit_fields.begin(), implicit_fields.end());
+
+ auto anonymous_fields = std::vector<std::string>{};
+ anonymous_fields.reserve(anonymous_defs.size());
+ std::transform(anonymous_defs.begin(),
+ anonymous_defs.end(),
+ std::back_inserter(anonymous_fields),
+ [](auto const& el) { return el.first; });
+ std::sort(anonymous_fields.begin(), anonymous_fields.end());
+
+ std::sort(target_fields.begin(), target_fields.end());
+ std::sort(string_fields.begin(), string_fields.end());
+ std::sort(config_fields.begin(), config_fields.end());
+
+ auto dups = GetDuplicates</*kTriangular=*/false>(kReservedKeywords,
+ target_fields,
+ string_fields,
+ config_fields,
+ implicit_fields,
+ anonymous_fields);
+ if (not dups.empty()) {
+ logger(
+ fmt::format("User-defined fields cannot be any of the reserved "
+ "fields [{}]",
+ JoinContainer(kReservedKeywords, ",")));
+ return nullptr;
+ }
+
+ dups = GetDuplicates</*kTriangular=*/true>(target_fields,
+ string_fields,
+ config_fields,
+ implicit_fields,
+ anonymous_fields);
+
+ if (not dups.empty()) {
+ logger(
+ fmt::format("A field can have only one type, but the following "
+ "have more: [{}]",
+ JoinContainer(dups, ",")));
+ return nullptr;
+ }
+
+ auto transition_targets = std::vector<std::string>{};
+ transition_targets.reserve(config_transitions.size());
+ std::transform(config_transitions.begin(),
+ config_transitions.end(),
+ std::back_inserter(transition_targets),
+ [](auto const& el) { return el.first; });
+ std::sort(transition_targets.begin(), transition_targets.end());
+
+ dups = GetDuplicates</*kTriangular=*/false>(transition_targets,
+ target_fields,
+ implicit_fields,
+ anonymous_fields);
+ if (dups != decltype(dups){transition_targets.begin(),
+ transition_targets.end()}) {
+ logger(
+ fmt::format("Config transitions has to be a map from target "
+ "fields to transition expressions, but found [{}]",
+ JoinContainer(transition_targets, ",")));
+ return nullptr;
+ }
+
+ auto const setter = [&config_transitions](auto const field) {
+ config_transitions.emplace(
+ field,
+ ExpressionFunction::kEmptyTransition); // wont overwrite
+ };
+ config_transitions.reserve(target_fields.size() +
+ implicit_fields.size() +
+ anonymous_fields.size());
+ std::for_each(target_fields.begin(), target_fields.end(), setter);
+ std::for_each(implicit_fields.begin(), implicit_fields.end(), setter);
+ std::for_each(anonymous_fields.begin(), anonymous_fields.end(), setter);
+
+ implicit_exp_t implicit_target_exp;
+ implicit_target_exp.reserve(implicit_targets.size());
+ for (auto const& [target_name, target_entity_vec] : implicit_targets) {
+ std::vector<ExpressionPtr> target_exps;
+ target_exps.reserve(target_entity_vec.size());
+ for (auto const& target_entity : target_entity_vec) {
+ target_exps.emplace_back(ExpressionPtr{target_entity});
+ }
+ implicit_target_exp.emplace(target_name, target_exps);
+ }
+
+ return std::make_shared<UserRule>(
+ std::move(target_fields),
+ std::move(string_fields),
+ std::move(config_fields),
+ implicit_targets,
+ std::move(implicit_target_exp),
+ std::move(anonymous_defs),
+ config_vars,
+ std::set<std::string>{tainted.begin(), tainted.end()},
+ std::move(config_transitions),
+ expr);
+ }
+
+ UserRule(std::vector<std::string> target_fields,
+ std::vector<std::string> string_fields,
+ std::vector<std::string> config_fields,
+ implicit_t implicit_targets,
+ implicit_exp_t implicit_target_exp,
+ anonymous_defs_t anonymous_defs,
+ std::vector<std::string> config_vars,
+ std::set<std::string> tainted,
+ config_trans_t config_transitions,
+ ExpressionFunctionPtr expr) noexcept
+ : target_fields_{std::move(target_fields)},
+ string_fields_{std::move(string_fields)},
+ config_fields_{std::move(config_fields)},
+ implicit_targets_{std::move(implicit_targets)},
+ implicit_target_exp_{std::move(implicit_target_exp)},
+ anonymous_defs_{std::move(anonymous_defs)},
+ config_vars_{std::move(config_vars)},
+ tainted_{std::move(tainted)},
+ config_transitions_{std::move(config_transitions)},
+ expr_{std::move(expr)} {}
+
+ [[nodiscard]] auto TargetFields() const& noexcept
+ -> std::vector<std::string> const& {
+ return target_fields_;
+ }
+
+ [[nodiscard]] auto TargetFields() && noexcept -> std::vector<std::string> {
+ return std::move(target_fields_);
+ }
+
+ [[nodiscard]] auto StringFields() const& noexcept
+ -> std::vector<std::string> const& {
+ return string_fields_;
+ }
+
+ [[nodiscard]] auto StringFields() && noexcept -> std::vector<std::string> {
+ return std::move(string_fields_);
+ }
+
+ [[nodiscard]] auto ConfigFields() const& noexcept
+ -> std::vector<std::string> const& {
+ return config_fields_;
+ }
+
+ [[nodiscard]] auto ConfigFields() && noexcept -> std::vector<std::string> {
+ return std::move(config_fields_);
+ }
+
+ [[nodiscard]] auto ImplicitTargets() const& noexcept -> implicit_t const& {
+ return implicit_targets_;
+ }
+
+ [[nodiscard]] auto ImplicitTargets() && noexcept -> implicit_t {
+ return std::move(implicit_targets_);
+ }
+
+ [[nodiscard]] auto ImplicitTargetExps() const& noexcept
+ -> implicit_exp_t const& {
+ return implicit_target_exp_;
+ }
+
+ [[nodiscard]] auto ExpectedFields() const& noexcept
+ -> std::unordered_set<std::string> const& {
+ return expected_entries_;
+ }
+
+ [[nodiscard]] auto ConfigVars() const& noexcept
+ -> std::vector<std::string> const& {
+ return config_vars_;
+ }
+
+ [[nodiscard]] auto ConfigVars() && noexcept -> std::vector<std::string> {
+ return std::move(config_vars_);
+ }
+
+ [[nodiscard]] auto Tainted() const& noexcept
+ -> std::set<std::string> const& {
+ return tainted_;
+ }
+
+ [[nodiscard]] auto Tainted() && noexcept -> std::set<std::string> {
+ return std::move(tainted_);
+ }
+
+ [[nodiscard]] auto ConfigTransitions() const& noexcept
+ -> config_trans_t const& {
+ return config_transitions_;
+ }
+
+ [[nodiscard]] auto ConfigTransitions() && noexcept -> config_trans_t {
+ return std::move(config_transitions_);
+ }
+
+ [[nodiscard]] auto Expression() const& noexcept
+ -> ExpressionFunctionPtr const& {
+ return expr_;
+ }
+
+ [[nodiscard]] auto Expression() && noexcept -> ExpressionFunctionPtr {
+ return std::move(expr_);
+ }
+
+ [[nodiscard]] auto AnonymousDefinitions() const& noexcept
+ -> anonymous_defs_t {
+ return anonymous_defs_;
+ }
+
+ [[nodiscard]] auto AnonymousDefinitions() && noexcept -> anonymous_defs_t {
+ return std::move(anonymous_defs_);
+ }
+
+ private:
+ // NOTE: Must be sorted
+ static inline std::vector<std::string> const kReservedKeywords{
+ "arguments_config",
+ "tainted",
+ "type"};
+
+ static auto ComputeExpectedEntries(std::vector<std::string> tfields,
+ std::vector<std::string> sfields,
+ std::vector<std::string> cfields)
+ -> std::unordered_set<std::string> {
+ size_t n = 0;
+ n += tfields.size();
+ n += sfields.size();
+ n += cfields.size();
+ n += kReservedKeywords.size();
+ std::unordered_set<std::string> expected_entries{};
+ expected_entries.reserve(n);
+ expected_entries.insert(tfields.begin(), tfields.end());
+ expected_entries.insert(sfields.begin(), sfields.end());
+ expected_entries.insert(cfields.begin(), cfields.end());
+ expected_entries.insert(kReservedKeywords.begin(),
+ kReservedKeywords.end());
+ return expected_entries;
+ }
+
+ std::vector<std::string> target_fields_{};
+ std::vector<std::string> string_fields_{};
+ std::vector<std::string> config_fields_{};
+ implicit_t implicit_targets_{};
+ implicit_exp_t implicit_target_exp_{};
+ anonymous_defs_t anonymous_defs_{};
+ std::vector<std::string> config_vars_{};
+ std::set<std::string> tainted_{};
+ config_trans_t config_transitions_{};
+ ExpressionFunctionPtr expr_{};
+ std::unordered_set<std::string> expected_entries_{
+ ComputeExpectedEntries(target_fields_, string_fields_, config_fields_)};
+};
+
+using UserRulePtr = UserRule::Ptr;
+
+namespace detail {
+
+template <HasSize T_Container, HasSize... T_Rest>
+[[nodiscard]] static inline auto MaxSize(T_Container const& first,
+ T_Rest const&... rest) -> std::size_t {
+ if constexpr (sizeof...(rest) > 0) {
+ return std::max(first.size(), MaxSize(rest...));
+ }
+ return first.size();
+}
+
+template <bool kTriangular,
+ OutputIterableContainer T_Result,
+ InputIterableContainer T_First,
+ InputIterableContainer T_Second,
+ InputIterableContainer... T_Rest>
+static auto inline FindDuplicates(gsl::not_null<T_Result*> const& dups,
+ T_First const& first,
+ T_Second const& second,
+ T_Rest const&... rest) -> void {
+ gsl_ExpectsAudit(std::is_sorted(first.begin(), first.end()) and
+ std::is_sorted(second.begin(), second.end()));
+ std::set_intersection(first.begin(),
+ first.end(),
+ second.begin(),
+ second.end(),
+ std::inserter(*dups, dups->begin()));
+ if constexpr (sizeof...(rest) > 0) {
+ // n comparisons with rest: first<->rest[0], ..., first<->rest[n]
+ FindDuplicates</*kTriangular=*/false>(dups, first, rest...);
+ if constexpr (kTriangular) {
+ // do triangular compare of second with rest
+ FindDuplicates</*kTriangular=*/true>(dups, second, rest...);
+ }
+ }
+}
+
+} // namespace detail
+
+template <bool kTriangular,
+ InputIterableContainer T_Container,
+ InputIterableContainer... T_Rest,
+ OutputIterableContainer T_Result>
+[[nodiscard]] static inline auto GetDuplicates(T_Container const& first,
+ T_Rest const&... rest)
+ -> T_Result {
+ auto dups = T_Result{};
+ constexpr auto kNumContainers = 1 + sizeof...(rest);
+ if constexpr (kNumContainers > 1) {
+ std::size_t size{};
+ if constexpr (kTriangular) {
+ // worst case if all containers are of the same size
+ size = kNumContainers * detail::MaxSize(first, rest...) / 2;
+ }
+ else {
+ size = std::min(first.size(), detail::MaxSize(rest...));
+ }
+ dups.reserve(size);
+ detail::FindDuplicates<kTriangular, T_Result>(&dups, first, rest...);
+ }
+ return dups;
+}
+
+template <InputIterableStringContainer T_Container>
+[[nodiscard]] static inline auto JoinContainer(T_Container const& c,
+ std::string const& sep)
+ -> std::string {
+ std::ostringstream oss{};
+ std::size_t insert_sep{};
+ for (auto const& i : c) {
+ oss << (insert_sep++ ? sep.c_str() : "");
+ oss << i;
+ }
+ return oss.str();
+};
+
+} // namespace BuildMaps::Base
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_BASE_MAPS_USER_RULE_HPP
diff --git a/src/buildtool/build_engine/expression/TARGETS b/src/buildtool/build_engine/expression/TARGETS
new file mode 100644
index 00000000..4f719185
--- /dev/null
+++ b/src/buildtool/build_engine/expression/TARGETS
@@ -0,0 +1,46 @@
+{ "linked_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["linked_map"]
+ , "hdrs": ["linked_map.hpp"]
+ , "deps":
+ [ ["@", "fmt", "", "fmt"]
+ , ["src/utils/cpp", "hash_combine"]
+ , ["src/utils/cpp", "atomic"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "expression"]
+ }
+, "expression":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["expression"]
+ , "hdrs":
+ [ "configuration.hpp"
+ , "expression_ptr.hpp"
+ , "expression.hpp"
+ , "function_map.hpp"
+ , "evaluator.hpp"
+ , "target_result.hpp"
+ , "target_node.hpp"
+ ]
+ , "srcs":
+ [ "expression_ptr.cpp"
+ , "expression.cpp"
+ , "evaluator.cpp"
+ , "target_node.cpp"
+ ]
+ , "deps":
+ [ "linked_map"
+ , ["src/buildtool/build_engine/base_maps", "entity_name_data"]
+ , ["src/buildtool/common", "artifact_description"]
+ , ["src/buildtool/crypto", "hash_generator"]
+ , ["src/buildtool/logging", "logging"]
+ , ["src/utils/cpp", "type_safe_arithmetic"]
+ , ["src/utils/cpp", "json"]
+ , ["src/utils/cpp", "hash_combine"]
+ , ["src/utils/cpp", "hex_string"]
+ , ["src/utils/cpp", "concepts"]
+ , ["src/utils/cpp", "atomic"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "expression"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/build_engine/expression/configuration.hpp b/src/buildtool/build_engine/expression/configuration.hpp
new file mode 100644
index 00000000..5d0b9c2a
--- /dev/null
+++ b/src/buildtool/build_engine/expression/configuration.hpp
@@ -0,0 +1,154 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_CONFIGURATION_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_CONFIGURATION_HPP
+
+#include <algorithm>
+#include <sstream>
+#include <string>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/utils/cpp/concepts.hpp"
+
+// Decorator for Expression containing a map. Adds Prune() and Update().
+class Configuration {
+ public:
+ explicit Configuration(ExpressionPtr expr) noexcept
+ : expr_{std::move(expr)} {
+ gsl_ExpectsAudit(expr_->IsMap());
+ }
+ explicit Configuration(Expression::map_t const& map) noexcept
+ : expr_{ExpressionPtr{map}} {}
+
+ Configuration() noexcept = default;
+ ~Configuration() noexcept = default;
+ Configuration(Configuration const&) noexcept = default;
+ Configuration(Configuration&&) noexcept = default;
+ auto operator=(Configuration const&) noexcept -> Configuration& = default;
+ auto operator=(Configuration&&) noexcept -> Configuration& = default;
+
+ [[nodiscard]] auto operator[](std::string const& key) const
+ -> ExpressionPtr {
+ return expr_->Get(key, Expression::none_t{});
+ }
+ [[nodiscard]] auto operator[](ExpressionPtr const& key) const
+ -> ExpressionPtr {
+ return expr_->Get(key->String(), Expression::none_t{});
+ }
+ [[nodiscard]] auto ToString() const -> std::string {
+ return expr_->ToString();
+ }
+ [[nodiscard]] auto ToJson() const -> nlohmann::json {
+ return expr_->ToJson();
+ }
+ [[nodiscard]] auto Enumerate(const std::string& prefix, size_t width) const
+ -> std::string {
+ std::stringstream ss{};
+ if (width > prefix.size()) {
+ size_t actual_width = width - prefix.size();
+ for (auto const& [key, value] : expr_->Map()) {
+ std::string key_str = Expression{key}.ToString();
+ std::string val_str = value->ToString();
+ if (actual_width > key_str.size() + 3) {
+ ss << prefix << key_str << " : ";
+ size_t remain = actual_width - key_str.size() - 3;
+ if (val_str.size() >= remain) {
+ ss << val_str.substr(0, remain - 3) << "...";
+ }
+ else {
+ ss << val_str;
+ }
+ }
+ else {
+ ss << prefix << key_str.substr(0, actual_width);
+ }
+ ss << std::endl;
+ }
+ }
+ return ss.str();
+ }
+
+ [[nodiscard]] auto operator==(const Configuration& other) const -> bool {
+ return expr_ == other.expr_;
+ }
+
+ [[nodiscard]] auto hash() const noexcept -> std::size_t {
+ return std::hash<ExpressionPtr>{}(expr_);
+ }
+
+ template <InputIterableStringContainer T>
+ [[nodiscard]] auto Prune(T const& vars) const -> Configuration {
+ auto subset = Expression::map_t::underlying_map_t{};
+ std::for_each(vars.begin(), vars.end(), [&](auto const& k) {
+ auto const& map = expr_->Map();
+ auto v = map.Find(k);
+ if (v) {
+ subset.emplace(k, v->get());
+ }
+ else {
+ subset.emplace(k, Expression::kNone);
+ }
+ });
+ return Configuration{Expression::map_t{subset}};
+ }
+
+ [[nodiscard]] auto Prune(ExpressionPtr const& vars) const -> Configuration {
+ auto subset = Expression::map_t::underlying_map_t{};
+ auto const& list = vars->List();
+ std::for_each(list.begin(), list.end(), [&](auto const& k) {
+ auto const& map = expr_->Map();
+ auto const key = k->String();
+ auto v = map.Find(key);
+ if (v) {
+ subset.emplace(key, v->get());
+ }
+ else {
+ subset.emplace(key, ExpressionPtr{Expression::none_t{}});
+ }
+ });
+ return Configuration{Expression::map_t{subset}};
+ }
+
+ template <class T>
+ requires(Expression::IsValidType<T>() or std::is_same_v<T, ExpressionPtr>)
+ [[nodiscard]] auto Update(std::string const& name, T const& value) const
+ -> Configuration {
+ auto update = Expression::map_t::underlying_map_t{};
+ update.emplace(name, value);
+ return Configuration{Expression::map_t{expr_, update}};
+ }
+
+ [[nodiscard]] auto Update(
+ Expression::map_t::underlying_map_t const& map) const -> Configuration {
+ if (map.empty()) {
+ return *this;
+ }
+ return Configuration{Expression::map_t{expr_, map}};
+ }
+
+ [[nodiscard]] auto Update(ExpressionPtr const& map) const -> Configuration {
+ gsl_ExpectsAudit(map->IsMap());
+ if (map->Map().empty()) {
+ return *this;
+ }
+ return Configuration{Expression::map_t{expr_, map}};
+ }
+
+ [[nodiscard]] auto VariableFixed(std::string const& x) const -> bool {
+ return expr_->Map().Find(x).has_value();
+ }
+
+ private:
+ ExpressionPtr expr_{Expression::kEmptyMap};
+};
+
+namespace std {
+template <>
+struct hash<Configuration> {
+ [[nodiscard]] auto operator()(Configuration const& p) const noexcept
+ -> std::size_t {
+ return p.hash();
+ }
+};
+} // namespace std
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_CONFIGURATION_HPP
diff --git a/src/buildtool/build_engine/expression/evaluator.cpp b/src/buildtool/build_engine/expression/evaluator.cpp
new file mode 100644
index 00000000..b93fa9f9
--- /dev/null
+++ b/src/buildtool/build_engine/expression/evaluator.cpp
@@ -0,0 +1,936 @@
+#include "src/buildtool/build_engine/expression/evaluator.hpp"
+
+#include <algorithm>
+#include <exception>
+#include <filesystem>
+#include <sstream>
+#include <string>
+#include <unordered_set>
+
+#include "fmt/core.h"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/build_engine/expression/function_map.hpp"
+
+namespace {
+
+using namespace std::string_literals;
+using number_t = Expression::number_t;
+using list_t = Expression::list_t;
+using map_t = Expression::map_t;
+
+auto ValueIsTrue(ExpressionPtr const& val) -> bool {
+ if (val->IsNone()) {
+ return false;
+ }
+ if (val->IsBool()) {
+ return *val != false;
+ }
+ if (val->IsNumber()) {
+ return *val != number_t{0};
+ }
+ if (val->IsString()) {
+ return *val != ""s and *val != "0"s and *val != "NO"s;
+ }
+ if (val->IsList()) {
+ return not val->List().empty();
+ }
+ if (val->IsMap()) {
+ return not val->Map().empty();
+ }
+ return true;
+}
+
+auto Flatten(ExpressionPtr const& expr) -> ExpressionPtr {
+ if (not expr->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "Flatten expects list but instead got: {}.", expr->ToString())};
+ }
+ if (expr->List().empty()) {
+ return expr;
+ }
+ auto const& list = expr->List();
+ size_t size{};
+ std::for_each(list.begin(), list.end(), [&](auto const& l) {
+ if (not l->IsList()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("Non-list entry found for argument in flatten: {}.",
+ l->ToString())};
+ }
+ size += l->List().size();
+ });
+ auto result = Expression::list_t{};
+ result.reserve(size);
+ std::for_each(list.begin(), list.end(), [&](auto const& l) {
+ std::copy(
+ l->List().begin(), l->List().end(), std::back_inserter(result));
+ });
+ return ExpressionPtr{result};
+}
+
+auto All(ExpressionPtr const& list) -> ExpressionPtr {
+ for (auto const& c : list->List()) {
+ if (not ValueIsTrue(c)) {
+ return ExpressionPtr{false};
+ }
+ }
+ return ExpressionPtr{true};
+}
+
+auto Any(ExpressionPtr const& list) -> ExpressionPtr {
+ for (auto const& c : list->List()) {
+ if (ValueIsTrue(c)) {
+ return ExpressionPtr{true};
+ }
+ }
+ return ExpressionPtr{false};
+}
+
+// logical AND with short-circuit evaluation
+auto LogicalAnd(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ if (auto const list = expr->At("$1")) {
+ auto const& l = list->get();
+ if (not l->IsList()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("Non-list entry found for argument in and: {}.",
+ l->ToString())};
+ }
+ for (auto const& c : l->List()) {
+ if (not ValueIsTrue(eval(c, env))) {
+ return ExpressionPtr{false};
+ }
+ }
+ }
+ return ExpressionPtr{true};
+}
+
+// logical OR with short-circuit evaluation
+auto LogicalOr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ if (auto const list = expr->At("$1")) {
+ auto const& l = list->get();
+ if (not l->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "Non-list entry found for argument in or: {}.", l->ToString())};
+ }
+ for (auto const& c : l->List()) {
+ if (ValueIsTrue(eval(c, env))) {
+ return ExpressionPtr{true};
+ }
+ }
+ }
+ return ExpressionPtr{false};
+}
+
+auto Keys(ExpressionPtr const& d) -> ExpressionPtr {
+ auto const& m = d->Map();
+ auto result = Expression::list_t{};
+ result.reserve(m.size());
+ std::for_each(m.begin(), m.end(), [&](auto const& item) {
+ result.emplace_back(ExpressionPtr{item.first});
+ });
+ return ExpressionPtr{result};
+}
+
+auto Values(ExpressionPtr const& d) -> ExpressionPtr {
+ return ExpressionPtr{d->Map().Values()};
+}
+
+auto NubRight(ExpressionPtr const& expr) -> ExpressionPtr {
+ if (not expr->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "nub_right expects list but instead got: {}.", expr->ToString())};
+ }
+ if (expr->List().empty()) {
+ return expr;
+ }
+ auto const& list = expr->List();
+ auto reverse_result = Expression::list_t{};
+ reverse_result.reserve(list.size());
+ auto seen = std::unordered_set<ExpressionPtr>{};
+ seen.reserve(list.size());
+ std::for_each(list.rbegin(), list.rend(), [&](auto const& l) {
+ if (not seen.contains(l)) {
+ reverse_result.push_back(l);
+ seen.insert(l);
+ }
+ });
+ std::reverse(reverse_result.begin(), reverse_result.end());
+ return ExpressionPtr{reverse_result};
+}
+
+auto ChangeEndingTo(ExpressionPtr const& name, ExpressionPtr const& ending)
+ -> ExpressionPtr {
+ std::filesystem::path path{name->String()};
+ return ExpressionPtr{(path.parent_path() / path.stem()).string() +
+ ending->String()};
+}
+
+auto BaseName(ExpressionPtr const& name) -> ExpressionPtr {
+ std::filesystem::path path{name->String()};
+ return ExpressionPtr{path.filename().string()};
+}
+
+auto ShellQuote(std::string arg) -> std::string {
+ auto start_pos = size_t{};
+ std::string from{"'"};
+ std::string to{"'\\''"};
+ while ((start_pos = arg.find(from, start_pos)) != std::string::npos) {
+ arg.replace(start_pos, from.length(), to);
+ start_pos += to.length();
+ }
+ return fmt::format("'{}'", arg);
+}
+
+template <bool kDoQuote = false>
+auto Join(ExpressionPtr const& expr, std::string const& sep) -> ExpressionPtr {
+ if (expr->IsString()) {
+ auto string = expr->String();
+ if constexpr (kDoQuote) {
+ string = ShellQuote(std::move(string));
+ }
+ return ExpressionPtr{std::move(string)};
+ }
+ if (expr->IsList()) {
+ auto const& list = expr->List();
+ int insert_sep{};
+ std::stringstream ss{};
+ std::for_each(list.begin(), list.end(), [&](auto const& e) {
+ ss << (insert_sep++ ? sep : "");
+ auto string = e->String();
+ if constexpr (kDoQuote) {
+ string = ShellQuote(std::move(string));
+ }
+ ss << std::move(string);
+ });
+ return ExpressionPtr{ss.str()};
+ }
+ throw Evaluator::EvaluationError{fmt::format(
+ "Join expects string or list but got: {}.", expr->ToString())};
+}
+
+template <bool kDisjoint = false>
+auto Union(Expression::list_t const& dicts, size_t from, size_t to)
+ -> ExpressionPtr {
+ if (to <= from) {
+ return Expression::kEmptyMap;
+ }
+ if (to == from + 1) {
+ return dicts[from];
+ }
+ size_t mid = from + (to - from) / 2;
+ auto left = Union(dicts, from, mid);
+ auto right = Union(dicts, mid, to);
+ if (left->Map().empty()) {
+ return right;
+ }
+ if (right->Map().empty()) {
+ return left;
+ }
+ if constexpr (kDisjoint) {
+ auto dup = left->Map().FindConflictingDuplicate(right->Map());
+ if (dup) {
+ throw Evaluator::EvaluationError{
+ fmt::format("Map union not essentially disjoint as claimed, "
+ "duplicate key '{}'.",
+ dup->get())};
+ }
+ }
+ return ExpressionPtr{Expression::map_t{left, right}};
+}
+
+template <bool kDisjoint = false>
+auto Union(ExpressionPtr const& expr) -> ExpressionPtr {
+ if (not expr->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "Union expects list of maps but got: {}.", expr->ToString())};
+ }
+ auto const& list = expr->List();
+ if (list.empty()) {
+ return Expression::kEmptyMap;
+ }
+ return Union<kDisjoint>(list, 0, list.size());
+}
+
+auto ConcatTargetName(ExpressionPtr const& expr, ExpressionPtr const& append)
+ -> ExpressionPtr {
+ if (expr->IsString()) {
+ return ExpressionPtr{expr->String() + append->String()};
+ }
+ if (expr->IsList()) {
+ auto list = Expression::list_t{};
+ auto not_last = expr->List().size();
+ bool all_string = true;
+ std::for_each(
+ expr->List().begin(), expr->List().end(), [&](auto const& e) {
+ all_string = all_string and e->IsString();
+ if (all_string) {
+ list.emplace_back(ExpressionPtr{
+ e->String() + (--not_last ? "" : append->String())});
+ }
+ });
+ if (all_string) {
+ return ExpressionPtr{list};
+ }
+ }
+ throw Evaluator::EvaluationError{fmt::format(
+ "Unsupported expression for concat: {}.", expr->ToString())};
+}
+
+auto EvalArgument(ExpressionPtr const& expr,
+ std::string const& argument,
+ const SubExprEvaluator& eval,
+ Configuration const& env) -> ExpressionPtr {
+ try {
+ return eval(expr[argument], env);
+ } catch (Evaluator::EvaluationError const& ex) {
+ throw Evaluator::EvaluationError::WhileEval(
+ fmt::format("Evaluating argument {}:", argument), ex);
+ } catch (std::exception const& ex) {
+ throw Evaluator::EvaluationError::WhileEvaluating(
+ fmt::format("Evaluating argument {}:", argument), ex);
+ }
+}
+
+auto UnaryExpr(std::function<ExpressionPtr(ExpressionPtr const&)> const& f)
+ -> std::function<ExpressionPtr(SubExprEvaluator&&,
+ ExpressionPtr const&,
+ Configuration const&)> {
+ return [f](auto&& eval, auto const& expr, auto const& env) {
+ auto argument = EvalArgument(expr, "$1", eval, env);
+ try {
+ return f(argument);
+ } catch (Evaluator::EvaluationError const& ex) {
+ throw Evaluator::EvaluationError::WhileEval(
+ fmt::format("Having evaluted the argument to {}:",
+ argument->ToString()),
+ ex);
+ } catch (std::exception const& ex) {
+ throw Evaluator::EvaluationError::WhileEvaluating(
+ fmt::format("Having evaluted the argument to {}:",
+ argument->ToString()),
+ ex);
+ }
+ };
+}
+
+auto AndExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ if (auto const conds = expr->At("$1")) {
+ return conds->get()->IsList()
+ ? LogicalAnd(std::move(eval), expr, env)
+ : UnaryExpr(All)(std::move(eval), expr, env);
+ }
+ return ExpressionPtr{true};
+}
+
+auto OrExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ if (auto const conds = expr->At("$1")) {
+ return conds->get()->IsList()
+ ? LogicalOr(std::move(eval), expr, env)
+ : UnaryExpr(Any)(std::move(eval), expr, env);
+ }
+ return ExpressionPtr{false};
+}
+
+auto VarExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto result = env[expr["name"]];
+ if (result->IsNone()) {
+ return eval(expr->Get("default", Expression::none_t{}), env);
+ }
+ return result;
+}
+
+auto IfExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ if (ValueIsTrue(EvalArgument(expr, "cond", eval, env))) {
+ return EvalArgument(expr, "then", eval, env);
+ }
+ return eval(expr->Get("else", list_t{}), env);
+}
+
+auto CondExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto const& cond = expr->At("cond");
+ if (cond) {
+ if (not cond->get()->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "cond in cond has to be a list of pairs, but found {}",
+ cond->get()->ToString())};
+ }
+ for (const auto& pair : cond->get()->List()) {
+ if (not pair->IsList() or pair->List().size() != 2) {
+ throw Evaluator::EvaluationError{
+ fmt::format("cond in cond has to be a list of pairs, "
+ "but found entry {}",
+ pair->ToString())};
+ }
+ if (ValueIsTrue(eval(pair->List()[0], env))) {
+ return eval(pair->List()[1], env);
+ }
+ }
+ }
+ return eval(expr->Get("default", list_t{}), env);
+}
+
+auto CaseExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto const& cases = expr->At("case");
+ if (cases) {
+ if (not cases->get()->IsMap()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "case in case has to be a map of expressions, but found {}",
+ cases->get()->ToString())};
+ }
+ auto const& e = expr->At("expr");
+ if (not e) {
+ throw Evaluator::EvaluationError{"missing expr in case"};
+ }
+ auto const& key = eval(e->get(), env);
+ if (not key->IsString()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "expr in case must evaluate to string, but found {}",
+ key->ToString())};
+ }
+ if (auto const& val = cases->get()->At(key->String())) {
+ return eval(val->get(), env);
+ }
+ }
+ return eval(expr->Get("default", list_t{}), env);
+}
+
+auto SeqCaseExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto const& cases = expr->At("case");
+ if (cases) {
+ if (not cases->get()->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "case in case* has to be a list of pairs, but found {}",
+ cases->get()->ToString())};
+ }
+ auto const& e = expr->At("expr");
+ if (not e) {
+ throw Evaluator::EvaluationError{"missing expr in case"};
+ }
+ auto const& cmp = eval(e->get(), env);
+ for (const auto& pair : cases->get()->List()) {
+ if (not pair->IsList() or pair->List().size() != 2) {
+ throw Evaluator::EvaluationError{
+ fmt::format("case in case* has to be a list of pairs, "
+ "but found entry {}",
+ pair->ToString())};
+ }
+ if (cmp == eval(pair->List()[0], env)) {
+ return eval(pair->List()[1], env);
+ }
+ }
+ }
+ return eval(expr->Get("default", list_t{}), env);
+}
+
+auto EqualExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ return ExpressionPtr{EvalArgument(expr, "$1", eval, env) ==
+ EvalArgument(expr, "$2", eval, env)};
+}
+
+auto AddExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ return eval(expr["$1"], env) + eval(expr["$2"], env);
+}
+
+auto ChangeEndingExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto name = eval(expr->Get("$1", ""s), env);
+ auto ending = eval(expr->Get("ending", ""s), env);
+ return ChangeEndingTo(name, ending);
+}
+
+auto JoinExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto list = eval(expr->Get("$1", list_t{}), env);
+ auto separator = eval(expr->Get("separator", ""s), env);
+ return Join(list, separator->String());
+}
+
+auto JoinCmdExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto const& list = eval(expr->Get("$1", list_t{}), env);
+ return Join</*kDoQuote=*/true>(list, " ");
+}
+
+auto JsonEncodeExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto const& value = eval(expr->Get("$1", list_t{}), env);
+ return ExpressionPtr{
+ value->ToJson(Expression::JsonMode::NullForNonJson).dump()};
+}
+
+auto EscapeCharsExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto string = eval(expr->Get("$1", ""s), env);
+ auto chars = eval(expr->Get("chars", ""s), env);
+ auto escape_prefix = eval(expr->Get("escape_prefix", "\\"s), env);
+ std::stringstream ss{};
+ std::for_each(
+ string->String().begin(), string->String().end(), [&](auto const& c) {
+ auto do_escape = chars->String().find(c) != std::string::npos;
+ ss << (do_escape ? escape_prefix->String() : "") << c;
+ });
+ return ExpressionPtr{ss.str()};
+}
+
+auto LookupExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto k = eval(expr["key"], env);
+ auto d = eval(expr["map"], env);
+ if (not k->IsString()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "Key expected to be string but found {}.", k->ToString())};
+ }
+ if (not d->IsMap()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "Map expected to be mapping but found {}.", d->ToString())};
+ }
+ auto lookup = Expression::kNone;
+ if (d->Map().contains(k->String())) {
+ lookup = d->Map().at(k->String());
+ }
+ if (lookup->IsNone()) {
+ lookup = eval(expr->Get("default", Expression::none_t()), env);
+ }
+ return lookup;
+}
+
+auto EmptyMapExpr(SubExprEvaluator&& /*eval*/,
+ ExpressionPtr const& /*expr*/,
+ Configuration const &
+ /*env*/) -> ExpressionPtr {
+ return Expression::kEmptyMap;
+}
+
+auto SingletonMapExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto key = EvalArgument(expr, "key", eval, env);
+ auto value = EvalArgument(expr, "value", eval, env);
+ return ExpressionPtr{Expression::map_t{key->String(), value}};
+}
+
+auto ToSubdirExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto d = eval(expr["$1"], env);
+ auto s = eval(expr->Get("subdir", "."s), env);
+ auto flat = ValueIsTrue(eval(expr->Get("flat", false), env));
+ std::filesystem::path subdir{s->String()};
+ auto result = Expression::map_t::underlying_map_t{};
+ if (flat) {
+ for (auto const& el : d->Map()) {
+ std::filesystem::path k{el.first};
+ auto new_path = subdir / k.filename();
+ if (result.contains(new_path) && !(result[new_path] == el.second)) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "Flat staging of {} to subdir {} conflicts on path {}",
+ d->ToString(),
+ subdir.string(),
+ new_path.string())};
+ }
+ result[new_path] = el.second;
+ }
+ }
+ else {
+ for (auto const& el : d->Map()) {
+ result[(subdir / el.first).string()] = el.second;
+ }
+ }
+ return ExpressionPtr{Expression::map_t{result}};
+}
+
+auto ForeachExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto range_list = eval(expr->Get("range", list_t{}), env);
+ if (range_list->List().empty()) {
+ return Expression::kEmptyList;
+ }
+ auto const& var = expr->Get("var", "_"s);
+ auto const& body = expr->Get("body", list_t{});
+ auto result = Expression::list_t{};
+ result.reserve(range_list->List().size());
+ std::transform(range_list->List().begin(),
+ range_list->List().end(),
+ std::back_inserter(result),
+ [&](auto const& x) {
+ return eval(body, env.Update(var->String(), x));
+ });
+ return ExpressionPtr{result};
+}
+
+auto ForeachMapExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto range_map = eval(expr->Get("range", Expression::kEmptyMapExpr), env);
+ if (range_map->Map().empty()) {
+ return Expression::kEmptyList;
+ }
+ auto const& var = expr->Get("var_key", "_"s);
+ auto const& var_val = expr->Get("var_val", "$_"s);
+ auto const& body = expr->Get("body", list_t{});
+ auto result = Expression::list_t{};
+ result.reserve(range_map->Map().size());
+ std::transform(range_map->Map().begin(),
+ range_map->Map().end(),
+ std::back_inserter(result),
+ [&](auto const& it) {
+ return eval(body,
+ env.Update(var->String(), it.first)
+ .Update(var_val->String(), it.second));
+ });
+ return ExpressionPtr{result};
+}
+
+auto FoldLeftExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto const& var = expr->Get("var", "_"s);
+ auto const& accum_var = expr->Get("accum_var", "$1"s);
+ auto range_list = eval(expr["range"], env);
+ auto val = eval(expr->Get("start", list_t{}), env);
+ auto const& body = expr->Get("body", list_t{});
+ for (auto const& x : range_list->List()) {
+ val = eval(
+ body, env.Update({{var->String(), x}, {accum_var->String(), val}}));
+ }
+ return val;
+}
+
+auto LetExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto const& bindings = expr->At("bindings");
+ auto new_env = env;
+ if (bindings) {
+ if (not bindings->get()->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "bindings in let* has to be a list of pairs, but found {}",
+ bindings->get()->ToString())};
+ }
+ int pos = -1;
+ for (const auto& binding : bindings->get()->List()) {
+ ++pos;
+ if (not binding->IsList() or binding->List().size() != 2) {
+ throw Evaluator::EvaluationError{
+ fmt::format("bindings in let* has to be a list of pairs, "
+ "but found entry {}",
+ binding->ToString())};
+ }
+ auto const& x_exp = binding[0];
+ if (not x_exp->IsString()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("variable names in let* have to be strings, "
+ "but found binding entry {}",
+ binding->ToString())};
+ }
+ ExpressionPtr val;
+ try {
+ val = eval(binding[1], new_env);
+ } catch (Evaluator::EvaluationError const& ex) {
+ throw Evaluator::EvaluationError::WhileEval(
+ fmt::format("Evaluating entry {} in bindings, binding {}:",
+ pos,
+ x_exp->ToString()),
+ ex);
+ } catch (std::exception const& ex) {
+ throw Evaluator::EvaluationError::WhileEvaluating(
+ fmt::format("Evaluating entry {} in bindings, binding {}:",
+ pos,
+ x_exp->ToString()),
+ ex);
+ }
+ new_env = new_env.Update(x_exp->String(), val);
+ }
+ }
+ auto const& body = expr->Get("body", map_t{});
+ try {
+ return eval(body, new_env);
+ } catch (Evaluator::EvaluationError const& ex) {
+ throw Evaluator::EvaluationError::WhileEval("Evaluating the body:", ex);
+ } catch (std::exception const& ex) {
+ throw Evaluator::EvaluationError::WhileEvaluating(
+ "Evaluating the body:", ex);
+ }
+}
+
+auto ConcatTargetNameExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto p1 = eval(expr->Get("$1", ""s), env);
+ auto p2 = eval(expr->Get("$2", ""s), env);
+ return ConcatTargetName(p1, Join(p2, ""));
+}
+
+auto ContextExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ try {
+ return eval(expr->Get("$1", Expression::kNone), env);
+ } catch (std::exception const& ex) {
+ auto msg_expr = expr->Get("msg", map_t{});
+ std::string context{};
+ try {
+ auto msg_val = eval(msg_expr, env);
+ context = msg_val->ToString();
+ } catch (std::exception const&) {
+ context = "[non evaluating term] " + msg_expr->ToString();
+ }
+ std::stringstream ss{};
+ ss << "In Context " << context << std::endl;
+ ss << ex.what();
+ throw Evaluator::EvaluationError(ss.str(), true, true);
+ }
+}
+
+auto DisjointUnionExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto argument = EvalArgument(expr, "$1", eval, env);
+ try {
+ return Union</*kDisjoint=*/true>(argument);
+ } catch (std::exception const& ex) {
+ auto msg_expr = expr->Map().Find("msg");
+ if (not msg_expr) {
+ throw Evaluator::EvaluationError::WhileEvaluating(
+ fmt::format("Having evaluted the argument to {}:",
+ argument->ToString()),
+ ex);
+ }
+ std::string msg;
+ try {
+ auto msg_val = eval(msg_expr->get(), env);
+ msg = msg_val->ToString();
+ } catch (std::exception const&) {
+ msg = "[non evaluating term] " + msg_expr->get()->ToString();
+ }
+ std::stringstream ss{};
+ ss << msg << std::endl;
+ ss << "Reason: " << ex.what() << std::endl;
+ ss << "The argument of the union was " << argument->ToString();
+ throw Evaluator::EvaluationError(ss.str(), false, true);
+ }
+}
+
+auto FailExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto msg = eval(expr->Get("msg", Expression::kNone), env);
+ throw Evaluator::EvaluationError(
+ msg->ToString(), false, /* user error*/ true);
+}
+
+auto AssertNonEmptyExpr(SubExprEvaluator&& eval,
+ ExpressionPtr const& expr,
+ Configuration const& env) -> ExpressionPtr {
+ auto val = eval(expr["$1"], env);
+ if ((val->IsString() and (not val->String().empty())) or
+ (val->IsList() and (not val->List().empty())) or
+ (val->IsMap() and (not val->Map().empty()))) {
+ return val;
+ }
+ auto msg_expr = expr->Get("msg", Expression::kNone);
+ std::string msg;
+ try {
+ auto msg_val = eval(msg_expr, env);
+ msg = msg_val->ToString();
+ } catch (std::exception const&) {
+ msg = "[non evaluating term] " + msg_expr->ToString();
+ }
+ std::stringstream ss{};
+ ss << msg << std::endl;
+ ss << "Expected non-empty value but found: " << val->ToString();
+ throw Evaluator::EvaluationError(ss.str(), false, true);
+}
+
+auto built_in_functions =
+ FunctionMap::MakePtr({{"var", VarExpr},
+ {"if", IfExpr},
+ {"cond", CondExpr},
+ {"case", CaseExpr},
+ {"case*", SeqCaseExpr},
+ {"fail", FailExpr},
+ {"assert_non_empty", AssertNonEmptyExpr},
+ {"context", ContextExpr},
+ {"==", EqualExpr},
+ {"and", AndExpr},
+ {"or", OrExpr},
+ {"+", AddExpr},
+ {"++", UnaryExpr(Flatten)},
+ {"nub_right", UnaryExpr(NubRight)},
+ {"change_ending", ChangeEndingExpr},
+ {"basename", UnaryExpr(BaseName)},
+ {"join", JoinExpr},
+ {"join_cmd", JoinCmdExpr},
+ {"json_encode", JsonEncodeExpr},
+ {"escape_chars", EscapeCharsExpr},
+ {"keys", UnaryExpr(Keys)},
+ {"values", UnaryExpr(Values)},
+ {"lookup", LookupExpr},
+ {"empty_map", EmptyMapExpr},
+ {"singleton_map", SingletonMapExpr},
+ {"disjoint_map_union", DisjointUnionExpr},
+ {"map_union", UnaryExpr([](auto const& exp) {
+ return Union</*kDisjoint=*/false>(exp);
+ })},
+ {"to_subdir", ToSubdirExpr},
+ {"foreach", ForeachExpr},
+ {"foreach_map", ForeachMapExpr},
+ {"foldl", FoldLeftExpr},
+ {"let*", LetExpr},
+ {"concat_target_name", ConcatTargetNameExpr}});
+
+} // namespace
+
+auto Evaluator::EvaluationError::WhileEvaluating(ExpressionPtr const& expr,
+ Configuration const& env,
+ std::exception const& ex)
+ -> Evaluator::EvaluationError {
+ std::stringstream ss{};
+ ss << "* ";
+ if (expr->IsMap() and expr->Map().contains("type") and
+ expr["type"]->IsString()) {
+ ss << expr["type"]->ToString() << "-expression ";
+ }
+ ss << expr->ToString() << std::endl;
+ ss << " environment " << std::endl;
+ ss << env.Enumerate(" - ", kLineWidth) << std::endl;
+ ss << ex.what();
+ return EvaluationError{ss.str(), true /* while_eval */};
+}
+
+auto Evaluator::EvaluationError::WhileEval(ExpressionPtr const& expr,
+ Configuration const& env,
+ Evaluator::EvaluationError const& ex)
+ -> Evaluator::EvaluationError {
+ if (ex.UserContext()) {
+ return ex;
+ }
+ return Evaluator::EvaluationError::WhileEvaluating(expr, env, ex);
+}
+
+auto Evaluator::EvaluationError::WhileEvaluating(const std::string& where,
+ std::exception const& ex)
+ -> Evaluator::EvaluationError {
+ std::stringstream ss{};
+ ss << where << std::endl;
+ ss << ex.what();
+ return EvaluationError{ss.str(), true /* while_eval */};
+}
+
+auto Evaluator::EvaluationError::WhileEval(const std::string& where,
+ Evaluator::EvaluationError const& ex)
+ -> Evaluator::EvaluationError {
+ if (ex.UserContext()) {
+ return ex;
+ }
+ return Evaluator::EvaluationError::WhileEvaluating(where, ex);
+}
+
+auto Evaluator::EvaluateExpression(
+ ExpressionPtr const& expr,
+ Configuration const& env,
+ FunctionMapPtr const& provider_functions,
+ std::function<void(std::string const&)> const& logger,
+ std::function<void(void)> const& note_user_context) noexcept
+ -> ExpressionPtr {
+ std::stringstream ss{};
+ try {
+ return Evaluate(
+ expr,
+ env,
+ FunctionMap::MakePtr(built_in_functions, provider_functions));
+ } catch (EvaluationError const& ex) {
+ if (ex.UserContext()) {
+ try {
+ note_user_context();
+ } catch (...) {
+ // should not throw
+ }
+ }
+ else {
+ if (ex.WhileEvaluation()) {
+ ss << "Expression evaluation traceback (most recent call last):"
+ << std::endl;
+ }
+ }
+ ss << ex.what();
+ } catch (std::exception const& ex) {
+ ss << ex.what();
+ }
+ try {
+ logger(ss.str());
+ } catch (...) {
+ // should not throw
+ }
+ return ExpressionPtr{nullptr};
+}
+
+auto Evaluator::Evaluate(ExpressionPtr const& expr,
+ Configuration const& env,
+ FunctionMapPtr const& functions) -> ExpressionPtr {
+ try {
+ if (expr->IsList()) {
+ if (expr->List().empty()) {
+ return expr;
+ }
+ auto list = Expression::list_t{};
+ std::transform(
+ expr->List().cbegin(),
+ expr->List().cend(),
+ std::back_inserter(list),
+ [&](auto const& e) { return Evaluate(e, env, functions); });
+ return ExpressionPtr{list};
+ }
+ if (not expr->IsMap()) {
+ return expr;
+ }
+ if (not expr->Map().contains("type")) {
+ throw EvaluationError{fmt::format(
+ "Object without keyword 'type': {}", expr->ToString())};
+ }
+ auto const& type = expr["type"]->String();
+ auto func = functions->Find(type);
+ if (func) {
+ return func->get()(
+ [&functions](auto const& subexpr, auto const& subenv) {
+ return Evaluator::Evaluate(subexpr, subenv, functions);
+ },
+ expr,
+ env);
+ }
+ throw EvaluationError{
+ fmt::format("Unknown syntactical construct {}", type)};
+ } catch (EvaluationError const& ex) {
+ throw EvaluationError::WhileEval(expr, env, ex);
+ } catch (std::exception const& ex) {
+ throw EvaluationError::WhileEvaluating(expr, env, ex);
+ }
+}
diff --git a/src/buildtool/build_engine/expression/evaluator.hpp b/src/buildtool/build_engine/expression/evaluator.hpp
new file mode 100644
index 00000000..b4cd5979
--- /dev/null
+++ b/src/buildtool/build_engine/expression/evaluator.hpp
@@ -0,0 +1,76 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EVALUATOR_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EVALUATOR_HPP
+
+#include <exception>
+#include <string>
+
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/build_engine/expression/function_map.hpp"
+
+class Configuration;
+
+class Evaluator {
+ public:
+ class EvaluationError : public std::exception {
+ public:
+ explicit EvaluationError(std::string const& msg,
+ bool while_eval = false,
+ bool user_context = false) noexcept
+ : msg_{(while_eval ? ""
+ : (user_context ? "UserError: "
+ : "EvaluationError: ")) +
+ msg},
+ while_eval_{while_eval},
+ user_context_{user_context} {}
+ [[nodiscard]] auto what() const noexcept -> char const* final {
+ return msg_.c_str();
+ }
+
+ [[nodiscard]] auto WhileEvaluation() const -> bool {
+ return while_eval_;
+ }
+
+ [[nodiscard]] auto UserContext() const -> bool { return user_context_; }
+
+ [[nodiscard]] static auto WhileEvaluating(ExpressionPtr const& expr,
+ Configuration const& env,
+ std::exception const& ex)
+ -> EvaluationError;
+
+ [[nodiscard]] static auto WhileEval(ExpressionPtr const& expr,
+ Configuration const& env,
+ EvaluationError const& ex)
+ -> EvaluationError;
+
+ [[nodiscard]] static auto WhileEvaluating(const std::string& where,
+ std::exception const& ex)
+ -> Evaluator::EvaluationError;
+
+ [[nodiscard]] static auto WhileEval(const std::string& where,
+ EvaluationError const& ex)
+ -> Evaluator::EvaluationError;
+
+ private:
+ std::string msg_;
+ bool while_eval_;
+ bool user_context_;
+ };
+
+ // Exception-free evaluation of expression
+ [[nodiscard]] static auto EvaluateExpression(
+ ExpressionPtr const& expr,
+ Configuration const& env,
+ FunctionMapPtr const& provider_functions,
+ std::function<void(std::string const&)> const& logger,
+ std::function<void(void)> const& note_user_context = []() {}) noexcept
+ -> ExpressionPtr;
+
+ private:
+ constexpr static std::size_t kLineWidth = 80;
+ [[nodiscard]] static auto Evaluate(ExpressionPtr const& expr,
+ Configuration const& env,
+ FunctionMapPtr const& functions)
+ -> ExpressionPtr;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EVALUATOR_HPP
diff --git a/src/buildtool/build_engine/expression/expression.cpp b/src/buildtool/build_engine/expression/expression.cpp
new file mode 100644
index 00000000..5a161468
--- /dev/null
+++ b/src/buildtool/build_engine/expression/expression.cpp
@@ -0,0 +1,249 @@
+#include "src/buildtool/build_engine/expression/expression.hpp"
+
+#include <exception>
+#include <optional>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+#include "fmt/core.h"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/expression/evaluator.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/utils/cpp/json.hpp"
+
+auto Expression::operator[](
+ std::string const& key) const& -> ExpressionPtr const& {
+ auto value = Map().Find(key);
+ if (value) {
+ return value->get();
+ }
+ throw ExpressionTypeError{
+ fmt::format("Map does not contain key '{}'.", key)};
+}
+
+auto Expression::operator[](std::string const& key) && -> ExpressionPtr {
+ auto value = std::move(*this).Map().Find(key);
+ if (value) {
+ return std::move(*value);
+ }
+ throw ExpressionTypeError{
+ fmt::format("Map does not contain key '{}'.", key)};
+}
+
+auto Expression::operator[](
+ ExpressionPtr const& key) const& -> ExpressionPtr const& {
+ return (*this)[key->String()];
+}
+
+auto Expression::operator[](ExpressionPtr const& key) && -> ExpressionPtr {
+ return std::move(*this)[key->String()];
+}
+
+auto Expression::operator[](size_t pos) const& -> ExpressionPtr const& {
+ if (pos < List().size()) {
+ return List().at(pos);
+ }
+ throw ExpressionTypeError{
+ fmt::format("List pos '{}' is out of bounds.", pos)};
+}
+
+auto Expression::operator[](size_t pos) && -> ExpressionPtr {
+ auto&& list = std::move(*this).List();
+ if (pos < list.size()) {
+ return list.at(pos);
+ }
+ throw ExpressionTypeError{
+ fmt::format("List pos '{}' is out of bounds.", pos)};
+}
+
+auto Expression::ToJson(Expression::JsonMode mode) const -> nlohmann::json {
+ if (IsBool()) {
+ return Bool();
+ }
+ if (IsNumber()) {
+ return Number();
+ }
+ if (IsString()) {
+ return String();
+ }
+ if (IsArtifact() and mode != JsonMode::NullForNonJson) {
+ return Artifact().ToJson();
+ }
+ if (IsResult() and mode != JsonMode::NullForNonJson) {
+ auto const& result = Result();
+ return Expression{map_t{{{"artifact_stage", result.artifact_stage},
+ {"runfiles", result.runfiles},
+ {"provides", result.provides}}}}
+ .ToJson(JsonMode::SerializeAllButNodes);
+ }
+ if (IsNode() and mode != JsonMode::NullForNonJson) {
+ switch (mode) {
+ case JsonMode::SerializeAll:
+ return Node().ToJson();
+ case JsonMode::SerializeAllButNodes:
+ return {{"type", "NODE"}, {"id", ToIdentifier()}};
+ default:
+ break;
+ }
+ }
+ if (IsList()) {
+ auto json = nlohmann::json::array();
+ auto const& list = List();
+ std::transform(list.begin(),
+ list.end(),
+ std::back_inserter(json),
+ [mode](auto const& e) { return e->ToJson(mode); });
+ return json;
+ }
+ if (IsMap()) {
+ auto json = nlohmann::json::object();
+ auto const& map = Value<map_t>()->get();
+ std::for_each(map.begin(), map.end(), [&](auto const& p) {
+ json.emplace(p.first, p.second->ToJson(mode));
+ });
+ return json;
+ }
+ if (IsName() and mode != JsonMode::NullForNonJson) {
+ return Name().ToJson();
+ }
+ return nlohmann::json{};
+}
+
+auto Expression::IsCacheable() const -> bool {
+ // Must be updated whenever we add a new non-cacheable value
+ if (IsName()) {
+ return false;
+ }
+ if (IsResult()) {
+ return Result().is_cacheable;
+ }
+ if (IsNode()) {
+ return Node().IsCacheable();
+ }
+ if (IsList()) {
+ for (auto const& entry : List()) {
+ if (not entry->IsCacheable()) {
+ return false;
+ }
+ }
+ }
+ if (IsMap()) {
+ for (auto const& [key, entry] : Map()) {
+ if (not entry->IsCacheable()) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+auto Expression::ToString() const -> std::string {
+ return ToJson().dump();
+}
+
+auto Expression::ToHash() const noexcept -> std::string {
+ if (hash_.load() == nullptr) {
+ if (not hash_loading_.exchange(true)) {
+ hash_ = std::make_shared<std::string>(ComputeHash());
+ hash_.notify_all();
+ }
+ else {
+ hash_.wait(nullptr);
+ }
+ }
+ return *hash_.load();
+}
+
+auto Expression::FromJson(nlohmann::json const& json) noexcept
+ -> ExpressionPtr {
+ if (json.is_null()) {
+ return ExpressionPtr{none_t{}};
+ }
+ try { // try-catch because json.get<>() could throw, although checked
+ if (json.is_boolean()) {
+ return ExpressionPtr{json.get<bool>()};
+ }
+ if (json.is_number()) {
+ return ExpressionPtr{json.get<number_t>()};
+ }
+ if (json.is_string()) {
+ return ExpressionPtr{std::string{json.get<std::string>()}};
+ }
+ if (json.is_array()) {
+ auto l = Expression::list_t{};
+ l.reserve(json.size());
+ std::transform(json.begin(),
+ json.end(),
+ std::back_inserter(l),
+ [](auto const& j) { return FromJson(j); });
+ return ExpressionPtr{l};
+ }
+ if (json.is_object()) {
+ auto m = Expression::map_t::underlying_map_t{};
+ for (auto& el : json.items()) {
+ m.emplace(el.key(), FromJson(el.value()));
+ }
+ return ExpressionPtr{Expression::map_t{m}};
+ }
+ } catch (...) {
+ gsl_EnsuresAudit(false); // ensure that the try-block never throws
+ }
+ return ExpressionPtr{nullptr};
+}
+
+template <size_t kIndex>
+auto Expression::TypeStringForIndex() const noexcept -> std::string {
+ using var_t = decltype(data_);
+ if (kIndex == data_.index()) {
+ return TypeToString<std::variant_alternative_t<kIndex, var_t>>();
+ }
+ constexpr auto size = std::variant_size_v<var_t>;
+ if constexpr (kIndex < size - 1) {
+ return TypeStringForIndex<kIndex + 1>();
+ }
+ return TypeToString<std::variant_alternative_t<size - 1, var_t>>();
+}
+
+auto Expression::TypeString() const noexcept -> std::string {
+ return TypeStringForIndex();
+}
+
+auto Expression::ComputeHash() const noexcept -> std::string {
+ auto hash = std::string{};
+ if (IsNone() or IsBool() or IsNumber() or IsString() or IsArtifact() or
+ IsResult() or IsNode() or IsName()) {
+ // just hash the JSON representation, but prepend "@" for artifact,
+ // "=" for result, "#" for node, and "$" for name.
+ std::string prefix{
+ IsArtifact()
+ ? "@"
+ : IsResult() ? "=" : IsNode() ? "#" : IsName() ? "$" : ""};
+ hash = hash_gen_.Run(prefix + ToString()).Bytes();
+ }
+ else {
+ auto hasher = hash_gen_.IncrementalHasher();
+ if (IsList()) {
+ auto list = Value<Expression::list_t>();
+ hasher.Update("[");
+ for (auto const& el : list->get()) {
+ hasher.Update(el->ToHash());
+ }
+ }
+ else if (IsMap()) {
+ auto map = Value<Expression::map_t>();
+ hasher.Update("{");
+ for (auto const& el : map->get()) {
+ hasher.Update(hash_gen_.Run(el.first).Bytes());
+ hasher.Update(el.second->ToHash());
+ }
+ }
+ auto digest = std::move(hasher).Finalize();
+ if (not digest) {
+ Logger::Log(LogLevel::Error, "Failed to finalize hash.");
+ std::terminate();
+ }
+ hash = digest->Bytes();
+ }
+ return hash;
+}
diff --git a/src/buildtool/build_engine/expression/expression.hpp b/src/buildtool/build_engine/expression/expression.hpp
new file mode 100644
index 00000000..ffcd01c8
--- /dev/null
+++ b/src/buildtool/build_engine/expression/expression.hpp
@@ -0,0 +1,380 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_HPP
+
+#include <exception>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <type_traits>
+#include <variant>
+#include <vector>
+
+#include "fmt/core.h"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name_data.hpp"
+#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
+#include "src/buildtool/build_engine/expression/function_map.hpp"
+#include "src/buildtool/build_engine/expression/linked_map.hpp"
+#include "src/buildtool/build_engine/expression/target_node.hpp"
+#include "src/buildtool/build_engine/expression/target_result.hpp"
+#include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/crypto/hash_generator.hpp"
+#include "src/utils/cpp/atomic.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+#include "src/utils/cpp/json.hpp"
+
+class Expression {
+ friend auto operator+(Expression const& /*lhs*/, Expression const & /*rhs*/)
+ -> Expression;
+
+ public:
+ using none_t = std::monostate;
+ using number_t = double;
+ using artifact_t = ArtifactDescription;
+ using result_t = TargetResult;
+ using node_t = TargetNode;
+ using list_t = std::vector<ExpressionPtr>;
+ using map_t = LinkedMap<std::string, ExpressionPtr, ExpressionPtr>;
+ using name_t = BuildMaps::Base::EntityName;
+
+ template <class T, size_t kIndex = 0>
+ static consteval auto IsValidType() -> bool {
+ if constexpr (kIndex < std::variant_size_v<decltype(data_)>) {
+ return std::is_same_v<
+ T,
+ std::variant_alternative_t<kIndex, decltype(data_)>> or
+ IsValidType<T, kIndex + 1>();
+ }
+ return false;
+ }
+
+ class ExpressionTypeError : public std::exception {
+ public:
+ explicit ExpressionTypeError(std::string const& msg) noexcept
+ : msg_{"ExpressionTypeError: " + msg} {}
+ [[nodiscard]] auto what() const noexcept -> char const* final {
+ return msg_.c_str();
+ }
+
+ private:
+ std::string msg_;
+ };
+
+ Expression() noexcept = default;
+ ~Expression() noexcept = default;
+ Expression(Expression const& other) noexcept
+ : data_{other.data_}, hash_{other.hash_.load()} {}
+ Expression(Expression&& other) noexcept
+ : data_{std::move(other.data_)}, hash_{other.hash_.load()} {}
+ auto operator=(Expression const& other) noexcept -> Expression& {
+ if (this != &other) {
+ data_ = other.data_;
+ }
+ hash_ = other.hash_.load();
+ return *this;
+ }
+ auto operator=(Expression&& other) noexcept -> Expression& {
+ data_ = std::move(other.data_);
+ hash_ = other.hash_.load();
+ return *this;
+ }
+
+ template <class T>
+ requires(IsValidType<std::remove_cvref_t<T>>())
+ // NOLINTNEXTLINE(bugprone-forwarding-reference-overload)
+ explicit Expression(T&& data) noexcept
+ : data_{std::forward<T>(data)} {}
+
+ [[nodiscard]] auto IsNone() const noexcept -> bool { return IsA<none_t>(); }
+ [[nodiscard]] auto IsBool() const noexcept -> bool { return IsA<bool>(); }
+ [[nodiscard]] auto IsNumber() const noexcept -> bool {
+ return IsA<number_t>();
+ }
+ [[nodiscard]] auto IsString() const noexcept -> bool {
+ return IsA<std::string>();
+ }
+ [[nodiscard]] auto IsName() const noexcept -> bool { return IsA<name_t>(); }
+ [[nodiscard]] auto IsArtifact() const noexcept -> bool {
+ return IsA<artifact_t>();
+ }
+ [[nodiscard]] auto IsResult() const noexcept -> bool {
+ return IsA<result_t>();
+ }
+ [[nodiscard]] auto IsNode() const noexcept -> bool { return IsA<node_t>(); }
+ [[nodiscard]] auto IsList() const noexcept -> bool { return IsA<list_t>(); }
+ [[nodiscard]] auto IsMap() const noexcept -> bool { return IsA<map_t>(); }
+
+ [[nodiscard]] auto Bool() const -> bool { return Cast<bool>(); }
+ [[nodiscard]] auto Number() const -> number_t { return Cast<number_t>(); }
+ [[nodiscard]] auto Name() const -> name_t { return Cast<name_t>(); }
+ [[nodiscard]] auto String() const& -> std::string const& {
+ return Cast<std::string>();
+ }
+ [[nodiscard]] auto String() && -> std::string {
+ return std::move(*this).Cast<std::string>();
+ }
+ [[nodiscard]] auto Artifact() const& -> artifact_t const& {
+ return Cast<artifact_t>();
+ }
+ [[nodiscard]] auto Artifact() && -> artifact_t {
+ return std::move(*this).Cast<artifact_t>();
+ }
+ [[nodiscard]] auto Result() const& -> result_t const& {
+ return Cast<result_t>();
+ }
+ [[nodiscard]] auto Result() && -> result_t {
+ return std::move(*this).Cast<result_t>();
+ }
+ [[nodiscard]] auto Node() const& -> node_t const& { return Cast<node_t>(); }
+ [[nodiscard]] auto Node() && -> node_t {
+ return std::move(*this).Cast<node_t>();
+ }
+ [[nodiscard]] auto List() const& -> list_t const& { return Cast<list_t>(); }
+ [[nodiscard]] auto List() && -> list_t {
+ return std::move(*this).Cast<list_t>();
+ }
+ [[nodiscard]] auto Map() const& -> map_t const& { return Cast<map_t>(); }
+ [[nodiscard]] auto Map() && -> map_t {
+ return std::move(*this).Cast<map_t>();
+ }
+
+ [[nodiscard]] auto At(std::string const& key)
+ const& -> std::optional<std::reference_wrapper<ExpressionPtr const>> {
+ auto value = Map().Find(key);
+ if (value) {
+ return value;
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto At(
+ std::string const& key) && -> std::optional<ExpressionPtr> {
+ auto value = std::move(*this).Map().Find(key);
+ if (value) {
+ return std::move(*value);
+ }
+ return std::nullopt;
+ }
+
+ template <class T>
+ requires(IsValidType<std::remove_cvref_t<T>>() or
+ std::is_same_v<std::remove_cvref_t<T>, ExpressionPtr>)
+ [[nodiscard]] auto Get(std::string const& key, T&& default_value) const
+ -> ExpressionPtr {
+ auto value = At(key);
+ if (value) {
+ return value->get();
+ }
+ if constexpr (std::is_same_v<std::remove_cvref_t<T>, ExpressionPtr>) {
+ return std::forward<T>(default_value);
+ }
+ else {
+ return ExpressionPtr{std::forward<T>(default_value)};
+ }
+ }
+
+ template <class T>
+ requires(IsValidType<T>()) [[nodiscard]] auto Value() const& noexcept
+ -> std::optional<std::reference_wrapper<T const>> {
+ if (GetIndexOf<T>() == data_.index()) {
+ return std::make_optional(std::ref(std::get<T>(data_)));
+ }
+ return std::nullopt;
+ }
+
+ template <class T>
+ requires(IsValidType<T>()) [[nodiscard]] auto Value() && noexcept
+ -> std::optional<T> {
+ if (GetIndexOf<T>() == data_.index()) {
+ return std::make_optional(std::move(std::get<T>(data_)));
+ }
+ return std::nullopt;
+ }
+
+ template <class T>
+ [[nodiscard]] auto operator==(T const& other) const noexcept -> bool {
+ if constexpr (std::is_same_v<T, Expression>) {
+ return (&data_ == &other.data_) or (ToHash() == other.ToHash());
+ }
+ else {
+ return IsValidType<T>() and (GetIndexOf<T>() == data_.index()) and
+ ((static_cast<void const*>(&data_) ==
+ static_cast<void const*>(&other)) or
+ (std::get<T>(data_) == other));
+ }
+ }
+
+ template <class T>
+ [[nodiscard]] auto operator!=(T const& other) const noexcept -> bool {
+ return !(*this == other);
+ }
+ [[nodiscard]] auto operator[](
+ std::string const& key) const& -> ExpressionPtr const&;
+ [[nodiscard]] auto operator[](std::string const& key) && -> ExpressionPtr;
+ [[nodiscard]] auto operator[](
+ ExpressionPtr const& key) const& -> ExpressionPtr const&;
+ [[nodiscard]] auto operator[](ExpressionPtr const& key) && -> ExpressionPtr;
+ [[nodiscard]] auto operator[](size_t pos) const& -> ExpressionPtr const&;
+ [[nodiscard]] auto operator[](size_t pos) && -> ExpressionPtr;
+
+ enum class JsonMode { SerializeAll, SerializeAllButNodes, NullForNonJson };
+
+ [[nodiscard]] auto ToJson(JsonMode mode = JsonMode::SerializeAll) const
+ -> nlohmann::json;
+ [[nodiscard]] auto IsCacheable() const -> bool;
+ [[nodiscard]] auto ToString() const -> std::string;
+ [[nodiscard]] auto ToHash() const noexcept -> std::string;
+ [[nodiscard]] auto ToIdentifier() const noexcept -> std::string {
+ return ToHexString(ToHash());
+ }
+
+ [[nodiscard]] static auto FromJson(nlohmann::json const& json) noexcept
+ -> ExpressionPtr;
+
+ inline static ExpressionPtr const kNone = Expression::FromJson("null"_json);
+ inline static ExpressionPtr const kEmptyMap =
+ Expression::FromJson("{}"_json);
+ inline static ExpressionPtr const kEmptyList =
+ Expression::FromJson("[]"_json);
+ inline static ExpressionPtr const kEmptyMapExpr =
+ Expression::FromJson(R"({"type": "empty_map"})"_json);
+
+ private:
+ inline static HashGenerator const hash_gen_{
+ HashGenerator::HashType::SHA256};
+
+ std::variant<none_t,
+ bool,
+ number_t,
+ std::string,
+ name_t,
+ artifact_t,
+ result_t,
+ node_t,
+ list_t,
+ map_t>
+ data_{none_t{}};
+
+ mutable atomic_shared_ptr<std::string> hash_{};
+ mutable std::atomic<bool> hash_loading_{};
+
+ template <class T, std::size_t kIndex = 0>
+ requires(IsValidType<T>()) [[nodiscard]] static consteval auto GetIndexOf()
+ -> std::size_t {
+ static_assert(kIndex < std::variant_size_v<decltype(data_)>,
+ "kIndex out of range");
+ if constexpr (std::is_same_v<
+ T,
+ std::variant_alternative_t<kIndex,
+ decltype(data_)>>) {
+ return kIndex;
+ }
+ else {
+ return GetIndexOf<T, kIndex + 1>();
+ }
+ }
+
+ template <class T>
+ [[nodiscard]] auto IsA() const noexcept -> bool {
+ return std::holds_alternative<T>(data_);
+ }
+
+ template <class T>
+ [[nodiscard]] auto Cast() const& -> T const& {
+ if (GetIndexOf<T>() == data_.index()) {
+ return std::get<T>(data_);
+ }
+ // throw descriptive ExpressionTypeError
+ throw ExpressionTypeError{
+ fmt::format("Expression is not of type '{}' but '{}'.",
+ TypeToString<T>(),
+ TypeString())};
+ }
+
+ template <class T>
+ [[nodiscard]] auto Cast() && -> T {
+ if (GetIndexOf<T>() == data_.index()) {
+ return std::move(std::get<T>(data_));
+ }
+ // throw descriptive ExpressionTypeError
+ throw ExpressionTypeError{
+ fmt::format("Expression is not of type '{}' but '{}'.",
+ TypeToString<T>(),
+ TypeString())};
+ }
+
+ template <class T>
+ requires(Expression::IsValidType<T>())
+ [[nodiscard]] static auto TypeToString() noexcept -> std::string {
+ if constexpr (std::is_same_v<T, bool>) {
+ return "bool";
+ }
+ else if constexpr (std::is_same_v<T, Expression::number_t>) {
+ return "number";
+ }
+ else if constexpr (std::is_same_v<T, Expression::name_t>) {
+ return "name";
+ }
+ else if constexpr (std::is_same_v<T, std::string>) {
+ return "string";
+ }
+ else if constexpr (std::is_same_v<T, Expression::artifact_t>) {
+ return "artifact";
+ }
+ else if constexpr (std::is_same_v<T, Expression::result_t>) {
+ return "result";
+ }
+ else if constexpr (std::is_same_v<T, Expression::node_t>) {
+ return "node";
+ }
+ else if constexpr (std::is_same_v<T, Expression::list_t>) {
+ return "list";
+ }
+ else if constexpr (std::is_same_v<T, Expression::map_t>) {
+ return "map";
+ }
+ return "none";
+ }
+
+ template <size_t kIndex = 0>
+ [[nodiscard]] auto TypeStringForIndex() const noexcept -> std::string;
+ [[nodiscard]] auto TypeString() const noexcept -> std::string;
+ [[nodiscard]] auto ComputeHash() const noexcept -> std::string;
+};
+
+[[nodiscard]] inline auto operator+(Expression const& lhs,
+ Expression const& rhs) -> Expression {
+ if (lhs.data_.index() != rhs.data_.index()) {
+ throw Expression::ExpressionTypeError{
+ fmt::format("Cannot add expressions of different type: {} != {}",
+ lhs.TypeString(),
+ rhs.TypeString())};
+ }
+ if (not lhs.IsList()) {
+ throw Expression::ExpressionTypeError{fmt::format(
+ "Cannot add expressions of type '{}'.", lhs.TypeString())};
+ }
+ auto list = Expression::list_t{};
+ auto const& llist = lhs.List();
+ auto const& rlist = rhs.List();
+ list.reserve(llist.size() + rlist.size());
+ list.insert(list.begin(), llist.begin(), llist.end());
+ list.insert(list.end(), rlist.begin(), rlist.end());
+ return Expression{list};
+}
+
+namespace std {
+template <>
+struct hash<Expression> {
+ [[nodiscard]] auto operator()(Expression const& e) const noexcept
+ -> std::size_t {
+ auto hash = std::size_t{};
+ auto bytes = e.ToHash();
+ std::memcpy(&hash, bytes.data(), std::min(sizeof(hash), bytes.size()));
+ return hash;
+ }
+};
+} // namespace std
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_HPP
diff --git a/src/buildtool/build_engine/expression/expression_ptr.cpp b/src/buildtool/build_engine/expression/expression_ptr.cpp
new file mode 100644
index 00000000..97cdb138
--- /dev/null
+++ b/src/buildtool/build_engine/expression/expression_ptr.cpp
@@ -0,0 +1,89 @@
+#include <string>
+
+#include "src/buildtool/build_engine/expression/evaluator.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+
+ExpressionPtr::ExpressionPtr() noexcept : ptr_{Expression::kNone.ptr_} {}
+
+auto ExpressionPtr::operator*() && -> Expression {
+ return *ptr_;
+}
+
+auto ExpressionPtr::operator[](
+ std::string const& key) const& -> ExpressionPtr const& {
+ return (*ptr_)[key];
+}
+
+auto ExpressionPtr::operator[](std::string const& key) && -> ExpressionPtr {
+ return (*ptr_)[key];
+}
+
+auto ExpressionPtr::operator[](
+ ExpressionPtr const& key) const& -> ExpressionPtr const& {
+ return (*ptr_)[key];
+}
+
+auto ExpressionPtr::operator[](ExpressionPtr const& key) && -> ExpressionPtr {
+ return (*ptr_)[key];
+}
+
+auto ExpressionPtr::operator[](size_t pos) const& -> ExpressionPtr const& {
+ return (*ptr_)[pos];
+}
+
+auto ExpressionPtr::operator[](size_t pos) && -> ExpressionPtr {
+ return (*ptr_)[pos];
+}
+
+auto ExpressionPtr::operator<(ExpressionPtr const& other) const -> bool {
+ return ptr_->ToHash() < other.ptr_->ToHash();
+}
+
+auto ExpressionPtr::operator==(ExpressionPtr const& other) const -> bool {
+ return ptr_ == other.ptr_ or (ptr_ and other.ptr_ and *ptr_ == *other.ptr_);
+}
+
+auto ExpressionPtr::Evaluate(
+ Configuration const& env,
+ FunctionMapPtr const& functions,
+ std::function<void(std::string const&)> const& logger,
+ std::function<void(void)> const& note_user_context) const noexcept
+ -> ExpressionPtr {
+ return Evaluator::EvaluateExpression(
+ *this, env, functions, logger, note_user_context);
+}
+
+auto ExpressionPtr::IsCacheable() const noexcept -> bool {
+ return ptr_ and ptr_->IsCacheable();
+}
+
+auto ExpressionPtr::ToIdentifier() const noexcept -> std::string {
+ return ptr_ ? ptr_->ToIdentifier() : std::string{};
+}
+
+auto ExpressionPtr::ToJson() const noexcept -> nlohmann::json {
+ return ptr_ ? ptr_->ToJson() : nlohmann::json::object();
+}
+
+auto ExpressionPtr::IsNotNull() const noexcept -> bool {
+ // ExpressionPtr is nullptr in error case and none_t default empty case.
+ return static_cast<bool>(ptr_) and not(ptr_->IsNone());
+}
+
+auto ExpressionPtr::LinkedMap() const& -> ExpressionPtr::linked_map_t const& {
+ return ptr_->Map();
+}
+
+auto ExpressionPtr::Make(linked_map_t&& map) -> ExpressionPtr {
+ return ExpressionPtr{std::move(map)};
+}
+
+auto operator+(ExpressionPtr const& lhs, ExpressionPtr const& rhs)
+ -> ExpressionPtr {
+ return ExpressionPtr{*lhs + *rhs};
+}
+
+auto std::hash<ExpressionPtr>::operator()(ExpressionPtr const& p) const noexcept
+ -> std::size_t {
+ return std::hash<Expression>{}(*p);
+}
diff --git a/src/buildtool/build_engine/expression/expression_ptr.hpp b/src/buildtool/build_engine/expression/expression_ptr.hpp
new file mode 100644
index 00000000..8cc26c50
--- /dev/null
+++ b/src/buildtool/build_engine/expression/expression_ptr.hpp
@@ -0,0 +1,95 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_PTR_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_PTR_HPP
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <type_traits>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/build_engine/expression/function_map.hpp"
+#include "src/buildtool/build_engine/expression/linked_map.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+class Configuration;
+class Expression;
+
+class ExpressionPtr {
+ public:
+ // Initialize to nullptr
+ explicit ExpressionPtr(std::nullptr_t /*ptr*/) noexcept : ptr_{nullptr} {}
+
+ // Initialize from Expression's variant type or Expression
+ template <class T>
+ requires(not std::is_same_v<std::remove_cvref_t<T>, ExpressionPtr>)
+ // NOLINTNEXTLINE(bugprone-forwarding-reference-overload)
+ explicit ExpressionPtr(T&& data) noexcept
+ : ptr_{std::make_shared<Expression>(std::forward<T>(data))} {}
+
+ ExpressionPtr() noexcept;
+ ExpressionPtr(ExpressionPtr const&) noexcept = default;
+ ExpressionPtr(ExpressionPtr&&) noexcept = default;
+ ~ExpressionPtr() noexcept = default;
+ auto operator=(ExpressionPtr const&) noexcept -> ExpressionPtr& = default;
+ auto operator=(ExpressionPtr&&) noexcept -> ExpressionPtr& = default;
+
+ explicit operator bool() const { return static_cast<bool>(ptr_); }
+ [[nodiscard]] auto operator*() const& -> Expression const& { return *ptr_; }
+ [[nodiscard]] auto operator*() && -> Expression;
+ [[nodiscard]] auto operator->() const& -> Expression const* {
+ return ptr_.get();
+ }
+ [[nodiscard]] auto operator->() && -> Expression const* = delete;
+ [[nodiscard]] auto operator[](
+ std::string const& key) const& -> ExpressionPtr const&;
+ [[nodiscard]] auto operator[](std::string const& key) && -> ExpressionPtr;
+ [[nodiscard]] auto operator[](
+ ExpressionPtr const& key) const& -> ExpressionPtr const&;
+ [[nodiscard]] auto operator[](ExpressionPtr const& key) && -> ExpressionPtr;
+ [[nodiscard]] auto operator[](size_t pos) const& -> ExpressionPtr const&;
+ [[nodiscard]] auto operator[](size_t pos) && -> ExpressionPtr;
+ [[nodiscard]] auto operator<(ExpressionPtr const& other) const -> bool;
+ [[nodiscard]] auto operator==(ExpressionPtr const& other) const -> bool;
+ template <class T>
+ [[nodiscard]] auto operator==(T const& other) const -> bool {
+ return ptr_ and *ptr_ == other;
+ }
+ template <class T>
+ [[nodiscard]] auto operator!=(T const& other) const -> bool {
+ return not(*this == other);
+ }
+ [[nodiscard]] auto Evaluate(
+ Configuration const& env,
+ FunctionMapPtr const& functions,
+ std::function<void(std::string const&)> const& logger =
+ [](std::string const& error) noexcept -> void {
+ Logger::Log(LogLevel::Error, error);
+ },
+ std::function<void(void)> const& note_user_context =
+ []() noexcept -> void {}) const noexcept -> ExpressionPtr;
+
+ [[nodiscard]] auto IsCacheable() const noexcept -> bool;
+ [[nodiscard]] auto ToIdentifier() const noexcept -> std::string;
+ [[nodiscard]] auto ToJson() const noexcept -> nlohmann::json;
+
+ using linked_map_t = LinkedMap<std::string, ExpressionPtr, ExpressionPtr>;
+ [[nodiscard]] auto IsNotNull() const noexcept -> bool;
+ [[nodiscard]] auto LinkedMap() const& -> linked_map_t const&;
+ [[nodiscard]] static auto Make(linked_map_t&& map) -> ExpressionPtr;
+
+ private:
+ std::shared_ptr<Expression> ptr_;
+};
+
+[[nodiscard]] auto operator+(ExpressionPtr const& lhs, ExpressionPtr const& rhs)
+ -> ExpressionPtr;
+
+namespace std {
+template <>
+struct hash<ExpressionPtr> {
+ [[nodiscard]] auto operator()(ExpressionPtr const& p) const noexcept
+ -> std::size_t;
+};
+} // namespace std
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_EXPRESSION_PTR_HPP
diff --git a/src/buildtool/build_engine/expression/function_map.hpp b/src/buildtool/build_engine/expression/function_map.hpp
new file mode 100644
index 00000000..967d5fe0
--- /dev/null
+++ b/src/buildtool/build_engine/expression/function_map.hpp
@@ -0,0 +1,23 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_FUNCTION_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_FUNCTION_MAP_HPP
+
+#include <functional>
+#include <string>
+
+#include "src/buildtool/build_engine/expression/linked_map.hpp"
+
+class ExpressionPtr;
+class Configuration;
+
+using SubExprEvaluator =
+ std::function<ExpressionPtr(ExpressionPtr const&, Configuration const&)>;
+
+using FunctionMap =
+ LinkedMap<std::string,
+ std::function<ExpressionPtr(SubExprEvaluator&&,
+ ExpressionPtr const&,
+ Configuration const&)>>;
+
+using FunctionMapPtr = FunctionMap::Ptr;
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_FUNCTION_MAP_HPP
diff --git a/src/buildtool/build_engine/expression/linked_map.hpp b/src/buildtool/build_engine/expression/linked_map.hpp
new file mode 100644
index 00000000..5c6da558
--- /dev/null
+++ b/src/buildtool/build_engine/expression/linked_map.hpp
@@ -0,0 +1,414 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_LINKED_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_LINKED_MAP_HPP
+
+#include <algorithm>
+#include <atomic>
+#include <condition_variable>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+#include "fmt/core.h"
+#include "src/utils/cpp/atomic.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+
+template <class K, class V, class NextPtr>
+class LinkedMap;
+
+// Default NextPtr for LinkedMap, based on std::shared_ptr.
+template <class K, class V>
+class LinkedMapPtr {
+ using ptr_t = LinkedMapPtr<K, V>;
+ using map_t = LinkedMap<K, V, ptr_t>;
+
+ public:
+ LinkedMapPtr() noexcept = default;
+ explicit LinkedMapPtr(std::shared_ptr<map_t> ptr) noexcept
+ : ptr_{std::move(ptr)} {}
+ explicit operator bool() const { return static_cast<bool>(ptr_); }
+ [[nodiscard]] auto operator*() const& -> map_t const& { return *ptr_; }
+ [[nodiscard]] auto operator->() const& -> map_t const* {
+ return ptr_.get();
+ }
+ [[nodiscard]] auto IsNotNull() const noexcept -> bool {
+ return static_cast<bool>(ptr_);
+ }
+ [[nodiscard]] auto LinkedMap() const& -> map_t const& { return *ptr_; }
+ [[nodiscard]] static auto Make(map_t&& map) -> ptr_t {
+ return ptr_t{std::make_shared<map_t>(std::move(map))};
+ }
+
+ private:
+ std::shared_ptr<map_t> ptr_{};
+};
+
+/// \brief Immutable LinkedMap.
+/// Uses smart pointers to build up a list of pointer-linked maps. The NextPtr
+/// that is used internally can be overloaded by any class implementing the
+/// following methods:
+/// 1. auto IsNotNull() const noexcept -> bool;
+/// 2. auto LinkedMap() const& -> LinkedMap<K, V, NextPtr> const&;
+/// 3. static auto Make(LinkedMap<K, V, NextPtr>&&) -> NextPtr;
+template <class K, class V, class NextPtr = LinkedMapPtr<K, V>>
+class LinkedMap {
+ using item_t = std::pair<K, V>;
+ using items_t = std::vector<item_t>;
+ using keys_t = std::vector<K>;
+ using values_t = std::vector<V>;
+
+ public:
+ using Ptr = NextPtr;
+ // When merging maps, we always rely on entries being traversed in key
+ // order; so keep the underlying map an ordered data structure.
+ using underlying_map_t = std::map<K, V>;
+
+ static constexpr auto MakePtr(underlying_map_t map) -> Ptr {
+ return Ptr::Make(LinkedMap<K, V, Ptr>{std::move(map)});
+ }
+
+ static constexpr auto MakePtr(item_t item) -> Ptr {
+ return Ptr::Make(LinkedMap<K, V, Ptr>{std::move(item)});
+ }
+
+ static constexpr auto MakePtr(K key, V value) -> Ptr {
+ return Ptr::Make(
+ LinkedMap<K, V, Ptr>{std::move(key), std::move(value)});
+ }
+
+ static constexpr auto MakePtr(Ptr next, Ptr content) -> Ptr {
+ return Ptr::Make(LinkedMap<K, V, Ptr>{next, content});
+ }
+
+ static constexpr auto MakePtr(Ptr next, underlying_map_t map) -> Ptr {
+ return Ptr::Make(LinkedMap<K, V, Ptr>{next, std::move(map)});
+ }
+
+ static constexpr auto MakePtr(Ptr const& next, item_t item) -> Ptr {
+ return Ptr::Make(LinkedMap<K, V, Ptr>{next, std::move(item)});
+ }
+
+ static constexpr auto MakePtr(Ptr const& next, K key, V value) -> Ptr {
+ return Ptr::Make(
+ LinkedMap<K, V, Ptr>{next, std::move(key), std::move(value)});
+ }
+
+ explicit LinkedMap(underlying_map_t map) noexcept : map_{std::move(map)} {}
+ explicit LinkedMap(item_t item) noexcept { map_.emplace(std::move(item)); }
+ LinkedMap(K key, V val) noexcept {
+ map_.emplace(std::move(key), std::move(val));
+ }
+ LinkedMap(Ptr next, Ptr content) noexcept
+ : next_{std::move(next)}, content_{std::move(content)} {}
+ LinkedMap(Ptr next, underlying_map_t map) noexcept
+ : next_{std::move(next)}, map_{std::move(map)} {}
+ LinkedMap(Ptr next, item_t item) noexcept : next_{std::move(next)} {
+ map_.emplace(std::move(item));
+ }
+ LinkedMap(Ptr next, K key, V val) noexcept : next_{std::move(next)} {
+ map_.emplace(std::move(key), std::move(val));
+ }
+
+ LinkedMap() noexcept = default;
+ LinkedMap(LinkedMap const& other) noexcept
+ : next_{other.next_},
+ content_{other.content_},
+ map_{other.map_},
+ items_{other.items_.load()} {}
+ LinkedMap(LinkedMap&& other) noexcept
+ : next_{std::move(other.next_)},
+ content_{std::move(other.content_)},
+ map_{std::move(other.map_)},
+ items_{other.items_.load()} {}
+ ~LinkedMap() noexcept = default;
+
+ auto operator=(LinkedMap const& other) noexcept -> LinkedMap& {
+ next_ = other.next_;
+ content_ = other.content_;
+ map_ = other.map_;
+ items_ = other.items_.load();
+ return *this;
+ }
+ auto operator=(LinkedMap&& other) noexcept -> LinkedMap& {
+ next_ = std::move(other.next_);
+ content_ = std::move(other.content_);
+ map_ = std::move(other.map_);
+ items_ = other.items_.load();
+ return *this;
+ }
+
+ [[nodiscard]] auto contains(K const& key) const noexcept -> bool {
+ return static_cast<bool>(Find(key));
+ }
+
+ [[nodiscard]] auto at(K const& key) const& -> V const& {
+ auto value = Find(key);
+ if (value) {
+ return value->get();
+ }
+ throw std::out_of_range{fmt::format("Missing key {}", key)};
+ }
+
+ [[nodiscard]] auto at(K const& key) && -> V {
+ auto value = Find(key);
+ if (value) {
+ return std::move(*value);
+ }
+ throw std::out_of_range{fmt::format("Missing key {}", key)};
+ }
+
+ [[nodiscard]] auto operator[](K const& key) const& -> V const& {
+ return at(key);
+ }
+
+ [[nodiscard]] auto empty() const noexcept -> bool {
+ return (content_.IsNotNull() ? content_.LinkedMap().empty()
+ : map_.empty()) and
+ (not next_.IsNotNull() or next_.LinkedMap().empty());
+ }
+
+ [[nodiscard]] auto Find(K const& key) const& noexcept
+ -> std::optional<std::reference_wrapper<V const>> {
+ if (content_.IsNotNull()) {
+ auto val = content_.LinkedMap().Find(key);
+ if (val) {
+ return val;
+ }
+ }
+ else {
+ auto it = map_.find(key);
+ if (it != map_.end()) {
+ return it->second;
+ }
+ }
+ if (next_.IsNotNull()) {
+ auto val = next_.LinkedMap().Find(key);
+ if (val) {
+ return val;
+ }
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto Find(K const& key) && noexcept -> std::optional<V> {
+ if (content_.IsNotNull()) {
+ auto val = content_.LinkedMap().Find(key);
+ if (val) {
+ return val->get();
+ }
+ }
+ else {
+ auto it = map_.find(key);
+ if (it != map_.end()) {
+ return std::move(it->second);
+ }
+ }
+ if (next_.IsNotNull()) {
+ auto val = next_.LinkedMap().Find(key);
+ if (val) {
+ return val->get();
+ }
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto FindConflictingDuplicate(LinkedMap const& other)
+ const& noexcept -> std::optional<std::reference_wrapper<K const>> {
+ auto const& my_items = Items();
+ auto const& other_items = other.Items();
+ // Search for duplicates, using that iteration over the items is
+ // orderd by keys.
+ auto me = my_items.begin();
+ auto they = other_items.begin();
+ while (me != my_items.end() and they != other_items.end()) {
+ if (me->first == they->first) {
+ if (not(me->second == they->second)) {
+ return me->first;
+ }
+ ++me;
+ ++they;
+ }
+ else if (me->first < they->first) {
+ ++me;
+ }
+ else {
+ ++they;
+ }
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto FindConflictingDuplicate(
+ LinkedMap const& other) && noexcept = delete;
+
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto size() const noexcept -> std::size_t {
+ return Items().size();
+ }
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto begin() const& -> typename items_t::const_iterator {
+ return Items().cbegin();
+ }
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto end() const& -> typename items_t::const_iterator {
+ return Items().cend();
+ }
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto cbegin() const& -> typename items_t::const_iterator {
+ return begin();
+ }
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto cend() const& -> typename items_t::const_iterator {
+ return end();
+ }
+
+ [[nodiscard]] auto begin() && -> typename items_t::const_iterator = delete;
+ [[nodiscard]] auto end() && -> typename items_t::const_iterator = delete;
+ [[nodiscard]] auto cbegin() && -> typename items_t::const_iterator = delete;
+ [[nodiscard]] auto cend() && -> typename items_t::const_iterator = delete;
+
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto operator==(
+ LinkedMap<K, V, NextPtr> const& other) const noexcept -> bool {
+ return this == &other or (this->empty() and other.empty()) or
+ this->Items() == other.Items();
+ }
+
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto Items() const& -> items_t const& {
+ if (items_.load() == nullptr) {
+ if (not items_loading_.exchange(true)) {
+ items_ = std::make_shared<items_t>(ComputeSortedItems());
+ items_.notify_all();
+ }
+ else {
+ items_.wait(nullptr);
+ }
+ }
+ return *items_.load();
+ }
+
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto Items() && -> items_t {
+ return items_.load() == nullptr ? ComputeSortedItems()
+ : std::move(*items_.load());
+ }
+
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto Keys() const -> keys_t {
+ auto keys = keys_t{};
+ auto const& items = Items();
+ keys.reserve(items.size());
+ std::transform(items.begin(),
+ items.end(),
+ std::back_inserter(keys),
+ [](auto const& item) { return item.first; });
+ return keys;
+ }
+
+ // NOTE: Expensive, needs to compute sorted items.
+ [[nodiscard]] auto Values() const -> values_t {
+ auto values = values_t{};
+ auto const& items = Items();
+ values.reserve(items.size());
+ std::transform(items.begin(),
+ items.end(),
+ std::back_inserter(values),
+ [](auto const& item) { return item.second; });
+ return values;
+ }
+
+ private:
+ Ptr next_{}; // map that is shadowed by this map
+ Ptr content_{}; // content of this map if set
+ underlying_map_t map_{}; // content of this map if content_ is not set
+
+ mutable atomic_shared_ptr<items_t> items_{};
+ mutable std::atomic<bool> items_loading_{};
+
+ [[nodiscard]] auto ComputeSortedItems() const noexcept -> items_t {
+ auto size =
+ content_.IsNotNull() ? content_.LinkedMap().size() : map_.size();
+ if (next_.IsNotNull()) {
+ size += next_.LinkedMap().size();
+ }
+
+ auto items = items_t{};
+ items.reserve(size);
+
+ auto empty = items_t{};
+ auto map_copy = items_t{};
+ typename items_t::const_iterator citemsit;
+ typename items_t::const_iterator citemsend;
+ typename items_t::const_iterator nitemsit;
+ typename items_t::const_iterator nitemsend;
+
+ if (content_.IsNotNull()) {
+ auto const& citems = content_.LinkedMap().Items();
+ citemsit = citems.begin();
+ citemsend = citems.end();
+ }
+ else {
+ map_copy.reserve(map_.size());
+ map_copy.insert(map_copy.end(), map_.begin(), map_.end());
+ citemsit = map_copy.begin();
+ citemsend = map_copy.end();
+ }
+ if (next_.IsNotNull()) {
+ auto const& nitems = next_.LinkedMap().Items();
+ nitemsit = nitems.begin();
+ nitemsend = nitems.end();
+ }
+ else {
+ nitemsit = empty.begin();
+ nitemsend = empty.end();
+ }
+
+ while (citemsit != citemsend and nitemsit != nitemsend) {
+ if (citemsit->first == nitemsit->first) {
+ items.push_back(*citemsit);
+ ++citemsit;
+ ++nitemsit;
+ }
+ else if (citemsit->first < nitemsit->first) {
+ items.push_back(*citemsit);
+ ++citemsit;
+ }
+ else {
+ items.push_back(*nitemsit);
+ ++nitemsit;
+ }
+ }
+
+ // No more comaprisons to be made; copy over the remaining
+ // entries
+ items.insert(items.end(), citemsit, citemsend);
+ items.insert(items.end(), nitemsit, nitemsend);
+
+ return items;
+ }
+};
+
+namespace std {
+template <class K, class V, class N>
+struct hash<LinkedMap<K, V, N>> {
+ [[nodiscard]] auto operator()(LinkedMap<K, V, N> const& m) const noexcept
+ -> std::size_t {
+ size_t seed{};
+ for (auto const& e : m) {
+ hash_combine(&seed, e.first);
+ hash_combine(&seed, e.second);
+ }
+ return seed;
+ }
+};
+template <class K, class V>
+struct hash<LinkedMapPtr<K, V>> {
+ [[nodiscard]] auto operator()(LinkedMapPtr<K, V> const& p) const noexcept
+ -> std::size_t {
+ return std::hash<std::remove_cvref_t<decltype(*p)>>{}(*p);
+ }
+};
+} // namespace std
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_EXPRESSION_LINKED_MAP_HPP
diff --git a/src/buildtool/build_engine/expression/target_node.cpp b/src/buildtool/build_engine/expression/target_node.cpp
new file mode 100644
index 00000000..03fc47f5
--- /dev/null
+++ b/src/buildtool/build_engine/expression/target_node.cpp
@@ -0,0 +1,20 @@
+#include "src/buildtool/build_engine/expression/target_node.hpp"
+
+#include "src/buildtool/build_engine/expression/expression.hpp"
+
+auto TargetNode::Abstract::IsCacheable() const noexcept -> bool {
+ return target_fields->IsCacheable();
+}
+
+auto TargetNode::ToJson() const -> nlohmann::json {
+ if (IsValue()) {
+ return {{"type", "VALUE_NODE"}, {"result", GetValue()->ToJson()}};
+ }
+ auto const& data = GetAbstract();
+ return {{"type", "ABSTRACT_NODE"},
+ {"node_type", data.node_type},
+ {"string_fields", data.string_fields->ToJson()},
+ {"target_fields",
+ data.target_fields->ToJson(
+ Expression::JsonMode::SerializeAllButNodes)}};
+}
diff --git a/src/buildtool/build_engine/expression/target_node.hpp b/src/buildtool/build_engine/expression/target_node.hpp
new file mode 100644
index 00000000..a2ab9c83
--- /dev/null
+++ b/src/buildtool/build_engine/expression/target_node.hpp
@@ -0,0 +1,83 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_NODE_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_NODE_HPP
+
+#include <type_traits>
+#include <variant>
+
+#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
+#include "src/buildtool/build_engine/expression/target_result.hpp"
+
+class TargetNode {
+ using Value = ExpressionPtr; // store result type
+
+ public:
+ struct Abstract {
+ std::string node_type; // arbitrary string that maps to rule
+ ExpressionPtr string_fields; // map to list of strings
+ ExpressionPtr target_fields; // map to list of targets
+ [[nodiscard]] auto IsCacheable() const noexcept -> bool;
+ };
+
+ template <class NodeType>
+ requires(
+ std::is_same_v<NodeType, Value> or
+ std::is_same_v<NodeType, Abstract>) explicit TargetNode(NodeType node)
+ : data_{std::move(node)},
+ is_cacheable_{std::get<NodeType>(data_).IsCacheable()} {}
+
+ [[nodiscard]] auto IsCacheable() const noexcept -> bool {
+ return is_cacheable_;
+ }
+
+ [[nodiscard]] auto IsValue() const noexcept {
+ return std::holds_alternative<Value>(data_);
+ }
+
+ [[nodiscard]] auto IsAbstract() const noexcept {
+ return std::holds_alternative<Abstract>(data_);
+ }
+
+ [[nodiscard]] auto GetValue() const -> Value const& {
+ return std::get<Value>(data_);
+ }
+
+ [[nodiscard]] auto GetAbstract() const -> Abstract const& {
+ return std::get<Abstract>(data_);
+ }
+
+ [[nodiscard]] auto operator==(TargetNode const& other) const noexcept
+ -> bool {
+ if (data_.index() == other.data_.index()) {
+ try {
+ if (IsValue()) {
+ return GetValue() == other.GetValue();
+ }
+ auto const& abs_l = GetAbstract();
+ auto const& abs_r = other.GetAbstract();
+ return abs_l.node_type == abs_r.node_type and
+ abs_l.string_fields == abs_r.string_fields and
+ abs_l.target_fields == abs_r.string_fields;
+ } catch (...) {
+ // should never happen
+ }
+ }
+ return false;
+ }
+
+ [[nodiscard]] auto ToString() const noexcept -> std::string {
+ try {
+ return ToJson().dump();
+ } catch (...) {
+ // should never happen
+ }
+ return {};
+ }
+
+ [[nodiscard]] auto ToJson() const -> nlohmann::json;
+
+ private:
+ std::variant<Value, Abstract> data_;
+ bool is_cacheable_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_NODE_HPP
diff --git a/src/buildtool/build_engine/expression/target_result.hpp b/src/buildtool/build_engine/expression/target_result.hpp
new file mode 100644
index 00000000..325d52fd
--- /dev/null
+++ b/src/buildtool/build_engine/expression/target_result.hpp
@@ -0,0 +1,33 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_RESULT_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_RESULT_HPP
+
+#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+
+struct TargetResult {
+ ExpressionPtr artifact_stage{};
+ ExpressionPtr provides{};
+ ExpressionPtr runfiles{};
+ bool is_cacheable{provides.IsCacheable()};
+
+ [[nodiscard]] auto operator==(TargetResult const& other) const noexcept
+ -> bool {
+ return artifact_stage == other.artifact_stage and
+ provides == other.provides and runfiles == other.runfiles;
+ }
+};
+
+namespace std {
+template <>
+struct std::hash<TargetResult> {
+ [[nodiscard]] auto operator()(TargetResult const& r) noexcept
+ -> std::size_t {
+ auto seed = std::hash<ExpressionPtr>{}(r.artifact_stage);
+ hash_combine(&seed, std::hash<ExpressionPtr>{}(r.provides));
+ hash_combine(&seed, std::hash<ExpressionPtr>{}(r.runfiles));
+ return seed;
+ }
+};
+} // namespace std
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILDENGINE_EXPRESSION_TARGET_RESULT_HPP
diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS
new file mode 100644
index 00000000..71c9dd78
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/TARGETS
@@ -0,0 +1,50 @@
+{ "configured_target":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["configured_target"]
+ , "hdrs": ["configured_target.hpp"]
+ , "deps":
+ [ ["@", "fmt", "", "fmt"]
+ , ["src/buildtool/build_engine/base_maps", "entity_name"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/utils/cpp", "hash_combine"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "target_map"]
+ }
+, "result_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["result_map"]
+ , "hdrs": ["result_map.hpp"]
+ , "deps":
+ [ ["src/buildtool/common", "tree"]
+ , ["src/buildtool/build_engine/analysed_target", "target"]
+ , ["src/buildtool/build_engine/target_map", "configured_target"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/multithreading", "task"]
+ , ["src/buildtool/multithreading", "task_system"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "target_map"]
+ }
+, "target_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["target_map"]
+ , "hdrs": ["target_map.hpp"]
+ , "srcs": ["utils.cpp", "built_in_rules.cpp", "export.cpp", "target_map.cpp"]
+ , "private-hdrs": ["built_in_rules.hpp", "export.hpp", "utils.hpp"]
+ , "deps":
+ [ "configured_target"
+ , "result_map"
+ , ["src/buildtool/build_engine/analysed_target", "target"]
+ , ["src/buildtool/build_engine/base_maps", "entity_name"]
+ , ["src/buildtool/build_engine/base_maps", "field_reader"]
+ , ["src/buildtool/build_engine/base_maps", "rule_map"]
+ , ["src/buildtool/build_engine/base_maps", "source_map"]
+ , ["src/buildtool/build_engine/base_maps", "targets_file_map"]
+ , ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/multithreading", "async_map_consumer"]
+ , ["src/utils/cpp", "hash_combine"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "build_engine", "target_map"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/build_engine/target_map/built_in_rules.cpp b/src/buildtool/build_engine/target_map/built_in_rules.cpp
new file mode 100644
index 00000000..b98484b6
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/built_in_rules.cpp
@@ -0,0 +1,857 @@
+#include "src/buildtool/build_engine/target_map/built_in_rules.hpp"
+
+#include <algorithm>
+#include <filesystem>
+#include <functional>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "src/buildtool/build_engine/base_maps/field_reader.hpp"
+#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
+#include "src/buildtool/build_engine/target_map/export.hpp"
+#include "src/buildtool/build_engine/target_map/utils.hpp"
+
+namespace {
+
+auto genericRuleFields = std::unordered_set<std::string>{"arguments_config",
+ "cmds",
+ "deps",
+ "env",
+ "tainted",
+ "type",
+ "outs"};
+
+auto fileGenRuleFields = std::unordered_set<std::string>{"arguments_config",
+ "data",
+ "deps",
+ "name",
+ "tainted",
+ "type"};
+
+auto installRuleFields = std::unordered_set<std::string>{"arguments_config",
+ "deps",
+ "dirs",
+ "files",
+ "tainted",
+ "type"};
+
+void FileGenRuleWithDeps(
+ const std::vector<BuildMaps::Target::ConfiguredTarget>& dependency_keys,
+ const std::vector<AnalysedTargetPtr const*>& dependency_values,
+ const BuildMaps::Base::FieldReader::Ptr& desc,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) {
+ // Associate keys and values
+ std::unordered_map<BuildMaps::Target::ConfiguredTarget, AnalysedTargetPtr>
+ deps_by_transition;
+ deps_by_transition.reserve(dependency_keys.size());
+ for (size_t i = 0; i < dependency_keys.size(); ++i) {
+ deps_by_transition.emplace(dependency_keys[i], *dependency_values[i]);
+ }
+
+ auto param_vars = desc->ReadStringList("arguments_config");
+ if (not param_vars) {
+ return;
+ }
+ auto param_config = key.config.Prune(*param_vars);
+
+ auto string_fields_fcts =
+ FunctionMap::MakePtr(FunctionMap::underlying_map_t{
+ {"outs",
+ [&deps_by_transition, &key](
+ auto&& eval, auto const& expr, auto const& env) {
+ return BuildMaps::Target::Utils::keys_expr(
+ BuildMaps::Target::Utils::obtainTargetByName(
+ eval, expr, env, key.target, deps_by_transition)
+ ->Artifacts());
+ }},
+ {"runfiles",
+ [&deps_by_transition, &key](
+ auto&& eval, auto const& expr, auto const& env) {
+ return BuildMaps::Target::Utils::keys_expr(
+ BuildMaps::Target::Utils::obtainTargetByName(
+ eval, expr, env, key.target, deps_by_transition)
+ ->RunFiles());
+ }}});
+
+ auto tainted = std::set<std::string>{};
+ auto got_tainted = BuildMaps::Target::Utils::getTainted(
+ &tainted,
+ param_config,
+ desc->ReadOptionalExpression("tainted", Expression::kEmptyList),
+ logger);
+ if (not got_tainted) {
+ return;
+ }
+ for (auto const& dep : dependency_values) {
+ if (not std::includes(tainted.begin(),
+ tainted.end(),
+ (*dep)->Tainted().begin(),
+ (*dep)->Tainted().end())) {
+ (*logger)(
+ "Not tainted with all strings the dependencies are tainted "
+ "with",
+ true);
+ return;
+ }
+ }
+
+ auto file_name_exp = desc->ReadOptionalExpression(
+ "name", ExpressionPtr{std::string{"out.txt"}});
+ if (not file_name_exp) {
+ return;
+ }
+ auto file_name_val = file_name_exp.Evaluate(
+ param_config, string_fields_fcts, [logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating name:\n{}", msg), true);
+ });
+ if (not file_name_val) {
+ return;
+ }
+ if (not file_name_val->IsString()) {
+ (*logger)(fmt::format("name should evaluate to a string, but got {}",
+ file_name_val->ToString()),
+ true);
+ return;
+ }
+ auto data_exp =
+ desc->ReadOptionalExpression("data", ExpressionPtr{std::string{""}});
+ if (not data_exp) {
+ return;
+ }
+ auto data_val = data_exp.Evaluate(
+ param_config, string_fields_fcts, [logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating data:\n{}", msg), true);
+ });
+ if (not data_val) {
+ return;
+ }
+ if (not data_val->IsString()) {
+ (*logger)(fmt::format("data should evaluate to a string, but got {}",
+ data_val->ToString()),
+ true);
+ return;
+ }
+ auto stage = ExpressionPtr{Expression::map_t{
+ file_name_val->String(),
+ ExpressionPtr{ArtifactDescription{
+ {ComputeHash(data_val->String()), data_val->String().size()},
+ ObjectType::File}}}};
+
+ auto vars_set = std::unordered_set<std::string>{};
+ vars_set.insert(param_vars->begin(), param_vars->end());
+ auto analysis_result = std::make_shared<AnalysedTarget>(
+ TargetResult{stage, ExpressionPtr{Expression::map_t{}}, stage},
+ std::vector<ActionDescription>{},
+ std::vector<std::string>{data_val->String()},
+ std::vector<Tree>{},
+ std::move(vars_set),
+ std::move(tainted));
+ analysis_result =
+ result_map->Add(key.target, param_config, std::move(analysis_result));
+ (*setter)(std::move(analysis_result));
+}
+
+void FileGenRule(
+ const nlohmann::json& desc_json,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) {
+ auto desc = BuildMaps::Base::FieldReader::CreatePtr(
+ desc_json, key.target, "file-generation target", logger);
+ desc->ExpectFields(fileGenRuleFields);
+ auto param_vars = desc->ReadStringList("arguments_config");
+ if (not param_vars) {
+ return;
+ }
+ auto param_config = key.config.Prune(*param_vars);
+
+ // Collect dependencies: deps
+ auto const& empty_list = Expression::kEmptyList;
+ auto deps_exp = desc->ReadOptionalExpression("deps", empty_list);
+ if (not deps_exp) {
+ return;
+ }
+ auto deps_value =
+ deps_exp.Evaluate(param_config, {}, [&logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating deps:\n{}", msg), true);
+ });
+ if (not deps_value) {
+ return;
+ }
+ if (not deps_value->IsList()) {
+ (*logger)(fmt::format("Expected deps to evaluate to a list of targets, "
+ "but found {}",
+ deps_value->ToString()),
+ true);
+ return;
+ }
+ std::vector<BuildMaps::Target::ConfiguredTarget> dependency_keys;
+ for (auto const& dep_name : deps_value->List()) {
+ auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression(
+ dep_name,
+ key.target,
+ [&logger, &dep_name](std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing dep entry {} failed with:\n{}",
+ dep_name->ToString(),
+ parse_err),
+ true);
+ });
+ if (not dep_target) {
+ return;
+ }
+ dependency_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{*dep_target, key.config});
+ }
+ (*subcaller)(
+ dependency_keys,
+ [dependency_keys, desc, setter, logger, key, result_map](
+ auto const& values) {
+ FileGenRuleWithDeps(
+ dependency_keys, values, desc, key, setter, logger, result_map);
+ },
+ logger);
+}
+
+void InstallRuleWithDeps(
+ const std::vector<BuildMaps::Target::ConfiguredTarget>& dependency_keys,
+ const std::vector<AnalysedTargetPtr const*>& dependency_values,
+ const BuildMaps::Base::FieldReader::Ptr& desc,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const std::vector<BuildMaps::Base::EntityName>& deps,
+ const std::unordered_map<std::string, BuildMaps::Base::EntityName>& files,
+ const std::vector<std::pair<BuildMaps::Base::EntityName, std::string>>&
+ dirs,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) {
+ // Associate keys and values
+ std::unordered_map<BuildMaps::Base::EntityName, AnalysedTargetPtr>
+ deps_by_target;
+ deps_by_target.reserve(dependency_keys.size());
+ for (size_t i = 0; i < dependency_keys.size(); ++i) {
+ deps_by_target.emplace(dependency_keys[i].target,
+ *dependency_values[i]);
+ }
+
+ // Compute the effective dependecy on config variables
+ std::unordered_set<std::string> effective_vars;
+ auto param_vars = desc->ReadStringList("arguments_config");
+ effective_vars.insert(param_vars->begin(), param_vars->end());
+ for (auto const& [target_name, target] : deps_by_target) {
+ effective_vars.insert(target->Vars().begin(), target->Vars().end());
+ }
+ auto effective_conf = key.config.Prune(effective_vars);
+
+ // Compute and verify taintedness
+ auto tainted = std::set<std::string>{};
+ auto got_tainted = BuildMaps::Target::Utils::getTainted(
+ &tainted,
+ key.config.Prune(*param_vars),
+ desc->ReadOptionalExpression("tainted", Expression::kEmptyList),
+ logger);
+ if (not got_tainted) {
+ return;
+ }
+ for (auto const& dep : dependency_values) {
+ if (not std::includes(tainted.begin(),
+ tainted.end(),
+ (*dep)->Tainted().begin(),
+ (*dep)->Tainted().end())) {
+ (*logger)(
+ "Not tainted with all strings the dependencies are tainted "
+ "with",
+ true);
+ return;
+ }
+ }
+
+ // Stage deps (runfiles only)
+ auto stage = ExpressionPtr{Expression::map_t{}};
+ for (auto const& dep : deps) {
+ auto to_stage = deps_by_target.at(dep)->RunFiles();
+ auto dup = stage->Map().FindConflictingDuplicate(to_stage->Map());
+ if (dup) {
+ (*logger)(fmt::format("Staging conflict for path {}", dup->get()),
+ true);
+ return;
+ }
+ stage = ExpressionPtr{Expression::map_t{stage, to_stage}};
+ }
+
+ // stage files (artifacts, but fall back to runfiles)
+ auto files_stage = Expression::map_t::underlying_map_t{};
+ for (auto const& [path, target] : files) {
+ if (stage->Map().contains(path)) {
+ (*logger)(fmt::format("Staging conflict for path {}", path), true);
+ return;
+ }
+ auto artifacts = deps_by_target[target]->Artifacts();
+ if (artifacts->Map().empty()) {
+ // If no artifacts are present, fall back to runfiles
+ artifacts = deps_by_target[target]->RunFiles();
+ }
+ if (artifacts->Map().empty()) {
+ (*logger)(fmt::format(
+ "No artifacts or runfiles for {} to be staged to {}",
+ target.ToString(),
+ path),
+ true);
+ return;
+ }
+ if (artifacts->Map().size() != 1) {
+ (*logger)(
+ fmt::format("Not precisely one entry for {} to be staged to {}",
+ target.ToString(),
+ path),
+ true);
+ return;
+ }
+ files_stage.emplace(path, artifacts->Map().Values()[0]);
+ }
+ stage = ExpressionPtr{Expression::map_t{stage, files_stage}};
+
+ // stage dirs (artifacts and runfiles)
+ for (auto const& subdir : dirs) {
+ auto subdir_stage = Expression::map_t::underlying_map_t{};
+ auto dir_path = std::filesystem::path{subdir.second};
+ auto target = deps_by_target.at(subdir.first);
+ // within a target, artifacts and runfiles may overlap, but artifacts
+ // take perference
+ for (auto const& [path, artifact] : target->Artifacts()->Map()) {
+ subdir_stage.emplace((dir_path / path).string(), artifact);
+ }
+ for (auto const& [path, artifact] : target->RunFiles()->Map()) {
+ subdir_stage.emplace((dir_path / path).string(), artifact);
+ }
+ auto to_stage = ExpressionPtr{Expression::map_t{subdir_stage}};
+ auto dup = stage->Map().FindConflictingDuplicate(to_stage->Map());
+ if (dup) {
+ (*logger)(fmt::format("Staging conflict for path {}", dup->get()),
+ true);
+ return;
+ }
+ stage = ExpressionPtr{Expression::map_t{stage, to_stage}};
+ }
+
+ auto const& empty_map = Expression::kEmptyMap;
+ auto result =
+ std::make_shared<AnalysedTarget>(TargetResult{stage, empty_map, stage},
+ std::vector<ActionDescription>{},
+ std::vector<std::string>{},
+ std::vector<Tree>{},
+ std::move(effective_vars),
+ std::move(tainted));
+
+ result = result_map->Add(key.target, effective_conf, std::move(result));
+ (*setter)(std::move(result));
+}
+
+void InstallRule(
+ const nlohmann::json& desc_json,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) {
+ auto desc = BuildMaps::Base::FieldReader::CreatePtr(
+ desc_json, key.target, "install target", logger);
+ desc->ExpectFields(installRuleFields);
+ auto param_vars = desc->ReadStringList("arguments_config");
+ if (not param_vars) {
+ return;
+ }
+ auto param_config = key.config.Prune(*param_vars);
+
+ // Collect dependencies: deps
+ auto const& empty_list = Expression::kEmptyList;
+ auto deps_exp = desc->ReadOptionalExpression("deps", empty_list);
+ if (not deps_exp) {
+ return;
+ }
+ auto deps_value =
+ deps_exp.Evaluate(param_config, {}, [&logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating deps:\n{}", msg), true);
+ });
+ if (not deps_value) {
+ return;
+ }
+ if (not deps_value->IsList()) {
+ (*logger)(fmt::format("Expected deps to evaluate to a list of targets, "
+ "but found {}",
+ deps_value->ToString()),
+ true);
+ return;
+ }
+ std::vector<BuildMaps::Target::ConfiguredTarget> dependency_keys;
+ std::vector<BuildMaps::Base::EntityName> deps;
+ deps.reserve(deps_value->List().size());
+ for (auto const& dep_name : deps_value->List()) {
+ auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression(
+ dep_name,
+ key.target,
+ [&logger, &dep_name](std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing dep entry {} failed with:\n{}",
+ dep_name->ToString(),
+ parse_err),
+ true);
+ });
+ if (not dep_target) {
+ return;
+ }
+ dependency_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{*dep_target, key.config});
+ deps.emplace_back(*dep_target);
+ }
+
+ // Collect dependencies: files
+ auto const& empty_map = Expression::kEmptyMap;
+ auto files_exp = desc->ReadOptionalExpression("files", empty_map);
+ if (not files_exp) {
+ return;
+ }
+ if (not files_exp->IsMap()) {
+ (*logger)(fmt::format("Expected files to be a map of target "
+ "expressions, but found {}",
+ files_exp->ToString()),
+ true);
+ return;
+ }
+ auto files = std::unordered_map<std::string, BuildMaps::Base::EntityName>{};
+ files.reserve(files_exp->Map().size());
+ for (auto const& [path, dep_exp] : files_exp->Map()) {
+ std::string path_ = path; // Have a variable to capture
+ auto dep_name = dep_exp.Evaluate(
+ param_config, {}, [&logger, &path_](auto const& msg) {
+ (*logger)(
+ fmt::format(
+ "While evaluating files entry for {}:\n{}", path_, msg),
+ true);
+ });
+ if (not dep_name) {
+ return;
+ }
+ auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression(
+ dep_name,
+ key.target,
+ [&logger, &dep_name, &path = path](std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing file entry {} for key {} failed "
+ "with:\n{}",
+ dep_name->ToString(),
+ path,
+ parse_err),
+ true);
+ });
+ if (not dep_target) {
+ return;
+ }
+ dependency_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{*dep_target, key.config});
+ files.emplace(path, *dep_target);
+ }
+
+ // Collect dependencies: dirs
+ auto dirs_exp = desc->ReadOptionalExpression("dirs", empty_list);
+ if (not dirs_exp) {
+ return;
+ }
+ auto dirs_value =
+ dirs_exp.Evaluate(param_config, {}, [&logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating deps:\n{}", msg), true);
+ });
+ if (not dirs_value) {
+ return;
+ }
+ if (not dirs_value->IsList()) {
+ (*logger)(fmt::format("Expected dirs to evaluate to a list of "
+ "path-target pairs, but found {}",
+ dirs_value->ToString()),
+ true);
+ return;
+ }
+ auto dirs =
+ std::vector<std::pair<BuildMaps::Base::EntityName, std::string>>{};
+ dirs.reserve(dirs_value->List().size());
+ for (auto const& entry : dirs_value->List()) {
+ if (not(entry->IsList() and entry->List().size() == 2 and
+ entry->List()[1]->IsString())) {
+ (*logger)(fmt::format("Expected dirs to evaluate to a list of "
+ "target-path pairs, but found entry {}",
+ entry->ToString()),
+ true);
+ return;
+ }
+ auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression(
+ entry->List()[0],
+ key.target,
+ [&logger, &entry](std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing dir entry {} for path {} failed "
+ "with:\n{}",
+ entry->List()[0]->ToString(),
+ entry->List()[1]->String(),
+ parse_err),
+ true);
+ });
+ if (not dep_target) {
+ return;
+ }
+ dependency_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{*dep_target, key.config});
+ dirs.emplace_back(std::pair<BuildMaps::Base::EntityName, std::string>{
+ *dep_target, entry->List()[1]->String()});
+ }
+
+ (*subcaller)(
+ dependency_keys,
+ [dependency_keys,
+ deps = std::move(deps),
+ files = std::move(files),
+ dirs = std::move(dirs),
+ desc,
+ setter,
+ logger,
+ key,
+ result_map](auto const& values) {
+ InstallRuleWithDeps(dependency_keys,
+ values,
+ desc,
+ key,
+ deps,
+ files,
+ dirs,
+ setter,
+ logger,
+ result_map);
+ },
+ logger);
+}
+
+void GenericRuleWithDeps(
+ const std::vector<BuildMaps::Target::ConfiguredTarget>& transition_keys,
+ const std::vector<AnalysedTargetPtr const*>& dependency_values,
+ const BuildMaps::Base::FieldReader::Ptr& desc,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) {
+ // Associate dependency keys with values
+ std::unordered_map<BuildMaps::Target::ConfiguredTarget, AnalysedTargetPtr>
+ deps_by_transition;
+ deps_by_transition.reserve(transition_keys.size());
+ for (size_t i = 0; i < transition_keys.size(); ++i) {
+ deps_by_transition.emplace(transition_keys[i], *dependency_values[i]);
+ }
+
+ // Compute the effective dependecy on config variables
+ std::unordered_set<std::string> effective_vars;
+ auto param_vars = desc->ReadStringList("arguments_config");
+ effective_vars.insert(param_vars->begin(), param_vars->end());
+ for (auto const& [transition, target] : deps_by_transition) {
+ effective_vars.insert(target->Vars().begin(), target->Vars().end());
+ }
+ auto effective_conf = key.config.Prune(effective_vars);
+
+ // Compute and verify taintedness
+ auto tainted = std::set<std::string>{};
+ auto got_tainted = BuildMaps::Target::Utils::getTainted(
+ &tainted,
+ key.config.Prune(*param_vars),
+ desc->ReadOptionalExpression("tainted", Expression::kEmptyList),
+ logger);
+ if (not got_tainted) {
+ return;
+ }
+ for (auto const& dep : dependency_values) {
+ if (not std::includes(tainted.begin(),
+ tainted.end(),
+ (*dep)->Tainted().begin(),
+ (*dep)->Tainted().end())) {
+ (*logger)(
+ "Not tainted with all strings the dependencies are tainted "
+ "with",
+ true);
+ return;
+ }
+ }
+
+ // Evaluate cmd, outs, env
+ auto string_fields_fcts =
+ FunctionMap::MakePtr(FunctionMap::underlying_map_t{
+ {"outs",
+ [&deps_by_transition, &key](
+ auto&& eval, auto const& expr, auto const& env) {
+ return BuildMaps::Target::Utils::keys_expr(
+ BuildMaps::Target::Utils::obtainTargetByName(
+ eval, expr, env, key.target, deps_by_transition)
+ ->Artifacts());
+ }},
+ {"runfiles",
+ [&deps_by_transition, &key](
+ auto&& eval, auto const& expr, auto const& env) {
+ return BuildMaps::Target::Utils::keys_expr(
+ BuildMaps::Target::Utils::obtainTargetByName(
+ eval, expr, env, key.target, deps_by_transition)
+ ->RunFiles());
+ }}});
+ auto const& empty_list = Expression::kEmptyList;
+ auto param_config = key.config.Prune(*param_vars);
+ auto outs_exp = desc->ReadOptionalExpression("outs", empty_list);
+ if (not outs_exp) {
+ return;
+ }
+ auto outs_value = outs_exp.Evaluate(
+ param_config, string_fields_fcts, [&logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating outs:\n{}", msg), true);
+ });
+ if (not outs_value) {
+ return;
+ }
+ if ((not outs_value->IsList()) or outs_value->List().empty()) {
+ (*logger)(fmt::format("outs has to evaluate to a non-empty list of "
+ "strings, but found {}",
+ outs_value->ToString()),
+ true);
+ return;
+ }
+ std::vector<std::string> outs{};
+ outs.reserve(outs_value->List().size());
+ for (auto const& x : outs_value->List()) {
+ if (not x->IsString()) {
+ (*logger)(fmt::format("outs has to evaluate to a non-empty list of "
+ "strings, but found entry {}",
+ x->ToString()),
+ true);
+ return;
+ }
+ outs.emplace_back(x->String());
+ }
+ auto cmd_exp = desc->ReadOptionalExpression("cmds", empty_list);
+ if (not cmd_exp) {
+ return;
+ }
+ auto cmd_value = cmd_exp.Evaluate(
+ param_config, string_fields_fcts, [&logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating cmds:\n{}", msg), true);
+ });
+ if (not cmd_value) {
+ return;
+ }
+ if (not cmd_value->IsList()) {
+ (*logger)(fmt::format(
+ "cmds has to evaluate to a list of strings, but found {}",
+ cmd_value->ToString()),
+ true);
+ return;
+ }
+ std::stringstream cmd_ss{};
+ for (auto const& x : cmd_value->List()) {
+ if (not x->IsString()) {
+ (*logger)(fmt::format("cmds has to evaluate to a list of strings, "
+ "but found entry {}",
+ x->ToString()),
+ true);
+ return;
+ }
+ cmd_ss << x->String();
+ cmd_ss << "\n";
+ }
+ auto const& empty_map_exp = Expression::kEmptyMapExpr;
+ auto env_exp = desc->ReadOptionalExpression("env", empty_map_exp);
+ if (not env_exp) {
+ return;
+ }
+ auto env_val = env_exp.Evaluate(
+ param_config, string_fields_fcts, [&logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating env:\n{}", msg), true);
+ });
+ if (not env_val) {
+ return;
+ }
+ if (not env_val->IsMap()) {
+ (*logger)(
+ fmt::format("cmds has to evaluate to map of strings, but found {}",
+ env_val->ToString()),
+ true);
+ }
+ for (auto const& [var_name, x] : env_val->Map()) {
+ if (not x->IsString()) {
+ (*logger)(fmt::format("env has to evaluate to map of strings, but "
+ "found entry {}",
+ x->ToString()),
+ true);
+ }
+ }
+
+ // Construct inputs; in case of conflicts, artifacts take precedence
+ // over runfiles.
+ auto inputs = ExpressionPtr{Expression::map_t{}};
+ for (auto const& dep : dependency_values) {
+ inputs = ExpressionPtr{Expression::map_t{inputs, (*dep)->RunFiles()}};
+ }
+ for (auto const& dep : dependency_values) {
+ inputs = ExpressionPtr{Expression::map_t{inputs, (*dep)->Artifacts()}};
+ }
+
+ // Construct our single action, and its artifacts
+ auto action =
+ BuildMaps::Target::Utils::createAction(outs,
+ {},
+ {"sh", "-c", cmd_ss.str()},
+ env_val,
+ std::nullopt,
+ false,
+ inputs);
+ auto action_identifier = action.Id();
+ Expression::map_t::underlying_map_t artifacts;
+ for (auto const& path : outs) {
+ artifacts.emplace(path,
+ ExpressionPtr{ArtifactDescription{
+ action_identifier, std::filesystem::path{path}}});
+ }
+
+ auto const& empty_map = Expression::kEmptyMap;
+ auto result = std::make_shared<AnalysedTarget>(
+ TargetResult{
+ ExpressionPtr{Expression::map_t{artifacts}}, empty_map, empty_map},
+ std::vector<ActionDescription>{action},
+ std::vector<std::string>{},
+ std::vector<Tree>{},
+ std::move(effective_vars),
+ std::move(tainted));
+
+ result = result_map->Add(key.target, effective_conf, std::move(result));
+ (*setter)(std::move(result));
+}
+
+void GenericRule(
+ const nlohmann::json& desc_json,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*> result_map) {
+ auto desc = BuildMaps::Base::FieldReader::CreatePtr(
+ desc_json, key.target, "generic target", logger);
+ desc->ExpectFields(genericRuleFields);
+ auto param_vars = desc->ReadStringList("arguments_config");
+ if (not param_vars) {
+ return;
+ }
+ auto param_config = key.config.Prune(*param_vars);
+ auto const& empty_list = Expression::kEmptyList;
+ auto deps_exp = desc->ReadOptionalExpression("deps", empty_list);
+ if (not deps_exp) {
+ return;
+ }
+ auto deps_value =
+ deps_exp.Evaluate(param_config, {}, [&logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating deps:\n{}", msg), true);
+ });
+ if (not deps_value->IsList()) {
+ (*logger)(fmt::format("Expected deps to evaluate to a list of targets, "
+ "but found {}",
+ deps_value->ToString()),
+ true);
+ return;
+ }
+ std::vector<BuildMaps::Target::ConfiguredTarget> dependency_keys;
+ std::vector<BuildMaps::Target::ConfiguredTarget> transition_keys;
+ dependency_keys.reserve(deps_value->List().size());
+ transition_keys.reserve(deps_value->List().size());
+ auto empty_transition = Configuration{Expression::kEmptyMap};
+ for (auto const& dep_name : deps_value->List()) {
+ auto dep_target = BuildMaps::Base::ParseEntityNameFromExpression(
+ dep_name,
+ key.target,
+ [&logger, &dep_name](std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing dep entry {} failed with:\n{}",
+ dep_name->ToString(),
+ parse_err),
+ true);
+ });
+ if (not dep_target) {
+ return;
+ }
+ dependency_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{*dep_target, key.config});
+ transition_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{*dep_target, empty_transition});
+ }
+ (*subcaller)(
+ dependency_keys,
+ [transition_keys = std::move(transition_keys),
+ desc,
+ setter,
+ logger,
+ key,
+ result_map](auto const& values) {
+ GenericRuleWithDeps(
+ transition_keys, values, desc, key, setter, logger, result_map);
+ },
+ logger);
+}
+
+auto built_ins = std::unordered_map<
+ std::string,
+ std::function<void(
+ const nlohmann::json&,
+ const BuildMaps::Target::ConfiguredTarget&,
+ const BuildMaps::Target::TargetMap::SubCallerPtr&,
+ const BuildMaps::Target::TargetMap::SetterPtr&,
+ const BuildMaps::Target::TargetMap::LoggerPtr&,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>)>>{
+ {"export", ExportRule},
+ {"file_gen", FileGenRule},
+ {"generic", GenericRule},
+ {"install", InstallRule}};
+} // namespace
+
+namespace BuildMaps::Target {
+
+auto IsBuiltInRule(nlohmann::json const& rule_type) -> bool {
+
+ if (not rule_type.is_string()) {
+ // Names for built-in rules are always strings
+ return false;
+ }
+ auto rule_name = rule_type.get<std::string>();
+ return built_ins.contains(rule_name);
+}
+
+auto HandleBuiltin(
+ const nlohmann::json& rule_type,
+ const nlohmann::json& desc,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map)
+ -> bool {
+ if (not rule_type.is_string()) {
+ // Names for built-in rules are always strings
+ return false;
+ }
+ auto rule_name = rule_type.get<std::string>();
+ auto it = built_ins.find(rule_name);
+ if (it == built_ins.end()) {
+ return false;
+ }
+ auto target_logger = std::make_shared<BuildMaps::Target::TargetMap::Logger>(
+ [logger, rule_name, key](auto msg, auto fatal) {
+ (*logger)(fmt::format("While evaluating {} target {}:\n{}",
+ rule_name,
+ key.target.ToString(),
+ msg),
+ fatal);
+ });
+ (it->second)(desc, key, subcaller, setter, target_logger, result_map);
+ return true;
+}
+} // namespace BuildMaps::Target
diff --git a/src/buildtool/build_engine/target_map/built_in_rules.hpp b/src/buildtool/build_engine/target_map/built_in_rules.hpp
new file mode 100644
index 00000000..0e56e38d
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/built_in_rules.hpp
@@ -0,0 +1,21 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_BUILT_IN_RULES_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_BUILT_IN_RULES_HPP
+
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/target_map/configured_target.hpp"
+#include "src/buildtool/build_engine/target_map/result_map.hpp"
+#include "src/buildtool/build_engine/target_map/target_map.hpp"
+
+namespace BuildMaps::Target {
+auto HandleBuiltin(
+ const nlohmann::json& rule_type,
+ const nlohmann::json& desc,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map)
+ -> bool;
+} // namespace BuildMaps::Target
+#endif
diff --git a/src/buildtool/build_engine/target_map/configured_target.hpp b/src/buildtool/build_engine/target_map/configured_target.hpp
new file mode 100644
index 00000000..b0443cbd
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/configured_target.hpp
@@ -0,0 +1,41 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_CONFIGURED_TARGET_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_CONFIGURED_TARGET_HPP
+
+#include "fmt/core.h"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+
+namespace BuildMaps::Target {
+
+struct ConfiguredTarget {
+ BuildMaps::Base::EntityName target;
+ Configuration config;
+
+ [[nodiscard]] auto operator==(
+ BuildMaps::Target::ConfiguredTarget const& other) const noexcept
+ -> bool {
+ return target == other.target && config == other.config;
+ }
+
+ [[nodiscard]] auto ToString() const noexcept -> std::string {
+ return fmt::format("[{},{}]", target.ToString(), config.ToString());
+ }
+};
+
+} // namespace BuildMaps::Target
+
+namespace std {
+template <>
+struct hash<BuildMaps::Target::ConfiguredTarget> {
+ [[nodiscard]] auto operator()(BuildMaps::Target::ConfiguredTarget const& ct)
+ const noexcept -> std::size_t {
+ size_t seed{};
+ hash_combine<>(&seed, ct.target);
+ hash_combine<>(&seed, ct.config);
+ return seed;
+ }
+};
+} // namespace std
+
+#endif
diff --git a/src/buildtool/build_engine/target_map/export.cpp b/src/buildtool/build_engine/target_map/export.cpp
new file mode 100644
index 00000000..cea76d36
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/export.cpp
@@ -0,0 +1,126 @@
+#include "src/buildtool/build_engine/target_map/export.hpp"
+
+#include <unordered_set>
+
+#include "src/buildtool/build_engine/base_maps/field_reader.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+
+namespace {
+auto expectedFields = std::unordered_set<std::string>{"config_doc",
+ "doc",
+ "fixed_config",
+ "flexible_config",
+ "target",
+ "type"};
+
+void FinalizeExport(
+ const std::vector<AnalysedTargetPtr const*>& exported,
+ const BuildMaps::Base::EntityName& target,
+ const std::vector<std::string>& vars,
+ const Configuration& effective_config,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) {
+ const auto* value = exported[0];
+ if (not(*value)->Tainted().empty()) {
+ (*logger)("Only untainted targets can be exported.", true);
+ return;
+ }
+ auto provides = (*value)->Provides();
+ if (not provides->IsCacheable()) {
+ (*logger)(fmt::format("Only cacheable values can be exported; but "
+ "target provides {}",
+ provides->ToString()),
+ true);
+ return;
+ }
+ std::unordered_set<std::string> vars_set{};
+ vars_set.insert(vars.begin(), vars.end());
+ // TODO(aehlig): wrap all artifacts into "save to target-cache" special
+ // action
+ auto analysis_result = std::make_shared<AnalysedTarget>(
+ TargetResult{(*value)->Artifacts(), provides, (*value)->RunFiles()},
+ std::vector<ActionDescription>{},
+ std::vector<std::string>{},
+ std::vector<Tree>{},
+ std::move(vars_set),
+ std::set<std::string>{});
+ analysis_result =
+ result_map->Add(target, effective_config, std::move(analysis_result));
+ (*setter)(std::move(analysis_result));
+}
+} // namespace
+
+void ExportRule(
+ const nlohmann::json& desc_json,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*> result_map) {
+ auto desc = BuildMaps::Base::FieldReader::CreatePtr(
+ desc_json, key.target, "export target", logger);
+ desc->ExpectFields(expectedFields);
+ auto exported_target_name = desc->ReadExpression("target");
+ if (not exported_target_name) {
+ return;
+ }
+ auto exported_target = BuildMaps::Base::ParseEntityNameFromExpression(
+ exported_target_name,
+ key.target,
+ [&logger, &exported_target_name](std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing target name {} failed with:\n{}",
+ exported_target_name->ToString(),
+ parse_err),
+ true);
+ });
+ if (not exported_target) {
+ return;
+ }
+ auto flexible_vars = desc->ReadStringList("flexible_config");
+ if (not flexible_vars) {
+ return;
+ }
+ auto effective_config = key.config.Prune(*flexible_vars);
+
+ // TODO(aehlig): if the respository is content-fixed, look up in target
+ // cache with key consistig of repository-description, target, and effective
+ // config.
+
+ auto fixed_config =
+ desc->ReadOptionalExpression("fixed_config", Expression::kEmptyMap);
+ if (not fixed_config->IsMap()) {
+ (*logger)(fmt::format("fixed_config has to be a map, but found {}",
+ fixed_config->ToString()),
+ true);
+ return;
+ }
+ for (auto const& var : fixed_config->Map().Keys()) {
+ if (effective_config.VariableFixed(var)) {
+ (*logger)(
+ fmt::format("Variable {} is both fixed and flexible.", var),
+ true);
+ return;
+ }
+ }
+ auto target_config = effective_config.Update(fixed_config);
+
+ (*subcaller)(
+ {BuildMaps::Target::ConfiguredTarget{std::move(*exported_target),
+ std::move(target_config)}},
+ [setter,
+ logger,
+ vars = std::move(*flexible_vars),
+ result_map,
+ effective_config = std::move(effective_config),
+ target = key.target](auto const& values) {
+ FinalizeExport(values,
+ target,
+ vars,
+ effective_config,
+ logger,
+ setter,
+ result_map);
+ },
+ logger);
+}
diff --git a/src/buildtool/build_engine/target_map/export.hpp b/src/buildtool/build_engine/target_map/export.hpp
new file mode 100644
index 00000000..8b2e17f2
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/export.hpp
@@ -0,0 +1,17 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_EXPORT_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_EXPORT_HPP
+
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/target_map/configured_target.hpp"
+#include "src/buildtool/build_engine/target_map/result_map.hpp"
+#include "src/buildtool/build_engine/target_map/target_map.hpp"
+
+void ExportRule(const nlohmann::json& desc_json,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ gsl::not_null<BuildMaps::Target::ResultTargetMap*> result_map);
+
+#endif
diff --git a/src/buildtool/build_engine/target_map/result_map.hpp b/src/buildtool/build_engine/target_map/result_map.hpp
new file mode 100644
index 00000000..b5825ca4
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/result_map.hpp
@@ -0,0 +1,291 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_RESULT_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_RESULT_MAP_HPP
+
+#include <algorithm>
+#include <fstream>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/build_engine/target_map/configured_target.hpp"
+#include "src/buildtool/common/tree.hpp"
+#include "src/buildtool/multithreading/task.hpp"
+#include "src/buildtool/multithreading/task_system.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+
+namespace BuildMaps::Target {
+
+// Class collecting analysed targets for their canonical configuration.
+class ResultTargetMap {
+ public:
+ struct ActionWithOrigin {
+ ActionDescription desc;
+ nlohmann::json origin;
+ };
+
+ template <bool kIncludeOrigins = false>
+ struct ResultType {
+ std::vector<ActionDescription> actions{};
+ std::vector<std::string> blobs{};
+ std::vector<Tree> trees{};
+ };
+
+ template <>
+ struct ResultType</*kIncludeOrigins=*/true> {
+ std::vector<ActionWithOrigin> actions{};
+ std::vector<std::string> blobs{};
+ std::vector<Tree> trees{};
+ };
+
+ explicit ResultTargetMap(std::size_t jobs) : width_{ComputeWidth(jobs)} {}
+
+ ResultTargetMap() = default;
+
+ // \brief Add the analysed target for the given target and
+ // configuration, if no entry is present for the given
+ // target-configuration pair. \returns the analysed target that is
+ // element of the map after insertion.
+ [[nodiscard]] auto Add(BuildMaps::Base::EntityName name,
+ Configuration conf,
+ gsl::not_null<AnalysedTargetPtr> result)
+ -> AnalysedTargetPtr {
+ auto part = std::hash<BuildMaps::Base::EntityName>{}(name) % width_;
+ std::unique_lock lock{m_[part]};
+ auto [entry, inserted] = targets_[part].emplace(
+ ConfiguredTarget{std::move(name), std::move(conf)},
+ std::move(result));
+ if (inserted) {
+ num_actions_[part] += entry->second->Actions().size();
+ num_blobs_[part] += entry->second->Blobs().size();
+ num_trees_[part] += entry->second->Trees().size();
+ }
+ return entry->second;
+ }
+
+ [[nodiscard]] auto ConfiguredTargets() const noexcept
+ -> std::vector<ConfiguredTarget> {
+ std::vector<ConfiguredTarget> targets{};
+ size_t s = 0;
+ for (const auto& target : targets_) {
+ s += target.size();
+ }
+ targets.reserve(s);
+ for (const auto& i : targets_) {
+ std::transform(i.begin(),
+ i.end(),
+ std::back_inserter(targets),
+ [](auto const& target) { return target.first; });
+ }
+ std::sort(targets.begin(),
+ targets.end(),
+ [](auto const& lhs, auto const& rhs) {
+ return lhs.ToString() < rhs.ToString();
+ });
+ return targets;
+ }
+
+ template <bool kIncludeOrigins = false>
+ [[nodiscard]] auto ToResult() const -> ResultType<kIncludeOrigins> {
+ ResultType<kIncludeOrigins> result{};
+ size_t na = 0;
+ size_t nb = 0;
+ size_t nt = 0;
+ for (std::size_t i = 0; i < width_; i++) {
+ na += num_actions_[i];
+ nb += num_blobs_[i];
+ nt += num_trees_[i];
+ }
+ result.actions.reserve(na);
+ result.blobs.reserve(nb);
+ result.trees.reserve(nt);
+
+ std::unordered_map<
+ std::string,
+ std::vector<std::pair<ConfiguredTarget, std::size_t>>>
+ origin_map;
+ origin_map.reserve(na);
+ if constexpr (kIncludeOrigins) {
+ for (const auto& target : targets_) {
+ std::for_each(
+ target.begin(), target.end(), [&](auto const& el) {
+ auto const& actions = el.second->Actions();
+ std::size_t pos{};
+ std::for_each(
+ actions.begin(),
+ actions.end(),
+ [&origin_map, &pos, &el](auto const& action) {
+ std::pair<ConfiguredTarget, std::size_t> origin{
+ el.first, pos++};
+ auto id = action.Id();
+ if (origin_map.contains(id)) {
+ origin_map[id].push_back(origin);
+ }
+ else {
+ origin_map[id] =
+ std::vector<std::pair<ConfiguredTarget,
+ std::size_t>>{
+ origin};
+ }
+ });
+ });
+ }
+ // Sort origins to get a reproducible order. We don't expect many
+ // origins for a single action, so the cost of comparison is not
+ // too important. Moreover, we expect most actions to have a single
+ // origin, so any precomputation would be more expensive.
+ for (auto const& i : origin_map) {
+ std::sort(origin_map[i.first].begin(),
+ origin_map[i.first].end(),
+ [](auto const& left, auto const& right) {
+ auto left_target = left.first.ToString();
+ auto right_target = right.first.ToString();
+ return (left_target < right_target) ||
+ (left_target == right_target &&
+ left.second < right.second);
+ });
+ }
+ }
+
+ for (const auto& target : targets_) {
+ std::for_each(target.begin(), target.end(), [&](auto const& el) {
+ auto const& actions = el.second->Actions();
+ if constexpr (kIncludeOrigins) {
+ std::for_each(actions.begin(),
+ actions.end(),
+ [&result, &origin_map](auto const& action) {
+ auto origins = nlohmann::json::array();
+ for (auto const& [ct, count] :
+ origin_map[action.Id()]) {
+ origins.push_back(nlohmann::json{
+ {"target", ct.target.ToJson()},
+ {"subtask", count},
+ {"config", ct.config.ToJson()}});
+ }
+ result.actions.emplace_back(
+ ActionWithOrigin{action, origins});
+ });
+ }
+ else {
+ std::for_each(actions.begin(),
+ actions.end(),
+ [&result](auto const& action) {
+ result.actions.emplace_back(action);
+ });
+ }
+ auto const& blobs = el.second->Blobs();
+ auto const& trees = el.second->Trees();
+ result.blobs.insert(
+ result.blobs.end(), blobs.begin(), blobs.end());
+ result.trees.insert(
+ result.trees.end(), trees.begin(), trees.end());
+ });
+ }
+
+ std::sort(result.blobs.begin(), result.blobs.end());
+ auto lastblob = std::unique(result.blobs.begin(), result.blobs.end());
+ result.blobs.erase(lastblob, result.blobs.end());
+
+ std::sort(result.trees.begin(),
+ result.trees.end(),
+ [](auto left, auto right) { return left.Id() < right.Id(); });
+ auto lasttree = std::unique(
+ result.trees.begin(),
+ result.trees.end(),
+ [](auto left, auto right) { return left.Id() == right.Id(); });
+ result.trees.erase(lasttree, result.trees.end());
+
+ std::sort(result.actions.begin(),
+ result.actions.end(),
+ [](auto left, auto right) {
+ if constexpr (kIncludeOrigins) {
+ return left.desc.Id() < right.desc.Id();
+ }
+ else {
+ return left.Id() < right.Id();
+ }
+ });
+ auto lastaction =
+ std::unique(result.actions.begin(),
+ result.actions.end(),
+ [](auto left, auto right) {
+ if constexpr (kIncludeOrigins) {
+ return left.desc.Id() == right.desc.Id();
+ }
+ else {
+ return left.Id() == right.Id();
+ }
+ });
+ result.actions.erase(lastaction, result.actions.end());
+
+ return result;
+ }
+
+ template <bool kIncludeOrigins = false>
+ [[nodiscard]] auto ToJson() const -> nlohmann::json {
+ auto const result = ToResult<kIncludeOrigins>();
+ auto actions = nlohmann::json::object();
+ auto trees = nlohmann::json::object();
+ std::for_each(result.actions.begin(),
+ result.actions.end(),
+ [&actions](auto const& action) {
+ if constexpr (kIncludeOrigins) {
+ auto const& id = action.desc.GraphAction().Id();
+ actions[id] = action.desc.ToJson();
+ actions[id]["origins"] = action.origin;
+ }
+ else {
+ auto const& id = action.GraphAction().Id();
+ actions[id] = action.ToJson();
+ }
+ });
+ std::for_each(
+ result.trees.begin(),
+ result.trees.end(),
+ [&trees](auto const& tree) { trees[tree.Id()] = tree.ToJson(); });
+ return nlohmann::json{
+ {"actions", actions}, {"blobs", result.blobs}, {"trees", trees}};
+ }
+
+ template <bool kIncludeOrigins = true>
+ auto ToFile(std::string const& graph_file, int indent = 2) const -> void {
+ std::ofstream os(graph_file);
+ os << std::setw(indent) << ToJson<kIncludeOrigins>() << std::endl;
+ }
+
+ void Clear(gsl::not_null<TaskSystem*> const& ts) {
+ for (std::size_t i = 0; i < width_; ++i) {
+ ts->QueueTask([i, this]() { targets_[i].clear(); });
+ }
+ }
+
+ private:
+ constexpr static std::size_t kScalingFactor = 2;
+ std::size_t width_{ComputeWidth(0)};
+ std::vector<std::mutex> m_{width_};
+ std::vector<
+ std::unordered_map<ConfiguredTarget, gsl::not_null<AnalysedTargetPtr>>>
+ targets_{width_};
+ std::vector<std::size_t> num_actions_{std::vector<std::size_t>(width_)};
+ std::vector<std::size_t> num_blobs_{std::vector<std::size_t>(width_)};
+ std::vector<std::size_t> num_trees_{std::vector<std::size_t>(width_)};
+
+ constexpr static auto ComputeWidth(std::size_t jobs) -> std::size_t {
+ if (jobs <= 0) {
+ // Non-positive indicates to use the default value
+ return ComputeWidth(
+ std::max(1U, std::thread::hardware_concurrency()));
+ }
+ return jobs * kScalingFactor + 1;
+ }
+
+}; // namespace BuildMaps::Target
+
+} // namespace BuildMaps::Target
+
+#endif // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_RESULT_MAP_HPP
diff --git a/src/buildtool/build_engine/target_map/target_map.cpp b/src/buildtool/build_engine/target_map/target_map.cpp
new file mode 100644
index 00000000..327dbe02
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_map.cpp
@@ -0,0 +1,1338 @@
+#include "src/buildtool/build_engine/target_map/target_map.hpp"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/build_engine/base_maps/field_reader.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/build_engine/expression/evaluator.hpp"
+#include "src/buildtool/build_engine/expression/function_map.hpp"
+#include "src/buildtool/build_engine/target_map/built_in_rules.hpp"
+#include "src/buildtool/build_engine/target_map/utils.hpp"
+
+namespace {
+
+using namespace std::string_literals;
+
+[[nodiscard]] auto ReadActionOutputExpr(ExpressionPtr const& out_exp,
+ std::string const& field_name)
+ -> ActionDescription::outputs_t {
+ if (not out_exp->IsList()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("{} has to be a list of strings, but found {}",
+ field_name,
+ out_exp->ToString())};
+ }
+ ActionDescription::outputs_t outputs;
+ outputs.reserve(out_exp->List().size());
+ for (auto const& out_path : out_exp->List()) {
+ if (not out_path->IsString()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("{} has to be a list of strings, but found {}",
+ field_name,
+ out_exp->ToString())};
+ }
+ outputs.emplace_back(out_path->String());
+ }
+ return outputs;
+}
+
+struct TargetData {
+ using Ptr = std::shared_ptr<TargetData>;
+
+ std::vector<std::string> target_vars;
+ std::unordered_map<std::string, ExpressionPtr> config_exprs;
+ std::unordered_map<std::string, ExpressionPtr> string_exprs;
+ std::unordered_map<std::string, ExpressionPtr> target_exprs;
+ ExpressionPtr tainted_expr;
+ bool parse_target_names{};
+
+ TargetData(std::vector<std::string> target_vars,
+ std::unordered_map<std::string, ExpressionPtr> config_exprs,
+ std::unordered_map<std::string, ExpressionPtr> string_exprs,
+ std::unordered_map<std::string, ExpressionPtr> target_exprs,
+ ExpressionPtr tainted_expr,
+ bool parse_target_names)
+ : target_vars{std::move(target_vars)},
+ config_exprs{std::move(config_exprs)},
+ string_exprs{std::move(string_exprs)},
+ target_exprs{std::move(target_exprs)},
+ tainted_expr{std::move(tainted_expr)},
+ parse_target_names{parse_target_names} {}
+
+ [[nodiscard]] static auto FromFieldReader(
+ BuildMaps::Base::UserRulePtr const& rule,
+ BuildMaps::Base::FieldReader::Ptr const& desc) -> TargetData::Ptr {
+ desc->ExpectFields(rule->ExpectedFields());
+
+ auto target_vars = desc->ReadStringList("arguments_config");
+ auto tainted_expr =
+ desc->ReadOptionalExpression("tainted", Expression::kEmptyList);
+
+ auto convert_to_exprs =
+ [&desc](gsl::not_null<
+ std::unordered_map<std::string, ExpressionPtr>*> const&
+ expr_map,
+ std::vector<std::string> const& field_names) -> bool {
+ for (auto const& field_name : field_names) {
+ auto expr = desc->ReadOptionalExpression(
+ field_name, Expression::kEmptyList);
+ if (not expr) {
+ return false;
+ }
+ expr_map->emplace(field_name, std::move(expr));
+ }
+ return true;
+ };
+
+ std::unordered_map<std::string, ExpressionPtr> config_exprs;
+ std::unordered_map<std::string, ExpressionPtr> string_exprs;
+ std::unordered_map<std::string, ExpressionPtr> target_exprs;
+ if (target_vars and tainted_expr and
+ convert_to_exprs(&config_exprs, rule->ConfigFields()) and
+ convert_to_exprs(&string_exprs, rule->StringFields()) and
+ convert_to_exprs(&target_exprs, rule->TargetFields())) {
+ return std::make_shared<TargetData>(std::move(*target_vars),
+ std::move(config_exprs),
+ std::move(string_exprs),
+ std::move(target_exprs),
+ std::move(tainted_expr),
+ /*parse_target_names=*/true);
+ }
+ return nullptr;
+ }
+
+ [[nodiscard]] static auto FromTargetNode(
+ BuildMaps::Base::UserRulePtr const& rule,
+ TargetNode::Abstract const& node,
+ ExpressionPtr const& rule_map,
+ gsl::not_null<AsyncMapConsumerLoggerPtr> const& logger)
+ -> TargetData::Ptr {
+
+ auto const& string_fields = node.string_fields->Map();
+ auto const& target_fields = node.target_fields->Map();
+
+ std::unordered_map<std::string, ExpressionPtr> config_exprs;
+ std::unordered_map<std::string, ExpressionPtr> string_exprs;
+ std::unordered_map<std::string, ExpressionPtr> target_exprs;
+
+ for (auto const& field_name : rule->ConfigFields()) {
+ if (target_fields.Find(field_name)) {
+ (*logger)(
+ fmt::format(
+ "Expected config field '{}' in string_fields of "
+ "abstract node type '{}', and not in target_fields",
+ field_name,
+ node.node_type),
+ /*fatal=*/true);
+ return nullptr;
+ }
+ auto const& config_expr =
+ string_fields.Find(field_name)
+ .value_or(std::reference_wrapper{Expression::kEmptyList})
+ .get();
+ config_exprs.emplace(field_name, config_expr);
+ }
+
+ for (auto const& field_name : rule->StringFields()) {
+ if (target_fields.Find(field_name)) {
+ (*logger)(
+ fmt::format(
+ "Expected string field '{}' in string_fields of "
+ "abstract node type '{}', and not in target_fields",
+ field_name,
+ node.node_type),
+ /*fatal=*/true);
+ return nullptr;
+ }
+ auto const& string_expr =
+ string_fields.Find(field_name)
+ .value_or(std::reference_wrapper{Expression::kEmptyList})
+ .get();
+ string_exprs.emplace(field_name, string_expr);
+ }
+
+ for (auto const& field_name : rule->TargetFields()) {
+ if (string_fields.Find(field_name)) {
+ (*logger)(
+ fmt::format(
+ "Expected target field '{}' in target_fields of "
+ "abstract node type '{}', and not in string_fields",
+ field_name,
+ node.node_type),
+ /*fatal=*/true);
+ return nullptr;
+ }
+ auto const& target_expr =
+ target_fields.Find(field_name)
+ .value_or(std::reference_wrapper{Expression::kEmptyList})
+ .get();
+ auto const& nodes = target_expr->List();
+ Expression::list_t targets{};
+ targets.reserve(nodes.size());
+ for (auto const& node_expr : nodes) {
+ targets.emplace_back(ExpressionPtr{BuildMaps::Base::EntityName{
+ BuildMaps::Base::AnonymousTarget{rule_map, node_expr}}});
+ }
+ target_exprs.emplace(field_name, targets);
+ }
+
+ return std::make_shared<TargetData>(std::vector<std::string>{},
+ std::move(config_exprs),
+ std::move(string_exprs),
+ std::move(target_exprs),
+ Expression::kEmptyList,
+ /*parse_target_names=*/false);
+ }
+};
+
+void withDependencies(
+ const std::vector<BuildMaps::Target::ConfiguredTarget>& transition_keys,
+ const std::vector<AnalysedTargetPtr const*>& dependency_values,
+ const BuildMaps::Base::UserRulePtr& rule,
+ const TargetData::Ptr& data,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ std::unordered_map<std::string, ExpressionPtr> params,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*>& result_map) {
+ // Associate dependency keys with values
+ std::unordered_map<BuildMaps::Target::ConfiguredTarget, AnalysedTargetPtr>
+ deps_by_transition;
+ deps_by_transition.reserve(transition_keys.size());
+ for (size_t i = 0; i < transition_keys.size(); ++i) {
+ deps_by_transition.emplace(transition_keys[i], *dependency_values[i]);
+ }
+
+ // Compute the effective dependecy on config variables
+ std::unordered_set<std::string> effective_vars;
+ auto const& param_vars = data->target_vars;
+ effective_vars.insert(param_vars.begin(), param_vars.end());
+ auto const& config_vars = rule->ConfigVars();
+ effective_vars.insert(config_vars.begin(), config_vars.end());
+ for (auto const& [transition, target] : deps_by_transition) {
+ for (auto const& x : target->Vars()) {
+ if (not transition.config.VariableFixed(x)) {
+ effective_vars.insert(x);
+ }
+ }
+ }
+ auto effective_conf = key.config.Prune(effective_vars);
+
+ // Compute and verify taintedness
+ auto tainted = std::set<std::string>{};
+ auto got_tainted = BuildMaps::Target::Utils::getTainted(
+ &tainted, key.config.Prune(param_vars), data->tainted_expr, logger);
+ if (not got_tainted) {
+ return;
+ }
+ tainted.insert(rule->Tainted().begin(), rule->Tainted().end());
+ for (auto const& dep : dependency_values) {
+ if (not std::includes(tainted.begin(),
+ tainted.end(),
+ (*dep)->Tainted().begin(),
+ (*dep)->Tainted().end())) {
+ (*logger)(
+ "Not tainted with all strings the dependencies are tainted "
+ "with",
+ true);
+ return;
+ }
+ }
+
+ // Evaluate string parameters
+ auto string_fields_fcts =
+ FunctionMap::MakePtr(FunctionMap::underlying_map_t{
+ {"outs",
+ [&deps_by_transition, &key](
+ auto&& eval, auto const& expr, auto const& env) {
+ return BuildMaps::Target::Utils::keys_expr(
+ BuildMaps::Target::Utils::obtainTargetByName(
+ eval, expr, env, key.target, deps_by_transition)
+ ->Artifacts());
+ }},
+ {"runfiles",
+ [&deps_by_transition, &key](
+ auto&& eval, auto const& expr, auto const& env) {
+ return BuildMaps::Target::Utils::keys_expr(
+ BuildMaps::Target::Utils::obtainTargetByName(
+ eval, expr, env, key.target, deps_by_transition)
+ ->RunFiles());
+ }}});
+ auto param_config = key.config.Prune(param_vars);
+ params.reserve(params.size() + rule->StringFields().size());
+ for (auto const& field_name : rule->StringFields()) {
+ auto const& field_exp = data->string_exprs[field_name];
+ auto field_value = field_exp.Evaluate(
+ param_config,
+ string_fields_fcts,
+ [&logger, &field_name](auto const& msg) {
+ (*logger)(fmt::format("While evaluating string field {}:\n{}",
+ field_name,
+ msg),
+ true);
+ });
+ if (not field_value) {
+ return;
+ }
+ if (not field_value->IsList()) {
+ (*logger)(fmt::format("String field {} should be a list of "
+ "strings, but found {}",
+ field_name,
+ field_value->ToString()),
+ true);
+ return;
+ }
+ for (auto const& entry : field_value->List()) {
+ if (not entry->IsString()) {
+ (*logger)(fmt::format("String field {} should be a list of "
+ "strings, but found entry {}",
+ field_name,
+ entry->ToString()),
+ true);
+ return;
+ }
+ }
+ params.emplace(field_name, std::move(field_value));
+ }
+
+ // Evaluate main expression
+ auto expression_config = key.config.Prune(config_vars);
+ std::vector<ActionDescription> actions{};
+ std::vector<std::string> blobs{};
+ std::vector<Tree> trees{};
+ auto main_exp_fcts = FunctionMap::MakePtr(
+ {{"FIELD",
+ [&params](auto&& eval, auto const& expr, auto const& env) {
+ auto name = eval(expr["name"], env);
+ if (not name->IsString()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("FIELD argument 'name' should evaluate to a "
+ "string, but got {}",
+ name->ToString())};
+ }
+ auto it = params.find(name->String());
+ if (it == params.end()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("FIELD '{}' unknown", name->String())};
+ }
+ return it->second;
+ }},
+ {"DEP_ARTIFACTS",
+ [&deps_by_transition](
+ auto&& eval, auto const& expr, auto const& env) {
+ return BuildMaps::Target::Utils::obtainTarget(
+ eval, expr, env, deps_by_transition)
+ ->Artifacts();
+ }},
+ {"DEP_RUNFILES",
+ [&deps_by_transition](
+ auto&& eval, auto const& expr, auto const& env) {
+ return BuildMaps::Target::Utils::obtainTarget(
+ eval, expr, env, deps_by_transition)
+ ->RunFiles();
+ }},
+ {"DEP_PROVIDES",
+ [&deps_by_transition](
+ auto&& eval, auto const& expr, auto const& env) {
+ auto const& provided = BuildMaps::Target::Utils::obtainTarget(
+ eval, expr, env, deps_by_transition)
+ ->Provides();
+ auto provider = eval(expr["provider"], env);
+ auto provided_value = provided->At(provider->String());
+ if (provided_value) {
+ return provided_value->get();
+ }
+ auto const& empty_list = Expression::kEmptyList;
+ return eval(expr->Get("default", empty_list), env);
+ }},
+ {"ACTION",
+ [&actions, &rule](auto&& eval, auto const& expr, auto const& env) {
+ auto const& empty_map_exp = Expression::kEmptyMapExpr;
+ auto inputs_exp = eval(expr->Get("inputs", empty_map_exp), env);
+ if (not inputs_exp->IsMap()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "inputs has to be a map of artifacts, but found {}",
+ inputs_exp->ToString())};
+ }
+ for (auto const& [input_path, artifact] : inputs_exp->Map()) {
+ if (not artifact->IsArtifact()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("inputs has to be a map of Artifacts, "
+ "but found {} for {}",
+ artifact->ToString(),
+ input_path)};
+ }
+ }
+ auto conflict =
+ BuildMaps::Target::Utils::tree_conflict(inputs_exp);
+ if (conflict) {
+ throw Evaluator::EvaluationError{
+ fmt::format("inputs conflicts on subtree {}", *conflict)};
+ }
+
+ Expression::map_t::underlying_map_t result;
+ auto outputs = ReadActionOutputExpr(
+ eval(expr->Get("outs", Expression::list_t{}), env), "outs");
+ auto output_dirs = ReadActionOutputExpr(
+ eval(expr->Get("out_dirs", Expression::list_t{}), env),
+ "out_dirs");
+ if (outputs.empty() and output_dirs.empty()) {
+ throw Evaluator::EvaluationError{
+ "either outs or out_dirs must be specified for ACTION"};
+ }
+
+ std::sort(outputs.begin(), outputs.end());
+ std::sort(output_dirs.begin(), output_dirs.end());
+ std::vector<std::string> dups{};
+ std::set_intersection(outputs.begin(),
+ outputs.end(),
+ output_dirs.begin(),
+ output_dirs.end(),
+ std::back_inserter(dups));
+ if (not dups.empty()) {
+ throw Evaluator::EvaluationError{
+ "outs and out_dirs for ACTION must be disjoint"};
+ }
+
+ std::vector<std::string> cmd;
+ auto cmd_exp = eval(expr->Get("cmd", Expression::list_t{}), env);
+ if (not cmd_exp->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "cmd has to be a list of strings, but found {}",
+ cmd_exp->ToString())};
+ }
+ if (cmd_exp->List().empty()) {
+ throw Evaluator::EvaluationError{
+ "cmd must not be an empty list"};
+ }
+ cmd.reserve(cmd_exp->List().size());
+ for (auto const& arg : cmd_exp->List()) {
+ if (not arg->IsString()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "cmd has to be a list of strings, but found {}",
+ cmd_exp->ToString())};
+ }
+ cmd.emplace_back(arg->String());
+ }
+ auto env_exp = eval(expr->Get("env", empty_map_exp), env);
+ if (not env_exp->IsMap()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("env has to be a map of string, but found {}",
+ env_exp->ToString())};
+ }
+ for (auto const& [env_var, env_value] : env_exp->Map()) {
+ if (not env_value->IsString()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "env has to be a map of string, but found {}",
+ env_exp->ToString())};
+ }
+ }
+ auto may_fail_exp = expr->Get("may_fail", Expression::list_t{});
+ if (not may_fail_exp->IsList()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("may_fail has to be a list of "
+ "strings, but found {}",
+ may_fail_exp->ToString())};
+ }
+ for (auto const& entry : may_fail_exp->List()) {
+ if (not entry->IsString()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("may_fail has to be a list of "
+ "strings, but found {}",
+ may_fail_exp->ToString())};
+ }
+ if (rule->Tainted().find(entry->String()) ==
+ rule->Tainted().end()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("may_fail contains entry {} the the rule "
+ "is not tainted with",
+ entry->ToString())};
+ }
+ }
+ std::optional<std::string> may_fail = std::nullopt;
+ if (not may_fail_exp->List().empty()) {
+ auto fail_msg =
+ eval(expr->Get("fail_message", "action failed"s), env);
+ if (not fail_msg->IsString()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "fail_message has to evalute to a string, but got {}",
+ fail_msg->ToString())};
+ }
+ may_fail = std::optional{fail_msg->String()};
+ }
+ auto no_cache_exp = expr->Get("no_cache", Expression::list_t{});
+ if (not no_cache_exp->IsList()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("no_cache has to be a list of"
+ "strings, but found {}",
+ no_cache_exp->ToString())};
+ }
+ for (auto const& entry : no_cache_exp->List()) {
+ if (not entry->IsString()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("no_cache has to be a list of"
+ "strings, but found {}",
+ no_cache_exp->ToString())};
+ }
+ if (rule->Tainted().find(entry->String()) ==
+ rule->Tainted().end()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("no_cache contains entry {} the the rule "
+ "is not tainted with",
+ entry->ToString())};
+ }
+ }
+ bool no_cache = not no_cache_exp->List().empty();
+ auto action =
+ BuildMaps::Target::Utils::createAction(outputs,
+ output_dirs,
+ std::move(cmd),
+ env_exp,
+ may_fail,
+ no_cache,
+ inputs_exp);
+ auto action_id = action.Id();
+ actions.emplace_back(std::move(action));
+ for (auto const& out : outputs) {
+ result.emplace(out,
+ ExpressionPtr{ArtifactDescription{
+ action_id, std::filesystem::path{out}}});
+ }
+ for (auto const& out : output_dirs) {
+ result.emplace(out,
+ ExpressionPtr{ArtifactDescription{
+ action_id, std::filesystem::path{out}}});
+ }
+
+ return ExpressionPtr{Expression::map_t{result}};
+ }},
+ {"BLOB",
+ [&blobs](auto&& eval, auto const& expr, auto const& env) {
+ auto data = eval(expr->Get("data", ""s), env);
+ if (not data->IsString()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("BLOB data has to be a string, but got {}",
+ data->ToString())};
+ }
+ blobs.emplace_back(data->String());
+ return ExpressionPtr{ArtifactDescription{
+ {ComputeHash(data->String()), data->String().size()},
+ ObjectType::File}};
+ }},
+ {"TREE",
+ [&trees](auto&& eval, auto const& expr, auto const& env) {
+ auto val = eval(expr->Get("$1", Expression::kEmptyMapExpr), env);
+ if (not val->IsMap()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("TREE argument has to be a map of artifacts, "
+ "but found {}",
+ val->ToString())};
+ }
+ std::unordered_map<std::string, ArtifactDescription> artifacts;
+ artifacts.reserve(val->Map().size());
+ for (auto const& [input_path, artifact] : val->Map()) {
+ if (not artifact->IsArtifact()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "TREE argument has to be a map of artifacts, "
+ "but found {} for {}",
+ artifact->ToString(),
+ input_path)};
+ }
+ auto norm_path = std::filesystem::path{input_path}
+ .lexically_normal()
+ .string();
+ if (norm_path == "." or norm_path.empty()) {
+ if (val->Map().size() > 1) {
+ throw Evaluator::EvaluationError{
+ "input path '.' or '' for TREE is only allowed "
+ "for trees with single input artifact"};
+ }
+ if (not artifact->Artifact().IsTree()) {
+ throw Evaluator::EvaluationError{
+ "input path '.' or '' for TREE must be tree "
+ "artifact"};
+ }
+ return artifact;
+ }
+ artifacts.emplace(std::move(norm_path), artifact->Artifact());
+ }
+ auto conflict = BuildMaps::Target::Utils::tree_conflict(val);
+ if (conflict) {
+ throw Evaluator::EvaluationError{
+ fmt::format("TREE conflicts on subtree {}", *conflict)};
+ }
+ auto tree = Tree{std::move(artifacts)};
+ auto tree_id = tree.Id();
+ trees.emplace_back(std::move(tree));
+ return ExpressionPtr{ArtifactDescription{tree_id}};
+ }},
+ {"VALUE_NODE",
+ [](auto&& eval, auto const& expr, auto const& env) {
+ auto val = eval(expr->Get("$1", Expression::kNone), env);
+ if (not val->IsResult()) {
+ throw Evaluator::EvaluationError{
+ "argument '$1' for VALUE_NODE not a RESULT type."};
+ }
+ return ExpressionPtr{TargetNode{std::move(val)}};
+ }},
+ {"ABSTRACT_NODE",
+ [](auto&& eval, auto const& expr, auto const& env) {
+ auto type = eval(expr->Get("node_type", Expression::kNone), env);
+ if (not type->IsString()) {
+ throw Evaluator::EvaluationError{
+ "argument 'node_type' for ABSTRACT_NODE not a string."};
+ }
+ auto string_fields = eval(
+ expr->Get("string_fields", Expression::kEmptyMapExpr), env);
+ if (not string_fields->IsMap()) {
+ throw Evaluator::EvaluationError{
+ "argument 'string_fields' for ABSTRACT_NODE not a map."};
+ }
+ auto target_fields = eval(
+ expr->Get("target_fields", Expression::kEmptyMapExpr), env);
+ if (not target_fields->IsMap()) {
+ throw Evaluator::EvaluationError{
+ "argument 'target_fields' for ABSTRACT_NODE not a map."};
+ }
+
+ std::optional<std::string> dup_key{std::nullopt};
+ auto check_entries =
+ [&dup_key](auto const& map,
+ auto const& type_check,
+ std::string const& fields_name,
+ std::string const& type_name,
+ std::optional<ExpressionPtr> const& disjoint_map =
+ std::nullopt) {
+ for (auto const& [key, list] : map->Map()) {
+ if (not list->IsList()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "value for key {} in argument '{}' for "
+ "ABSTRACT_NODE is not a list.",
+ key,
+ fields_name)};
+ }
+ for (auto const& entry : list->List()) {
+ if (not type_check(entry)) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "list entry for {} in argument '{}' for "
+ "ABSTRACT_NODE is not a {}:\n{}",
+ key,
+ fields_name,
+ type_name,
+ entry->ToString())};
+ }
+ }
+ if (disjoint_map) {
+ if ((*disjoint_map)->Map().Find(key)) {
+ dup_key = key;
+ return;
+ }
+ }
+ }
+ };
+
+ auto is_string = [](auto const& e) { return e->IsString(); };
+ check_entries(string_fields,
+ is_string,
+ "string_fields",
+ "string",
+ target_fields);
+ if (dup_key) {
+ throw Evaluator::EvaluationError{
+ fmt::format("string_fields and target_fields are not "
+ "disjoint maps, found duplicate key: {}.",
+ *dup_key)};
+ }
+
+ auto is_node = [](auto const& e) { return e->IsNode(); };
+ check_entries(
+ target_fields, is_node, "target_fields", "target node");
+
+ return ExpressionPtr{
+ TargetNode{TargetNode::Abstract{type->String(),
+ std::move(string_fields),
+ std::move(target_fields)}}};
+ }},
+ {"RESULT", [](auto&& eval, auto const& expr, auto const& env) {
+ auto const& empty_map_exp = Expression::kEmptyMapExpr;
+ auto artifacts = eval(expr->Get("artifacts", empty_map_exp), env);
+ auto runfiles = eval(expr->Get("runfiles", empty_map_exp), env);
+ auto provides = eval(expr->Get("provides", empty_map_exp), env);
+ if (not artifacts->IsMap()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "artifacts has to be a map of artifacts, but found {}",
+ artifacts->ToString())};
+ }
+ for (auto const& [path, entry] : artifacts->Map()) {
+ if (not entry->IsArtifact()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("artifacts has to be a map of artifacts, "
+ "but found {} for {}",
+ entry->ToString(),
+ path)};
+ }
+ }
+ if (not runfiles->IsMap()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "runfiles has to be a map of artifacts, but found {}",
+ runfiles->ToString())};
+ }
+ for (auto const& [path, entry] : runfiles->Map()) {
+ if (not entry->IsArtifact()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("runfiles has to be a map of artifacts, "
+ "but found {} for {}",
+ entry->ToString(),
+ path)};
+ }
+ }
+ if (not provides->IsMap()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("provides has to be a map, but found {}",
+ provides->ToString())};
+ }
+ return ExpressionPtr{TargetResult{artifacts, provides, runfiles}};
+ }}});
+
+ auto result = rule->Expression()->Evaluate(
+ expression_config, main_exp_fcts, [logger](auto const& msg) {
+ (*logger)(
+ fmt::format("While evaluating defining expression of rule:\n{}",
+ msg),
+ true);
+ });
+ if (not result) {
+ return;
+ }
+ if (not result->IsResult()) {
+ (*logger)(fmt::format("Defining expression should evaluate to a "
+ "RESULT, but got: {}",
+ result->ToString()),
+ true);
+ return;
+ }
+ auto analysis_result =
+ std::make_shared<AnalysedTarget>((*std::move(result)).Result(),
+ std::move(actions),
+ std::move(blobs),
+ std::move(trees),
+ std::move(effective_vars),
+ std::move(tainted));
+ analysis_result =
+ result_map->Add(key.target, effective_conf, std::move(analysis_result));
+ (*setter)(std::move(analysis_result));
+}
+
+[[nodiscard]] auto isTransition(
+ const ExpressionPtr& ptr,
+ std::function<void(std::string const&)> const& logger) -> bool {
+ if (not ptr->IsList()) {
+ logger(fmt::format("expected list, but got {}", ptr->ToString()));
+ return false;
+ }
+ for (const auto& entry : ptr->List()) {
+ if (not entry->IsMap()) {
+ logger(fmt::format("expected list of dicts, but found {}",
+ ptr->ToString()));
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void withRuleDefinition(
+ const BuildMaps::Base::UserRulePtr& rule,
+ const TargetData::Ptr& data,
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*> result_map) {
+ auto param_config = key.config.Prune(data->target_vars);
+
+ // Evaluate the config_fields
+
+ std::unordered_map<std::string, ExpressionPtr> params;
+ params.reserve(rule->ConfigFields().size() + rule->TargetFields().size() +
+ rule->ImplicitTargetExps().size());
+ for (auto field_name : rule->ConfigFields()) {
+ auto const& field_expression = data->config_exprs[field_name];
+ auto field_value = field_expression.Evaluate(
+ param_config, {}, [&logger, &field_name](auto const& msg) {
+ (*logger)(fmt::format("While evaluating config fieled {}:\n{}",
+ field_name,
+ msg),
+ true);
+ });
+ if (not field_value) {
+ return;
+ }
+ if (not field_value->IsList()) {
+ (*logger)(fmt::format("Config field {} should evaluate to a list "
+ "of strings, but got{}",
+ field_name,
+ field_value->ToString()),
+ true);
+ return;
+ }
+ for (auto const& entry : field_value->List()) {
+ if (not entry->IsString()) {
+ (*logger)(fmt::format("Config field {} should evaluate to a "
+ "list of strings, but got{}",
+ field_name,
+ field_value->ToString()),
+ true);
+ return;
+ }
+ }
+ params.emplace(field_name, field_value);
+ }
+
+ // Evaluate config transitions
+
+ auto config_trans_fcts = FunctionMap::MakePtr(
+ "FIELD", [&params](auto&& eval, auto const& expr, auto const& env) {
+ auto name = eval(expr["name"], env);
+ if (not name->IsString()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("FIELD argument 'name' should evaluate to a "
+ "string, but got {}",
+ name->ToString())};
+ }
+ auto it = params.find(name->String());
+ if (it == params.end()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("FIELD {} unknown", name->String())};
+ }
+ return it->second;
+ });
+
+ auto const& config_vars = rule->ConfigVars();
+ auto expression_config = key.config.Prune(config_vars);
+
+ std::unordered_map<std::string, ExpressionPtr> config_transitions;
+ config_transitions.reserve(rule->TargetFields().size() +
+ rule->ImplicitTargets().size() +
+ rule->AnonymousDefinitions().size());
+ for (auto const& target_field_name : rule->TargetFields()) {
+ auto exp = rule->ConfigTransitions().at(target_field_name);
+ auto transition_logger = [&logger,
+ &target_field_name](auto const& msg) {
+ (*logger)(
+ fmt::format("While evaluating config transition for {}:\n{}",
+ target_field_name,
+ msg),
+ true);
+ };
+ auto transition = exp->Evaluate(
+ expression_config, config_trans_fcts, transition_logger);
+ if (not transition) {
+ return;
+ }
+ if (not isTransition(transition, transition_logger)) {
+ return;
+ }
+ config_transitions.emplace(target_field_name, transition);
+ }
+ for (const auto& name_value : rule->ImplicitTargets()) {
+ auto implicit_field_name = name_value.first;
+ auto exp = rule->ConfigTransitions().at(implicit_field_name);
+ auto transition_logger = [&logger,
+ &implicit_field_name](auto const& msg) {
+ (*logger)(fmt::format("While evaluating config transition for "
+ "implicit {}:\n{}",
+ implicit_field_name,
+ msg),
+ true);
+ };
+ auto transition = exp->Evaluate(
+ expression_config, config_trans_fcts, transition_logger);
+ if (not transition) {
+ return;
+ }
+ if (not isTransition(transition, transition_logger)) {
+ return;
+ }
+ config_transitions.emplace(implicit_field_name, transition);
+ }
+ for (const auto& entry : rule->AnonymousDefinitions()) {
+ auto const& anon_field_name = entry.first;
+ auto exp = rule->ConfigTransitions().at(anon_field_name);
+ auto transition_logger = [&logger, &anon_field_name](auto const& msg) {
+ (*logger)(fmt::format("While evaluating config transition for "
+ "anonymous {}:\n{}",
+ anon_field_name,
+ msg),
+ true);
+ };
+ auto transition = exp->Evaluate(
+ expression_config, config_trans_fcts, transition_logger);
+ if (not transition) {
+ return;
+ }
+ if (not isTransition(transition, transition_logger)) {
+ return;
+ }
+ config_transitions.emplace(anon_field_name, transition);
+ }
+
+ // Request dependencies
+
+ std::unordered_map<std::string, std::vector<std::size_t>> anon_positions;
+ anon_positions.reserve(rule->AnonymousDefinitions().size());
+ for (auto const& [_, def] : rule->AnonymousDefinitions()) {
+ anon_positions.emplace(def.target, std::vector<std::size_t>{});
+ }
+
+ std::vector<BuildMaps::Target::ConfiguredTarget> dependency_keys;
+ std::vector<BuildMaps::Target::ConfiguredTarget> transition_keys;
+ for (auto target_field_name : rule->TargetFields()) {
+ auto const& deps_expression = data->target_exprs[target_field_name];
+ auto deps_names = deps_expression.Evaluate(
+ param_config, {}, [logger, target_field_name](auto const& msg) {
+ (*logger)(
+ fmt::format("While evaluating target parameter {}:\n{}",
+ target_field_name,
+ msg),
+ true);
+ });
+ if (not deps_names->IsList()) {
+ (*logger)(fmt::format("Target parameter {} should evaluate to a "
+ "list, but got {}",
+ target_field_name,
+ deps_names->ToString()),
+ true);
+ return;
+ }
+ Expression::list_t dep_target_exps;
+ if (data->parse_target_names) {
+ dep_target_exps.reserve(deps_names->List().size());
+ for (const auto& dep_name : deps_names->List()) {
+ auto target = BuildMaps::Base::ParseEntityNameFromExpression(
+ dep_name,
+ key.target,
+ [&logger, &target_field_name, &dep_name](
+ std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing entry {} in target "
+ "field {} failed with:\n{}",
+ dep_name->ToString(),
+ target_field_name,
+ parse_err),
+ true);
+ });
+ if (not target) {
+ return;
+ }
+ dep_target_exps.emplace_back(ExpressionPtr{*target});
+ }
+ }
+ else {
+ dep_target_exps = deps_names->List();
+ }
+ auto anon_pos = anon_positions.find(target_field_name);
+ auto const& transitions = config_transitions[target_field_name]->List();
+ for (const auto& transition : transitions) {
+ auto transitioned_config = key.config.Update(transition);
+ for (const auto& dep : dep_target_exps) {
+ if (anon_pos != anon_positions.end()) {
+ anon_pos->second.emplace_back(dependency_keys.size());
+ }
+
+ dependency_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{dep->Name(),
+ transitioned_config});
+ transition_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{
+ dep->Name(), Configuration{transition}});
+ }
+ }
+ params.emplace(target_field_name,
+ ExpressionPtr{std::move(dep_target_exps)});
+ }
+ for (auto const& [implicit_field_name, implicit_target] :
+ rule->ImplicitTargets()) {
+ auto anon_pos = anon_positions.find(implicit_field_name);
+ auto transitions = config_transitions[implicit_field_name]->List();
+ for (const auto& transition : transitions) {
+ auto transitioned_config = key.config.Update(transition);
+ for (const auto& dep : implicit_target) {
+ if (anon_pos != anon_positions.end()) {
+ anon_pos->second.emplace_back(dependency_keys.size());
+ }
+
+ dependency_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{dep,
+ transitioned_config});
+ transition_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{
+ dep, Configuration{transition}});
+ }
+ }
+ }
+ params.insert(rule->ImplicitTargetExps().begin(),
+ rule->ImplicitTargetExps().end());
+
+ (*subcaller)(
+ dependency_keys,
+ [transition_keys = std::move(transition_keys),
+ rule,
+ data,
+ key,
+ params = std::move(params),
+ setter,
+ logger,
+ result_map,
+ subcaller,
+ config_transitions = std::move(config_transitions),
+ anon_positions =
+ std::move(anon_positions)](auto const& values) mutable {
+ // Now that all non-anonymous targets have been evaluated we can
+ // read their provides map to construct and evaluate anonymous
+ // targets.
+ std::vector<BuildMaps::Target::ConfiguredTarget> anonymous_keys;
+ for (auto const& [name, def] : rule->AnonymousDefinitions()) {
+ Expression::list_t anon_names{};
+ for (auto pos : anon_positions.at(def.target)) {
+ auto const& provider_value =
+ (*values[pos])->Provides()->Map().Find(def.provider);
+ if (not provider_value) {
+ (*logger)(
+ fmt::format("Provider {} in {} does not exist",
+ def.provider,
+ def.target),
+ true);
+ return;
+ }
+ auto const& exprs = provider_value->get();
+ if (not exprs->IsList()) {
+ (*logger)(fmt::format("Provider {} in {} must be list "
+ "of target nodes but found: {}",
+ def.provider,
+ def.target,
+ exprs->ToString()),
+ true);
+ return;
+ }
+
+ auto const& list = exprs->List();
+ anon_names.reserve(anon_names.size() + list.size());
+ for (auto const& node : list) {
+ if (not node->IsNode()) {
+ (*logger)(
+ fmt::format("Entry in provider {} in {} must "
+ "be target node but found: {}",
+ def.provider,
+ def.target,
+ node->ToString()),
+ true);
+ return;
+ }
+ anon_names.emplace_back(BuildMaps::Base::EntityName{
+ BuildMaps::Base::AnonymousTarget{def.rule_map,
+ node}});
+ }
+ }
+
+ for (const auto& transition :
+ config_transitions.at(name)->List()) {
+ auto transitioned_config = key.config.Update(transition);
+ for (auto const& anon : anon_names) {
+ anonymous_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{
+ anon->Name(), transitioned_config});
+
+ transition_keys.emplace_back(
+ BuildMaps::Target::ConfiguredTarget{
+ anon->Name(), Configuration{transition}});
+ }
+ }
+
+ params.emplace(name, ExpressionPtr{std::move(anon_names)});
+ }
+ (*subcaller)(
+ anonymous_keys,
+ [dependency_values = values,
+ transition_keys = std::move(transition_keys),
+ rule,
+ data,
+ key,
+ params = std::move(params),
+ setter,
+ logger,
+ result_map](auto const& values) mutable {
+ // Join dependency values and anonymous values
+ dependency_values.insert(
+ dependency_values.end(), values.begin(), values.end());
+ withDependencies(transition_keys,
+ dependency_values,
+ rule,
+ data,
+ key,
+ params,
+ setter,
+ logger,
+ result_map);
+ },
+ logger);
+ },
+ logger);
+}
+
+void withTargetsFile(
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const nlohmann::json& targets_file,
+ const gsl::not_null<BuildMaps::Base::SourceTargetMap*>& source_target,
+ const gsl::not_null<BuildMaps::Base::UserRuleMap*>& rule_map,
+ const gsl::not_null<TaskSystem*>& ts,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*> result_map) {
+ auto desc_it = targets_file.find(key.target.name);
+ if (desc_it == targets_file.end()) {
+ // Not a defined taraget, treat as source target
+ source_target->ConsumeAfterKeysReady(
+ ts,
+ {key.target},
+ [setter](auto values) { (*setter)(AnalysedTargetPtr{*values[0]}); },
+ [logger, target = key.target](auto const& msg, auto fatal) {
+ (*logger)(fmt::format("While analysing target {} as implicit "
+ "source target:\n{}",
+ target.ToString(),
+ msg),
+ fatal);
+ });
+ }
+ else {
+ nlohmann::json desc = *desc_it;
+ auto rule_it = desc.find("type");
+ if (rule_it == desc.end()) {
+ (*logger)(
+ fmt::format("No type specified in the definition of target {}",
+ key.target.ToString()),
+ true);
+ return;
+ }
+ // Handle built-in rule, if it is
+ auto handled_as_builtin = BuildMaps::Target::HandleBuiltin(
+ *rule_it, desc, key, subcaller, setter, logger, result_map);
+ if (handled_as_builtin) {
+ return;
+ }
+
+ // Not a built-in rule, so has to be a user rule
+ auto rule_name = BuildMaps::Base::ParseEntityNameFromJson(
+ *rule_it,
+ key.target,
+ [&logger, &rule_it, &key](std::string const& parse_err) {
+ (*logger)(fmt::format("Parsing rule name {} for target {} "
+ "failed with:\n{}",
+ rule_it->dump(),
+ key.target.ToString(),
+ parse_err),
+ true);
+ });
+ if (not rule_name) {
+ return;
+ }
+ auto desc_reader = BuildMaps::Base::FieldReader::CreatePtr(
+ desc,
+ key.target,
+ fmt::format("{} target", rule_name->ToString()),
+ logger);
+ if (not desc_reader) {
+ return;
+ }
+ rule_map->ConsumeAfterKeysReady(
+ ts,
+ {*rule_name},
+ [desc = std::move(desc_reader),
+ subcaller,
+ setter,
+ logger,
+ key,
+ result_map,
+ rn = *rule_name](auto values) {
+ auto data = TargetData::FromFieldReader(*values[0], desc);
+ if (not data) {
+ (*logger)(fmt::format("Failed to read data from target {} "
+ "with rule {}",
+ key.target.ToString(),
+ rn.ToString()),
+ /*fatal=*/true);
+ return;
+ }
+ withRuleDefinition(
+ *values[0],
+ data,
+ key,
+ subcaller,
+ setter,
+ std::make_shared<AsyncMapConsumerLogger>(
+ [logger, target = key.target, rn](auto const& msg,
+ auto fatal) {
+ (*logger)(
+ fmt::format("While analysing {} target {}:\n{}",
+ rn.ToString(),
+ target.ToString(),
+ msg),
+ fatal);
+ }),
+ result_map);
+ },
+ [logger, target = key.target](auto const& msg, auto fatal) {
+ (*logger)(fmt::format("While looking up rule for {}:\n{}",
+ target.ToString(),
+ msg),
+ fatal);
+ });
+ }
+}
+
+void withTargetNode(
+ const BuildMaps::Target::ConfiguredTarget& key,
+ const gsl::not_null<BuildMaps::Base::UserRuleMap*>& rule_map,
+ const gsl::not_null<TaskSystem*>& ts,
+ const BuildMaps::Target::TargetMap::SubCallerPtr& subcaller,
+ const BuildMaps::Target::TargetMap::SetterPtr& setter,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger,
+ const gsl::not_null<BuildMaps::Target::ResultTargetMap*> result_map) {
+ auto const& target_node = key.target.anonymous->target_node->Node();
+ auto const& rule_mapping = key.target.anonymous->rule_map->Map();
+ if (target_node.IsValue()) {
+ // fixed value node, create analysed target from result
+ auto const& val = target_node.GetValue();
+ (*setter)(std::make_shared<AnalysedTarget>(
+ AnalysedTarget{val->Result(), {}, {}, {}, {}, {}}));
+ }
+ else {
+ // abstract target node, lookup rule and instantiate target
+ auto const& abs = target_node.GetAbstract();
+ auto rule_name = rule_mapping.Find(abs.node_type);
+ if (not rule_name) {
+ (*logger)(fmt::format("Cannot resolve type of node {} via rule map "
+ "{}",
+ target_node.ToString(),
+ key.target.anonymous->rule_map->ToString()),
+ /*fatal=*/true);
+ }
+ rule_map->ConsumeAfterKeysReady(
+ ts,
+ {rule_name->get()->Name()},
+ [abs,
+ subcaller,
+ setter,
+ logger,
+ key,
+ result_map,
+ rn = rule_name->get()](auto values) {
+ auto data = TargetData::FromTargetNode(
+ *values[0], abs, key.target.anonymous->rule_map, logger);
+ if (not data) {
+ (*logger)(fmt::format("Failed to read data from target {} "
+ "with rule {}",
+ key.target.ToString(),
+ rn->ToString()),
+ /*fatal=*/true);
+ return;
+ }
+ withRuleDefinition(*values[0],
+ data,
+ key,
+ subcaller,
+ setter,
+ std::make_shared<AsyncMapConsumerLogger>(
+ [logger, target = key.target, rn](
+ auto const& msg, auto fatal) {
+ (*logger)(
+ fmt::format("While analysing {} "
+ "target {}:\n{}",
+ rn->ToString(),
+ target.ToString(),
+ msg),
+ fatal);
+ }),
+ result_map);
+ },
+ [logger, target = key.target](auto const& msg, auto fatal) {
+ (*logger)(fmt::format("While looking up rule for {}:\n{}",
+ target.ToString(),
+ msg),
+ fatal);
+ });
+ }
+}
+
+} // namespace
+
+namespace BuildMaps::Target {
+auto CreateTargetMap(
+ const gsl::not_null<BuildMaps::Base::SourceTargetMap*>& source_target_map,
+ const gsl::not_null<BuildMaps::Base::TargetsFileMap*>& targets_file_map,
+ const gsl::not_null<BuildMaps::Base::UserRuleMap*>& rule_map,
+ const gsl::not_null<ResultTargetMap*>& result_map,
+ std::size_t jobs) -> TargetMap {
+ auto target_reader =
+ [source_target_map, targets_file_map, rule_map, result_map](
+ auto ts, auto setter, auto logger, auto subcaller, auto key) {
+ if (key.target.explicit_file_reference) {
+ // Not a defined target, treat as source target
+ source_target_map->ConsumeAfterKeysReady(
+ ts,
+ {key.target},
+ [setter](auto values) {
+ (*setter)(AnalysedTargetPtr{*values[0]});
+ },
+ [logger, target = key.target](auto const& msg, auto fatal) {
+ (*logger)(fmt::format("While analysing target {} as "
+ "explicit source target:\n{}",
+ target.ToString(),
+ msg),
+ fatal);
+ });
+ }
+ else if (key.target.IsAnonymousTarget()) {
+ withTargetNode(
+ key, rule_map, ts, subcaller, setter, logger, result_map);
+ }
+ else {
+ targets_file_map->ConsumeAfterKeysReady(
+ ts,
+ {key.target.ToModule()},
+ [key,
+ source_target_map,
+ rule_map,
+ ts,
+ subcaller = std::move(subcaller),
+ setter = std::move(setter),
+ logger,
+ result_map](auto values) {
+ withTargetsFile(key,
+ *values[0],
+ source_target_map,
+ rule_map,
+ ts,
+ subcaller,
+ setter,
+ logger,
+ result_map);
+ },
+ [logger, target = key.target](auto const& msg, auto fatal) {
+ (*logger)(fmt::format("While searching targets "
+ "description for {}:\n{}",
+ target.ToString(),
+ msg),
+ fatal);
+ });
+ }
+ };
+ return AsyncMapConsumer<ConfiguredTarget, AnalysedTargetPtr>(target_reader,
+ jobs);
+}
+} // namespace BuildMaps::Target
diff --git a/src/buildtool/build_engine/target_map/target_map.hpp b/src/buildtool/build_engine/target_map/target_map.hpp
new file mode 100644
index 00000000..4befc842
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/target_map.hpp
@@ -0,0 +1,27 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_TARGET_MAP_HPP
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
+#include "src/buildtool/build_engine/base_maps/rule_map.hpp"
+#include "src/buildtool/build_engine/base_maps/source_map.hpp"
+#include "src/buildtool/build_engine/base_maps/targets_file_map.hpp"
+#include "src/buildtool/build_engine/target_map/configured_target.hpp"
+#include "src/buildtool/build_engine/target_map/result_map.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+namespace BuildMaps::Target {
+
+using TargetMap = AsyncMapConsumer<ConfiguredTarget, AnalysedTargetPtr>;
+
+auto CreateTargetMap(const gsl::not_null<BuildMaps::Base::SourceTargetMap*>&,
+ const gsl::not_null<BuildMaps::Base::TargetsFileMap*>&,
+ const gsl::not_null<BuildMaps::Base::UserRuleMap*>&,
+ const gsl::not_null<ResultTargetMap*>&,
+ std::size_t jobs = 0) -> TargetMap;
+
+auto IsBuiltInRule(nlohmann::json const& rule_type) -> bool;
+
+} // namespace BuildMaps::Target
+
+#endif
diff --git a/src/buildtool/build_engine/target_map/utils.cpp b/src/buildtool/build_engine/target_map/utils.cpp
new file mode 100644
index 00000000..8c5353ce
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/utils.cpp
@@ -0,0 +1,197 @@
+#include "src/buildtool/build_engine/target_map/utils.hpp"
+
+#include <algorithm>
+#include <filesystem>
+#include <vector>
+
+auto BuildMaps::Target::Utils::obtainTargetByName(
+ const SubExprEvaluator& eval,
+ const ExpressionPtr& expr,
+ const Configuration& env,
+ const Base::EntityName& current,
+ std::unordered_map<BuildMaps::Target::ConfiguredTarget,
+ AnalysedTargetPtr> const& deps_by_transition)
+ -> AnalysedTargetPtr {
+ auto const& empty_map_exp = Expression::kEmptyMapExpr;
+ auto reference = eval(expr["dep"], env);
+ std::string error{};
+ auto target = BuildMaps::Base::ParseEntityNameFromExpression(
+ reference, current, [&error](std::string const& parse_err) {
+ error = parse_err;
+ });
+ if (not target) {
+ throw Evaluator::EvaluationError{
+ fmt::format("Parsing target name {} failed with:\n{}",
+ reference->ToString(),
+ error)};
+ }
+ auto transition = eval(expr->Get("transition", empty_map_exp), env);
+ auto it = deps_by_transition.find(BuildMaps::Target::ConfiguredTarget{
+ *target, Configuration{transition}});
+ if (it == deps_by_transition.end()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "Reference to undeclared dependency {} in transition {}",
+ reference->ToString(),
+ transition->ToString())};
+ }
+ return it->second;
+}
+
+auto BuildMaps::Target::Utils::obtainTarget(
+ const SubExprEvaluator& eval,
+ const ExpressionPtr& expr,
+ const Configuration& env,
+ std::unordered_map<BuildMaps::Target::ConfiguredTarget,
+ AnalysedTargetPtr> const& deps_by_transition)
+ -> AnalysedTargetPtr {
+ auto const& empty_map_exp = Expression::kEmptyMapExpr;
+ auto reference = eval(expr["dep"], env);
+ if (not reference->IsName()) {
+ throw Evaluator::EvaluationError{
+ fmt::format("Not a target name: {}", reference->ToString())};
+ }
+ auto transition = eval(expr->Get("transition", empty_map_exp), env);
+ auto it = deps_by_transition.find(BuildMaps::Target::ConfiguredTarget{
+ reference->Name(), Configuration{transition}});
+ if (it == deps_by_transition.end()) {
+ throw Evaluator::EvaluationError{fmt::format(
+ "Reference to undeclared dependency {} in transition {}",
+ reference->ToString(),
+ transition->ToString())};
+ }
+ return it->second;
+}
+
+auto BuildMaps::Target::Utils::keys_expr(const ExpressionPtr& map)
+ -> ExpressionPtr {
+ auto const& m = map->Map();
+ auto result = Expression::list_t{};
+ result.reserve(m.size());
+ std::for_each(m.begin(), m.end(), [&](auto const& item) {
+ result.emplace_back(ExpressionPtr{item.first});
+ });
+ return ExpressionPtr{result};
+}
+
+auto BuildMaps::Target::Utils::tree_conflict(const ExpressionPtr& map)
+ -> std::optional<std::string> {
+ std::vector<std::filesystem::path> trees{};
+ for (auto const& [path, artifact] : map->Map()) {
+ if (artifact->Artifact().IsTree()) {
+ trees.emplace_back(std::filesystem::path{path});
+ }
+ }
+ if (trees.empty()) {
+ return std::nullopt;
+ }
+ for (auto const& [path, artifact] : map->Map()) {
+ auto p = std::filesystem::path{path};
+ for (auto const& treepath : trees) {
+ if (not artifact->Artifact().IsTree()) {
+ if (std::mismatch(treepath.begin(), treepath.end(), p.begin())
+ .first == treepath.end()) {
+ return path;
+ }
+ }
+ }
+ }
+ return std::nullopt;
+}
+
+auto BuildMaps::Target::Utils::getTainted(
+ std::set<std::string>* tainted,
+ const Configuration& config,
+ const ExpressionPtr& tainted_exp,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger) -> bool {
+ if (not tainted_exp) {
+ return false;
+ }
+ auto tainted_val =
+ tainted_exp.Evaluate(config, {}, [logger](auto const& msg) {
+ (*logger)(fmt::format("While evaluating tainted:\n{}", msg), true);
+ });
+ if (not tainted_val) {
+ return false;
+ }
+ if (not tainted_val->IsList()) {
+ (*logger)(fmt::format("tainted should evaluate to a list of strings, "
+ "but got {}",
+ tainted_val->ToString()),
+ true);
+ return false;
+ }
+ for (auto const& entry : tainted_val->List()) {
+ if (not entry->IsString()) {
+ (*logger)(fmt::format("tainted should evaluate to a list of "
+ "strings, but got {}",
+ tainted_val->ToString()),
+ true);
+ return false;
+ }
+ tainted->insert(entry->String());
+ }
+ return true;
+}
+
+namespace {
+auto hash_vector(std::vector<std::string> const& vec) -> std::string {
+ auto hasher = HashGenerator{BuildMaps::Target::Utils::kActionHash}
+ .IncrementalHasher();
+ for (auto const& s : vec) {
+ hasher.Update(HashGenerator{BuildMaps::Target::Utils::kActionHash}
+ .Run(s)
+ .Bytes());
+ }
+ auto digest = std::move(hasher).Finalize();
+ if (not digest) {
+ Logger::Log(LogLevel::Error, "Failed to finalize hash.");
+ std::terminate();
+ }
+ return digest->Bytes();
+}
+} // namespace
+
+auto BuildMaps::Target::Utils::createAction(
+ ActionDescription::outputs_t output_files,
+ ActionDescription::outputs_t output_dirs,
+ std::vector<std::string> command,
+ const ExpressionPtr& env,
+ std::optional<std::string> may_fail,
+ bool no_cache,
+ const ExpressionPtr& inputs_exp) -> ActionDescription {
+ auto hasher = HashGenerator{BuildMaps::Target::Utils::kActionHash}
+ .IncrementalHasher();
+ hasher.Update(hash_vector(output_files));
+ hasher.Update(hash_vector(output_dirs));
+ hasher.Update(hash_vector(command));
+ hasher.Update(env->ToHash());
+ hasher.Update(hash_vector(may_fail ? std::vector<std::string>{*may_fail}
+ : std::vector<std::string>{}));
+ hasher.Update(no_cache ? std::string{"N"} : std::string{"Y"});
+ hasher.Update(inputs_exp->ToHash());
+
+ auto digest = std::move(hasher).Finalize();
+ if (not digest) {
+ Logger::Log(LogLevel::Error, "Failed to finalize hash.");
+ std::terminate();
+ }
+ auto action_id = digest->HexString();
+
+ std::map<std::string, std::string> env_vars{};
+ for (auto const& [env_var, env_value] : env->Map()) {
+ env_vars.emplace(env_var, env_value->String());
+ }
+ ActionDescription::inputs_t inputs;
+ inputs.reserve(inputs_exp->Map().size());
+ for (auto const& [input_path, artifact] : inputs_exp->Map()) {
+ inputs.emplace(input_path, artifact->Artifact());
+ }
+ return ActionDescription{std::move(output_files),
+ std::move(output_dirs),
+ Action{std::move(action_id),
+ std::move(command),
+ std::move(env_vars),
+ std::move(may_fail),
+ no_cache},
+ std::move(inputs)};
+}
diff --git a/src/buildtool/build_engine/target_map/utils.hpp b/src/buildtool/build_engine/target_map/utils.hpp
new file mode 100644
index 00000000..e92e6281
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/utils.hpp
@@ -0,0 +1,55 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_UTILS_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_UTILS_HPP
+
+#include <optional>
+#include <unordered_map>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/base_maps/field_reader.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/build_engine/expression/evaluator.hpp"
+#include "src/buildtool/build_engine/expression/function_map.hpp"
+#include "src/buildtool/build_engine/target_map/configured_target.hpp"
+#include "src/buildtool/build_engine/target_map/target_map.hpp"
+
+namespace BuildMaps::Target::Utils {
+
+constexpr HashGenerator::HashType kActionHash = HashGenerator::HashType::SHA256;
+
+auto obtainTargetByName(const SubExprEvaluator&,
+ const ExpressionPtr&,
+ const Configuration&,
+ const Base::EntityName&,
+ std::unordered_map<BuildMaps::Target::ConfiguredTarget,
+ AnalysedTargetPtr> const&)
+ -> AnalysedTargetPtr;
+
+auto obtainTarget(const SubExprEvaluator&,
+ const ExpressionPtr&,
+ const Configuration&,
+ std::unordered_map<BuildMaps::Target::ConfiguredTarget,
+ AnalysedTargetPtr> const&)
+ -> AnalysedTargetPtr;
+
+auto keys_expr(const ExpressionPtr& map) -> ExpressionPtr;
+
+auto tree_conflict(const ExpressionPtr & /* map */)
+ -> std::optional<std::string>;
+
+auto getTainted(std::set<std::string>* tainted,
+ const Configuration& config,
+ const ExpressionPtr& tainted_exp,
+ const BuildMaps::Target::TargetMap::LoggerPtr& logger) -> bool;
+
+auto createAction(ActionDescription::outputs_t output_files,
+ ActionDescription::outputs_t output_dirs,
+ std::vector<std::string> command,
+ const ExpressionPtr& env,
+ std::optional<std::string> may_fail,
+ bool no_cache,
+ const ExpressionPtr& inputs_exp) -> ActionDescription;
+
+} // namespace BuildMaps::Target::Utils
+#endif
diff --git a/src/buildtool/common/TARGETS b/src/buildtool/common/TARGETS
new file mode 100644
index 00000000..642ff0ff
--- /dev/null
+++ b/src/buildtool/common/TARGETS
@@ -0,0 +1,101 @@
+{ "cli":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["cli"]
+ , "hdrs": ["cli.hpp"]
+ , "deps":
+ [ ["src/buildtool/logging", "log_level"]
+ , ["@", "cli11", "", "cli11"]
+ , ["@", "json", "", "json"]
+ , ["@", "fmt", "", "fmt"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "common"]
+ }
+, "bazel_types":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["bazel_types"]
+ , "hdrs": ["bazel_types.hpp"]
+ , "deps": [["@", "grpc", "", "grpc++"]]
+ , "proto": [["@", "bazel_remote_apis", "", "remote_execution_proto"]]
+ , "stage": ["src", "buildtool", "common"]
+ }
+, "common":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["common"]
+ , "hdrs":
+ [ "action.hpp"
+ , "artifact_digest.hpp"
+ , "artifact.hpp"
+ , "identifier.hpp"
+ , "statistics.hpp"
+ ]
+ , "deps":
+ [ "bazel_types"
+ , ["src/buildtool/crypto", "hash_generator"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/utils/cpp", "type_safe_arithmetic"]
+ , ["src/utils/cpp", "hash_combine"]
+ , ["@", "json", "", "json"]
+ ]
+ , "stage": ["src", "buildtool", "common"]
+ }
+, "artifact_factory":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["artifact_factory"]
+ , "hdrs": ["artifact_factory.hpp"]
+ , "deps":
+ [ "common"
+ , "artifact_description"
+ , "action_description"
+ , ["src/buildtool/logging", "logging"]
+ , ["src/utils/cpp", "type_safe_arithmetic"]
+ ]
+ , "stage": ["src", "buildtool", "common"]
+ }
+, "artifact_description":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["artifact_description"]
+ , "hdrs": ["artifact_description.hpp"]
+ , "deps":
+ [ "common"
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/logging", "logging"]
+ , ["src/utils/cpp", "json"]
+ ]
+ , "stage": ["src", "buildtool", "common"]
+ }
+, "action_description":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["action_description"]
+ , "hdrs": ["action_description.hpp"]
+ , "deps":
+ [ "common"
+ , "artifact_description"
+ , ["src/buildtool/logging", "logging"]
+ , ["@", "json", "", "json"]
+ ]
+ , "stage": ["src", "buildtool", "common"]
+ }
+, "tree":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["tree"]
+ , "hdrs": ["tree.hpp"]
+ , "deps":
+ [ "common"
+ , "artifact_description"
+ , ["src/buildtool/logging", "logging"]
+ , ["@", "json", "", "json"]
+ ]
+ , "stage": ["src", "buildtool", "common"]
+ }
+, "config":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["config"]
+ , "hdrs": ["repository_config.hpp"]
+ , "deps":
+ [ ["src/buildtool/file_system", "file_root"]
+ , ["src/buildtool/file_system", "git_cas"]
+ ]
+ , "stage": ["src", "buildtool", "common"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/common/action.hpp b/src/buildtool/common/action.hpp
new file mode 100644
index 00000000..b547ecf3
--- /dev/null
+++ b/src/buildtool/common/action.hpp
@@ -0,0 +1,78 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_HPP
+
+#include <map>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "src/buildtool/common/identifier.hpp"
+
+class Action {
+ public:
+ using LocalPath = std::string;
+
+ Action(std::string action_id,
+ std::vector<std::string> command,
+ std::map<std::string, std::string> env_vars,
+ std::optional<std::string> may_fail,
+ bool no_cache)
+ : id_{std::move(action_id)},
+ command_{std::move(command)},
+ env_{std::move(env_vars)},
+ may_fail_{std::move(may_fail)},
+ no_cache_{no_cache} {}
+
+ Action(std::string action_id,
+ std::vector<std::string> command,
+ std::map<std::string, std::string> env_vars)
+ : Action(std::move(action_id),
+ std::move(command),
+ std::move(env_vars),
+ std::nullopt,
+ false) {}
+
+ [[nodiscard]] auto Id() const noexcept -> ActionIdentifier { return id_; }
+
+ [[nodiscard]] auto Command() && noexcept -> std::vector<std::string> {
+ return std::move(command_);
+ }
+
+ [[nodiscard]] auto Command() const& noexcept
+ -> std::vector<std::string> const& {
+ return command_;
+ }
+
+ [[nodiscard]] auto Env() const& noexcept
+ -> std::map<std::string, std::string> {
+ return env_;
+ }
+
+ [[nodiscard]] auto Env() && noexcept -> std::map<std::string, std::string> {
+ return std::move(env_);
+ }
+
+ [[nodiscard]] auto IsTreeAction() const -> bool { return is_tree_; }
+ [[nodiscard]] auto MayFail() const -> std::optional<std::string> {
+ return may_fail_;
+ }
+ [[nodiscard]] auto NoCache() const -> bool { return no_cache_; }
+
+ [[nodiscard]] static auto CreateTreeAction(ActionIdentifier const& id)
+ -> Action {
+ return Action{id};
+ }
+
+ private:
+ ActionIdentifier id_{};
+ std::vector<std::string> command_{};
+ std::map<std::string, std::string> env_{};
+ bool is_tree_{};
+ std::optional<std::string> may_fail_{};
+ bool no_cache_{};
+
+ explicit Action(ActionIdentifier id) : id_{std::move(id)}, is_tree_{true} {}
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_HPP
diff --git a/src/buildtool/common/action_description.hpp b/src/buildtool/common/action_description.hpp
new file mode 100644
index 00000000..9b3469c1
--- /dev/null
+++ b/src/buildtool/common/action_description.hpp
@@ -0,0 +1,200 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_DESCRIPTION_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_DESCRIPTION_HPP
+
+#include <map>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/common/action.hpp"
+#include "src/buildtool/common/artifact_description.hpp"
+
+class ActionDescription {
+ public:
+ using outputs_t = std::vector<std::string>;
+ using inputs_t = std::unordered_map<std::string, ArtifactDescription>;
+
+ ActionDescription(outputs_t output_files,
+ outputs_t output_dirs,
+ Action action,
+ inputs_t inputs)
+ : output_files_{std::move(output_files)},
+ output_dirs_{std::move(output_dirs)},
+ action_{std::move(action)},
+ inputs_{std::move(inputs)} {}
+
+ [[nodiscard]] static auto FromJson(std::string const& id,
+ nlohmann::json const& desc) noexcept
+ -> std::optional<ActionDescription> {
+ try {
+ auto outputs =
+ ExtractValueAs<std::vector<std::string>>(desc, "output");
+ auto output_dirs =
+ ExtractValueAs<std::vector<std::string>>(desc, "output_dirs");
+ auto command =
+ ExtractValueAs<std::vector<std::string>>(desc, "command");
+
+ if ((not outputs.has_value() or outputs->empty()) and
+ (not output_dirs.has_value() or output_dirs->empty())) {
+ Logger::Log(
+ LogLevel::Error,
+ "Action description for action \"{}\" incomplete: values "
+ "for either \"output\" or \"output_dir\" must be non-empty "
+ "array.",
+ id);
+ return std::nullopt;
+ }
+
+ if (not command.has_value() or command->empty()) {
+ Logger::Log(
+ LogLevel::Error,
+ "Action description for action \"{}\" incomplete: values "
+ "for \"command\" must be non-empty array.",
+ id);
+ return std::nullopt;
+ }
+
+ if (not outputs) {
+ outputs = std::vector<std::string>{};
+ }
+ if (not output_dirs) {
+ output_dirs = std::vector<std::string>{};
+ }
+
+ auto optional_key_value_reader =
+ [](nlohmann::json const& action_desc,
+ std::string const& key) -> nlohmann::json {
+ auto it = action_desc.find(key);
+ if (it == action_desc.end()) {
+ return nlohmann::json::object();
+ }
+ return *it;
+ };
+ auto const input = optional_key_value_reader(desc, "input");
+ auto const env = optional_key_value_reader(desc, "env");
+
+ if (not(input.is_object() and env.is_object())) {
+ Logger::Log(
+ LogLevel::Error,
+ "Action description for action \"{}\" type error: values "
+ "for \"input\" and \"env\" must be objects.",
+ id);
+ return std::nullopt;
+ }
+
+ inputs_t inputs{};
+ for (auto const& [path, input_desc] : input.items()) {
+ auto artifact = ArtifactDescription::FromJson(input_desc);
+ if (not artifact) {
+ return std::nullopt;
+ }
+ inputs.emplace(path, std::move(*artifact));
+ }
+ std::optional<std::string> may_fail{};
+ bool no_cache{};
+ auto may_fail_it = desc.find("may_fail");
+ if (may_fail_it != desc.end()) {
+ if (not may_fail_it->is_string()) {
+ Logger::Log(LogLevel::Error,
+ "may_fail has to be a boolean");
+ return std::nullopt;
+ }
+ may_fail = *may_fail_it;
+ }
+ auto no_cache_it = desc.find("no_cache");
+ if (no_cache_it != desc.end()) {
+ if (not no_cache_it->is_boolean()) {
+ Logger::Log(LogLevel::Error,
+ "no_cache has to be a boolean");
+ return std::nullopt;
+ }
+ no_cache = *no_cache_it;
+ }
+
+ return ActionDescription{
+ std::move(*outputs),
+ std::move(*output_dirs),
+ Action{id, std::move(*command), env, may_fail, no_cache},
+ inputs};
+ } catch (std::exception const& ex) {
+ Logger::Log(
+ LogLevel::Error,
+ "Failed to parse action description from JSON with error:\n{}",
+ ex.what());
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto Id() const noexcept -> ActionIdentifier {
+ return action_.Id();
+ }
+
+ [[nodiscard]] auto ToJson() const noexcept -> nlohmann::json {
+ auto json = nlohmann::json{{"command", action_.Command()}};
+ if (not output_files_.empty()) {
+ json["output"] = output_files_;
+ }
+ if (not output_dirs_.empty()) {
+ json["output_dirs"] = output_dirs_;
+ }
+ if (not inputs_.empty()) {
+ auto inputs = nlohmann::json::object();
+ for (auto const& [path, artifact] : inputs_) {
+ inputs[path] = artifact.ToJson();
+ }
+ json["input"] = inputs;
+ }
+ if (not action_.Env().empty()) {
+ json["env"] = action_.Env();
+ }
+ if (action_.MayFail()) {
+ json["may_fail"] = *action_.MayFail();
+ }
+ if (action_.NoCache()) {
+ json["no_cache"] = true;
+ }
+ return json;
+ }
+
+ [[nodiscard]] auto OutputFiles() const& noexcept -> outputs_t const& {
+ return output_files_;
+ }
+
+ [[nodiscard]] auto OutputFiles() && noexcept -> outputs_t {
+ return std::move(output_files_);
+ }
+
+ [[nodiscard]] auto OutputDirs() const& noexcept -> outputs_t const& {
+ return output_dirs_;
+ }
+
+ [[nodiscard]] auto OutputDirs() && noexcept -> outputs_t {
+ return std::move(output_dirs_);
+ }
+
+ [[nodiscard]] auto GraphAction() const& noexcept -> Action const& {
+ return action_;
+ }
+
+ [[nodiscard]] auto GraphAction() && noexcept -> Action {
+ return std::move(action_);
+ }
+
+ [[nodiscard]] auto Inputs() const& noexcept -> inputs_t const& {
+ return inputs_;
+ }
+
+ [[nodiscard]] auto Inputs() && noexcept -> inputs_t {
+ return std::move(inputs_);
+ }
+
+ private:
+ outputs_t output_files_;
+ outputs_t output_dirs_;
+ Action action_;
+ inputs_t inputs_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_ACTION_DESCRIPTION_HPP
diff --git a/src/buildtool/common/artifact.hpp b/src/buildtool/common/artifact.hpp
new file mode 100644
index 00000000..6c97ab24
--- /dev/null
+++ b/src/buildtool/common/artifact.hpp
@@ -0,0 +1,214 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_HPP
+
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/identifier.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+
+// Artifacts (source files, libraries, executables...) need to store their
+// identifier
+class Artifact {
+ public:
+ struct ObjectInfo {
+ ArtifactDigest digest{};
+ ObjectType type{};
+ bool failed{};
+
+ [[nodiscard]] auto operator==(ObjectInfo const& other) const {
+ return (digest == other.digest and type == other.type and
+ failed == other.failed);
+ }
+
+ [[nodiscard]] auto operator!=(ObjectInfo const& other) const {
+ return not(*this == other);
+ }
+
+ // Create string of the form '[hash:size:type]'
+ [[nodiscard]] auto ToString() const noexcept -> std::string {
+ return fmt::format("[{}:{}:{}]{}",
+ digest.hash(),
+ digest.size(),
+ ToChar(type),
+ failed ? " FAILED" : "");
+ }
+
+ // Create JSON of the form '{"id": "hash", "size": x, "file_type": "f"}'
+ // As the failed property is only internal to a run, discard it.
+ [[nodiscard]] auto ToJson() const noexcept -> nlohmann::json {
+ return {{"id", digest.hash()},
+ {"size", digest.size()},
+ {"file_type", std::string{ToChar(type)}}};
+ }
+
+ [[nodiscard]] static auto FromString(std::string const& s) noexcept
+ -> std::optional<ObjectInfo> {
+ std::istringstream iss(s);
+ std::string id{};
+ std::string size_str{};
+ std::string type{};
+ if (not(iss.get() == '[') or not std::getline(iss, id, ':') or
+ not std::getline(iss, size_str, ':') or
+ not std::getline(iss, type, ']')) {
+ Logger::Log(LogLevel::Error,
+ "failed parsing object info from string.");
+ return std::nullopt;
+ }
+ try {
+ std::size_t size = std::stoul(size_str);
+ return ObjectInfo{ArtifactDigest{id, size},
+ FromChar(*type.c_str())};
+ } catch (std::out_of_range const& e) {
+ Logger::Log(LogLevel::Error,
+ "size raised out_of_range exception.");
+ } catch (std::invalid_argument const& e) {
+ Logger::Log(LogLevel::Error,
+ "size raised invalid_argument exception.");
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] static auto FromJson(nlohmann::json const& j)
+ -> std::optional<ObjectInfo> {
+ if (j.is_object() and j["id"].is_string() and
+ j["size"].is_number() and j["type"].is_string()) {
+ return ObjectInfo{
+ ArtifactDigest{j["id"].get<std::string>(),
+ j["size"].get<std::size_t>()},
+ FromChar(*(j["type"].get<std::string>().c_str()))};
+ }
+ return std::nullopt;
+ }
+ };
+
+ explicit Artifact(ArtifactIdentifier id) noexcept : id_{std::move(id)} {}
+
+ Artifact(Artifact const& other) noexcept
+ : id_{other.id_}, file_path_{other.file_path_}, repo_{other.repo_} {
+ object_info_ = other.object_info_;
+ }
+
+ Artifact(Artifact&&) noexcept = default;
+ ~Artifact() noexcept = default;
+ auto operator=(Artifact const&) noexcept -> Artifact& = delete;
+ auto operator=(Artifact&&) noexcept -> Artifact& = default;
+
+ [[nodiscard]] auto Id() const& noexcept -> ArtifactIdentifier const& {
+ return id_;
+ }
+
+ [[nodiscard]] auto Id() && noexcept -> ArtifactIdentifier {
+ return std::move(id_);
+ }
+
+ [[nodiscard]] auto FilePath() const noexcept
+ -> std::optional<std::filesystem::path> {
+ return file_path_;
+ }
+
+ [[nodiscard]] auto Repository() const noexcept -> std::string {
+ return repo_;
+ }
+
+ [[nodiscard]] auto Digest() const noexcept
+ -> std::optional<ArtifactDigest> {
+ return object_info_ ? std::optional{object_info_->digest}
+ : std::nullopt;
+ }
+
+ [[nodiscard]] auto Type() const noexcept -> std::optional<ObjectType> {
+ return object_info_ ? std::optional{object_info_->type} : std::nullopt;
+ }
+
+ [[nodiscard]] auto Info() const& noexcept
+ -> std::optional<ObjectInfo> const& {
+ return object_info_;
+ }
+ [[nodiscard]] auto Info() && noexcept -> std::optional<ObjectInfo> {
+ return std::move(object_info_);
+ }
+
+ void SetObjectInfo(ObjectInfo const& object_info,
+ bool fail_info) const noexcept {
+ if (fail_info) {
+ object_info_ =
+ ObjectInfo{object_info.digest, object_info.type, true};
+ }
+ else {
+ object_info_ = object_info;
+ }
+ }
+
+ void SetObjectInfo(ArtifactDigest const& digest,
+ ObjectType type,
+ bool failed) const noexcept {
+ object_info_ = ObjectInfo{digest, type, failed};
+ }
+
+ [[nodiscard]] static auto CreateLocalArtifact(
+ std::string const& id,
+ std::filesystem::path const& file_path,
+ std::string const& repo) noexcept -> Artifact {
+ return Artifact{id, file_path, repo};
+ }
+
+ [[nodiscard]] static auto CreateKnownArtifact(
+ std::string const& id,
+ std::string const& hash,
+ std::size_t size,
+ ObjectType type,
+ std::optional<std::string> const& repo) noexcept -> Artifact {
+ return Artifact{id, {hash, size}, type, false, repo};
+ }
+
+ [[nodiscard]] static auto CreateActionArtifact(
+ std::string const& id) noexcept -> Artifact {
+ return Artifact{id};
+ }
+
+ private:
+ ArtifactIdentifier id_{};
+ std::optional<std::filesystem::path> file_path_{};
+ std::string repo_{};
+ mutable std::optional<ObjectInfo> object_info_{};
+
+ Artifact(ArtifactIdentifier id,
+ std::filesystem::path const& file_path,
+ std::string repo) noexcept
+ : id_{std::move(id)}, file_path_{file_path}, repo_{std::move(repo)} {}
+
+ Artifact(ArtifactIdentifier id,
+ ArtifactDigest const& digest,
+ ObjectType type,
+ bool failed,
+ std::optional<std::string> repo) noexcept
+ : id_{std::move(id)} {
+ SetObjectInfo(digest, type, failed);
+ if (repo) {
+ repo_ = std::move(*repo);
+ }
+ }
+};
+
+namespace std {
+template <>
+struct hash<Artifact::ObjectInfo> {
+ [[nodiscard]] auto operator()(
+ Artifact::ObjectInfo const& info) const noexcept -> std::size_t {
+ std::size_t seed{};
+ hash_combine(&seed, info.digest);
+ hash_combine(&seed, info.type);
+ hash_combine(&seed, info.failed);
+ return seed;
+ }
+};
+} // namespace std
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_HPP
diff --git a/src/buildtool/common/artifact_description.hpp b/src/buildtool/common/artifact_description.hpp
new file mode 100644
index 00000000..3820edc4
--- /dev/null
+++ b/src/buildtool/common/artifact_description.hpp
@@ -0,0 +1,316 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DESCRIPTION_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DESCRIPTION_HPP
+
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <variant>
+
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/utils/cpp/json.hpp"
+
+class ArtifactDescription {
+ using Local = std::pair<std::filesystem::path, std::string>;
+ using Known =
+ std::tuple<ArtifactDigest, ObjectType, std::optional<std::string>>;
+ using Action = std::pair<std::string, std::filesystem::path>;
+ using Tree = std::string;
+
+ public:
+ explicit ArtifactDescription(std::filesystem::path path,
+ std::string repository) noexcept
+ : data_{std::make_pair(std::move(path), std::move(repository))} {}
+
+ ArtifactDescription(ArtifactDigest digest,
+ ObjectType file_type,
+ std::optional<std::string> repo = std::nullopt) noexcept
+ : data_{
+ std::make_tuple(std::move(digest), file_type, std::move(repo))} {}
+
+ ArtifactDescription(std::string action_id,
+ std::filesystem::path path) noexcept
+ : data_{std::make_pair(std::move(action_id), std::move(path))} {}
+
+ explicit ArtifactDescription(std::string tree_id) noexcept
+ : data_{std::move(tree_id)} {}
+
+ [[nodiscard]] static auto FromJson(nlohmann::json const& json) noexcept
+ -> std::optional<ArtifactDescription> {
+ try {
+ auto const type = ExtractValueAs<std::string>(
+ json, "type", [](std::string const& error) {
+ Logger::Log(
+ LogLevel::Error,
+ "{}\ncan not retrieve value for \"type\" from artifact "
+ "description.",
+ error);
+ });
+ auto const data = ExtractValueAs<nlohmann::json>(
+ json, "data", [](std::string const& error) {
+ Logger::Log(
+ LogLevel::Error,
+ "{}\ncan not retrieve value for \"data\" from artifact "
+ "description.",
+ error);
+ });
+
+ if (not(type and data)) {
+ return std::nullopt;
+ }
+
+ if (*type == "LOCAL") {
+ return CreateLocalArtifactDescription(*data);
+ }
+ if (*type == "KNOWN") {
+ return CreateKnownArtifactDescription(*data);
+ }
+ if (*type == "ACTION") {
+ return CreateActionArtifactDescription(*data);
+ }
+ if (*type == "TREE") {
+ return CreateTreeArtifactDescription(*data);
+ }
+ Logger::Log(LogLevel::Error,
+ R"(artifact type must be one of "LOCAL", "KNOWN",
+ "ACTION", or "TREE")");
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "Failed to parse artifact description from JSON with "
+ "error:\n{}",
+ ex.what());
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto Id() const noexcept -> ArtifactIdentifier { return id_; }
+
+ [[nodiscard]] auto IsTree() const noexcept -> bool {
+ return std::holds_alternative<Tree>(data_);
+ }
+
+ [[nodiscard]] auto ToJson() const noexcept -> nlohmann::json {
+ try {
+ if (std::holds_alternative<Local>(data_)) {
+ auto const& [path, repo] = std::get<Local>(data_);
+ return DescribeLocalArtifact(path.string(), repo);
+ }
+ if (std::holds_alternative<Known>(data_)) {
+ auto const& [digest, file_type, _] = std::get<Known>(data_);
+ return DescribeKnownArtifact(
+ digest.hash(), digest.size(), file_type);
+ }
+ if (std::holds_alternative<Action>(data_)) {
+ auto const& [action_id, path] = std::get<Action>(data_);
+ return DescribeActionArtifact(action_id, path);
+ }
+ if (std::holds_alternative<Tree>(data_)) {
+ return DescribeTreeArtifact(std::get<Tree>(data_));
+ }
+ Logger::Log(LogLevel::Error,
+ "Internal error, unknown artifact type");
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "Serializing to JSON failed with error:\n{}",
+ ex.what());
+ }
+ gsl_Ensures(false); // unreachable
+ return {};
+ }
+
+ [[nodiscard]] auto ToArtifact() const noexcept -> Artifact {
+ try {
+ if (std::holds_alternative<Local>(data_)) {
+ auto const& [path, repo] = std::get<Local>(data_);
+ return Artifact::CreateLocalArtifact(id_, path.string(), repo);
+ }
+ if (std::holds_alternative<Known>(data_)) {
+ auto const& [digest, file_type, repo] = std::get<Known>(data_);
+ return Artifact::CreateKnownArtifact(
+ id_, digest.hash(), digest.size(), file_type, repo);
+ }
+ if (std::holds_alternative<Action>(data_) or
+ std::holds_alternative<Tree>(data_)) {
+ return Artifact::CreateActionArtifact(id_);
+ }
+ Logger::Log(LogLevel::Error,
+ "Internal error, unknown artifact type");
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "Creating artifact failed with error:\n{}",
+ ex.what());
+ }
+ gsl_Ensures(false); // unreachable
+ return Artifact{{}};
+ }
+
+ [[nodiscard]] auto ToString(int indent = 0) const noexcept -> std::string {
+ try {
+ return ToJson().dump(indent);
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "Serializing artifact failed with error:\n{}",
+ ex.what());
+ }
+ return {};
+ }
+
+ [[nodiscard]] auto operator==(
+ ArtifactDescription const& other) const noexcept -> bool {
+ return data_ == other.data_;
+ }
+
+ [[nodiscard]] auto operator!=(
+ ArtifactDescription const& other) const noexcept -> bool {
+ return not(*this == other);
+ }
+
+ private:
+ inline static HashGenerator const hash_gen_{
+ HashGenerator::HashType::SHA256};
+ std::variant<Local, Known, Action, Tree> data_;
+ ArtifactIdentifier id_{ComputeId(ToJson())};
+
+ [[nodiscard]] static auto ComputeId(nlohmann::json const& desc) noexcept
+ -> ArtifactIdentifier {
+ try {
+ return hash_gen_.Run(desc.dump()).Bytes();
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "Computing artifact id failed with error:\n{}",
+ ex.what());
+ }
+ return {};
+ }
+
+ [[nodiscard]] static auto DescribeLocalArtifact(
+ std::filesystem::path const& src_path,
+ std::string const& repository) noexcept -> nlohmann::json {
+ return {{"type", "LOCAL"},
+ {"data",
+ {{"path", src_path.string()}, {"repository", repository}}}};
+ }
+
+ [[nodiscard]] static auto DescribeKnownArtifact(
+ std::string const& blob_id,
+ std::size_t size,
+ ObjectType type = ObjectType::File) noexcept -> nlohmann::json {
+ std::string const typestr{ToChar(type)};
+ return {{"type", "KNOWN"},
+ {"data",
+ {{"id", blob_id}, {"size", size}, {"file_type", typestr}}}};
+ }
+
+ [[nodiscard]] static auto DescribeActionArtifact(
+ std::string const& action_id,
+ std::string const& out_path) noexcept -> nlohmann::json {
+ return {{"type", "ACTION"},
+ {"data", {{"id", action_id}, {"path", out_path}}}};
+ }
+
+ [[nodiscard]] static auto DescribeTreeArtifact(
+ std::string const& tree_id) noexcept -> nlohmann::json {
+ return {{"type", "TREE"}, {"data", {{"id", tree_id}}}};
+ }
+
+ [[nodiscard]] static auto CreateLocalArtifactDescription(
+ nlohmann::json const& data) -> std::optional<ArtifactDescription> {
+
+ auto const path = ExtractValueAs<std::string>(
+ data, "path", [](std::string const& error) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"path\" from "
+ "LOCAL artifact's data.",
+ error);
+ });
+ auto const repository = ExtractValueAs<std::string>(
+ data, "repository", [](std::string const& error) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"path\" from "
+ "LOCAL artifact's data.",
+ error);
+ });
+ if (path.has_value() and repository.has_value()) {
+ return ArtifactDescription{std::filesystem::path{*path},
+ *repository};
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] static auto CreateKnownArtifactDescription(
+ nlohmann::json const& data) -> std::optional<ArtifactDescription> {
+
+ auto const blob_id = ExtractValueAs<std::string>(
+ data, "id", [](std::string const& error) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"id\" from "
+ "KNOWN artifact's data.",
+ error);
+ });
+ auto const size = ExtractValueAs<std::size_t>(
+ data, "size", [](std::string const& error) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"size\" from "
+ "KNOWN artifact's data.",
+ error);
+ });
+ auto const file_type = ExtractValueAs<std::string>(
+ data, "file_type", [](std::string const& error) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"file_type\" from "
+ "KNOWN artifact's data.",
+ error);
+ });
+ if (blob_id.has_value() and size.has_value() and
+ file_type.has_value() and file_type->size() == 1) {
+ return ArtifactDescription{ArtifactDigest{*blob_id, *size},
+ FromChar((*file_type)[0])};
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] static auto CreateActionArtifactDescription(
+ nlohmann::json const& data) -> std::optional<ArtifactDescription> {
+
+ auto const action_id = ExtractValueAs<std::string>(
+ data, "id", [](std::string const& error) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"id\" from "
+ "ACTION artifact's data.",
+ error);
+ });
+
+ auto const path = ExtractValueAs<std::string>(
+ data, "path", [](std::string const& error) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"path\" from "
+ "ACTION artifact's data.",
+ error);
+ });
+ if (action_id.has_value() and path.has_value()) {
+ return ArtifactDescription{*action_id,
+ std::filesystem::path{*path}};
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] static auto CreateTreeArtifactDescription(
+ nlohmann::json const& data) -> std::optional<ArtifactDescription> {
+ auto const tree_id = ExtractValueAs<std::string>(
+ data, "id", [](std::string const& error) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"id\" from "
+ "TREE artifact's data.",
+ error);
+ });
+
+ if (tree_id.has_value()) {
+ return ArtifactDescription{*tree_id};
+ }
+ return std::nullopt;
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_ARTIFACT_DESCRIPTION_HPP
diff --git a/src/buildtool/common/artifact_digest.hpp b/src/buildtool/common/artifact_digest.hpp
new file mode 100644
index 00000000..7499e975
--- /dev/null
+++ b/src/buildtool/common/artifact_digest.hpp
@@ -0,0 +1,74 @@
+#ifndef INCLUDED_SRC_COMMON_ARTIFACT_DIGEST_HPP
+#define INCLUDED_SRC_COMMON_ARTIFACT_DIGEST_HPP
+
+#include <optional>
+#include <string>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/crypto/hash_generator.hpp"
+#include "src/utils/cpp/hash_combine.hpp"
+
+// Wrapper for bazel_re::Digest. Can be implicitly cast to bazel_re::Digest.
+// Provides getter for size with convenient non-protobuf type.
+class ArtifactDigest {
+ public:
+ ArtifactDigest() noexcept = default;
+ explicit ArtifactDigest(bazel_re::Digest digest) noexcept
+ : size_{gsl::narrow<std::size_t>(digest.size_bytes())},
+ digest_{std::move(digest)} {}
+ ArtifactDigest(std::string hash, std::size_t size) noexcept
+ : size_{size}, digest_{CreateBazelDigest(std::move(hash), size)} {}
+
+ [[nodiscard]] auto hash() const& noexcept -> std::string const& {
+ return digest_.hash();
+ }
+
+ [[nodiscard]] auto hash() && noexcept -> std::string {
+ return std::move(*digest_.mutable_hash());
+ }
+
+ [[nodiscard]] auto size() const noexcept -> std::size_t { return size_; }
+
+ // NOLINTNEXTLINE allow implicit casts
+ [[nodiscard]] operator bazel_re::Digest const &() const& { return digest_; }
+ // NOLINTNEXTLINE allow implicit casts
+ [[nodiscard]] operator bazel_re::Digest() && { return std::move(digest_); }
+
+ [[nodiscard]] auto operator==(ArtifactDigest const& other) const -> bool {
+ return std::equal_to<bazel_re::Digest>{}(*this, other);
+ }
+
+ [[nodiscard]] static auto Create(std::string const& content) noexcept
+ -> ArtifactDigest {
+ return ArtifactDigest{ComputeHash(content), content.size()};
+ }
+
+ private:
+ std::size_t size_{};
+ bazel_re::Digest digest_{};
+
+ [[nodiscard]] static auto CreateBazelDigest(std::string&& hash,
+ std::size_t size)
+ -> bazel_re::Digest {
+ bazel_re::Digest d;
+ d.set_hash(std::move(hash));
+ d.set_size_bytes(gsl::narrow<google::protobuf::int64>(size));
+ return d;
+ }
+};
+
+namespace std {
+template <>
+struct hash<ArtifactDigest> {
+ [[nodiscard]] auto operator()(ArtifactDigest const& digest) const noexcept
+ -> std::size_t {
+ std::size_t seed{};
+ hash_combine(&seed, digest.hash());
+ hash_combine(&seed, digest.size());
+ return seed;
+ }
+};
+} // namespace std
+
+#endif // INCLUDED_SRC_COMMON_ARTIFACT_DIGEST_HPP
diff --git a/src/buildtool/common/artifact_factory.hpp b/src/buildtool/common/artifact_factory.hpp
new file mode 100644
index 00000000..e3ef2d0c
--- /dev/null
+++ b/src/buildtool/common/artifact_factory.hpp
@@ -0,0 +1,91 @@
+#ifndef INCLUDED_SRC_COMMON_ARTIFACT_FACTORY_HPP
+#define INCLUDED_SRC_COMMON_ARTIFACT_FACTORY_HPP
+
+#include <algorithm>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "src/buildtool/common/action_description.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/common/identifier.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/utils/cpp/json.hpp"
+
+class ArtifactFactory {
+ public:
+ [[nodiscard]] static auto Identifier(nlohmann::json const& description)
+ -> ArtifactIdentifier {
+ auto desc = ArtifactDescription::FromJson(description);
+ if (desc) {
+ return desc->Id();
+ }
+ return {};
+ }
+
+ [[nodiscard]] static auto FromDescription(nlohmann::json const& description)
+ -> std::optional<Artifact> {
+ auto desc = ArtifactDescription::FromJson(description);
+ if (desc) {
+ return desc->ToArtifact();
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] static auto DescribeLocalArtifact(
+ std::filesystem::path const& src_path,
+ std::string repository) noexcept -> nlohmann::json {
+ return ArtifactDescription{src_path, std::move(repository)}.ToJson();
+ }
+
+ [[nodiscard]] static auto DescribeKnownArtifact(
+ std::string const& blob_id,
+ std::size_t size,
+ ObjectType type = ObjectType::File) noexcept -> nlohmann::json {
+ return ArtifactDescription{ArtifactDigest{blob_id, size}, type}
+ .ToJson();
+ }
+
+ [[nodiscard]] static auto DescribeActionArtifact(
+ std::string const& action_id,
+ std::string const& out_path) noexcept -> nlohmann::json {
+ return ArtifactDescription{action_id, std::filesystem::path{out_path}}
+ .ToJson();
+ }
+
+ [[nodiscard]] static auto DescribeTreeArtifact(
+ std::string const& tree_id) noexcept -> nlohmann::json {
+ return ArtifactDescription{tree_id}.ToJson();
+ }
+
+ [[nodiscard]] static auto DescribeAction(
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::vector<std::string> const& command) noexcept -> nlohmann::json {
+ return ActionDescription{
+ output_files, output_dirs, Action{"unused", command, {}}, {}}
+ .ToJson();
+ }
+
+ [[nodiscard]] static auto DescribeAction(
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::vector<std::string> const& command,
+ ActionDescription::inputs_t const& input,
+ std::map<std::string, std::string> const& env) noexcept
+ -> nlohmann::json {
+ return ActionDescription{
+ output_files, output_dirs, Action{"unused", command, env}, input}
+ .ToJson();
+ }
+
+ [[nodiscard]] static auto IsLocal(nlohmann::json const& description)
+ -> bool {
+ return description.at("type") == "LOCAL";
+ }
+}; // class ArtifactFactory
+
+#endif // INCLUDED_SRC_COMMON_ARTIFACT_FACTORY_HPP
diff --git a/src/buildtool/common/bazel_types.hpp b/src/buildtool/common/bazel_types.hpp
new file mode 100644
index 00000000..b7d64409
--- /dev/null
+++ b/src/buildtool/common/bazel_types.hpp
@@ -0,0 +1,86 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_TYPES_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_TYPES_HPP
+
+/// \file bazel_types.hpp
+/// \brief This file contains commonly used aliases for Bazel API
+/// Never include this file in any other header!
+
+#ifdef BOOTSTRAP_BUILD_TOOL
+
+namespace build::bazel::remote::execution::v2 {
+struct Digest {
+ std::string hash_;
+ int64_t size_bytes_;
+
+ auto hash() const& noexcept -> std::string const& { return hash_; }
+
+ auto size_bytes() const noexcept -> int64_t { return size_bytes_; }
+
+ void set_size_bytes(int64_t size_bytes) { size_bytes_ = size_bytes; }
+
+ void set_hash(std::string hash) { hash_ = hash; }
+
+ std::string* mutable_hash() { return &hash_; }
+};
+} // namespace build::bazel::remote::execution::v2
+
+namespace google::protobuf {
+using int64 = int64_t;
+}
+
+#else
+
+#include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h"
+
+#endif
+
+/// \brief Alias namespace for bazel remote execution
+// NOLINTNEXTLINE(misc-unused-alias-decls)
+namespace bazel_re = build::bazel::remote::execution::v2;
+
+#ifdef BOOTSTRAP_BUILD_TOOL
+// not using protobuffers
+#else
+
+/// \brief Alias namespace for 'google::protobuf'
+namespace pb {
+// NOLINTNEXTLINE(google-build-using-namespace)
+using namespace google::protobuf;
+
+/// \brief Alias function for 'RepeatedFieldBackInserter'
+template <typename T>
+auto back_inserter(RepeatedField<T>* const f) {
+ return RepeatedFieldBackInserter(f);
+}
+
+/// \brief Alias function for 'RepeatedPtrFieldBackInserter'
+template <typename T>
+auto back_inserter(RepeatedPtrField<T>* const f) {
+ return RepeatedPtrFieldBackInserter(f);
+}
+
+} // namespace pb
+#endif
+
+namespace std {
+
+/// \brief Hash function to support bazel_re::Digest as std::map* key.
+template <>
+struct hash<bazel_re::Digest> {
+ auto operator()(bazel_re::Digest const& d) const noexcept -> std::size_t {
+ return std::hash<std::string>{}(d.hash());
+ }
+};
+
+/// \brief Equality function to support bazel_re::Digest as std::map* key.
+template <>
+struct equal_to<bazel_re::Digest> {
+ auto operator()(bazel_re::Digest const& lhs,
+ bazel_re::Digest const& rhs) const noexcept -> bool {
+ return lhs.hash() == rhs.hash();
+ }
+};
+
+} // namespace std
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_BAZEL_TYPES_HPP
diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp
new file mode 100644
index 00000000..b4710147
--- /dev/null
+++ b/src/buildtool/common/cli.hpp
@@ -0,0 +1,365 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP
+
+#include <cstdlib>
+#include <filesystem>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "CLI/CLI.hpp"
+#include "fmt/core.h"
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/logging/log_level.hpp"
+
+constexpr auto kDefaultLogLevel = LogLevel::Info;
+
+/// \brief Arguments common to all commands.
+struct CommonArguments {
+ std::optional<std::filesystem::path> workspace_root{};
+ std::optional<std::filesystem::path> repository_config{};
+ std::optional<std::string> main{};
+ std::optional<std::filesystem::path> log_file{};
+ std::size_t jobs{std::max(1U, std::thread::hardware_concurrency())};
+ LogLevel log_limit{kDefaultLogLevel};
+};
+
+/// \brief Arguments required for analysing targets.
+struct AnalysisArguments {
+ std::filesystem::path config_file{};
+ std::optional<nlohmann::json> target{};
+ std::optional<std::string> target_file_name{};
+ std::optional<std::string> rule_file_name{};
+ std::optional<std::string> expression_file_name{};
+ std::optional<std::filesystem::path> target_root{};
+ std::optional<std::filesystem::path> rule_root{};
+ std::optional<std::filesystem::path> expression_root{};
+ std::optional<std::filesystem::path> graph_file{};
+ std::optional<std::filesystem::path> artifacts_to_build_file{};
+};
+
+/// \brief Arguments required for running diagnostics.
+struct DiagnosticArguments {
+ std::optional<std::string> dump_actions{std::nullopt};
+ std::optional<std::string> dump_blobs{std::nullopt};
+ std::optional<std::string> dump_trees{std::nullopt};
+ std::optional<std::string> dump_targets{std::nullopt};
+ std::optional<std::string> dump_anonymous{std::nullopt};
+ std::optional<std::string> dump_nodes{std::nullopt};
+};
+
+/// \brief Arguments required for specifying cache/build endpoint.
+struct EndpointArguments {
+ std::optional<std::filesystem::path> local_root{};
+ std::optional<std::string> remote_execution_address;
+};
+
+/// \brief Arguments required for building.
+struct BuildArguments {
+ std::optional<std::vector<std::string>> local_launcher{std::nullopt};
+ std::map<std::string, std::string> platform_properties;
+ std::size_t build_jobs{};
+ std::optional<std::string> dump_artifacts{std::nullopt};
+ std::optional<std::string> print_to_stdout{std::nullopt};
+ bool persistent_build_dir{false};
+ bool show_runfiles{false};
+};
+
+/// \brief Arguments required for staging.
+struct StageArguments {
+ std::filesystem::path output_dir{};
+};
+
+/// \brief Arguments required for rebuilding.
+struct RebuildArguments {
+ std::optional<std::string> cache_endpoint{};
+ std::optional<std::filesystem::path> dump_flaky{};
+};
+
+/// \brief Arguments for fetching artifacts from CAS.
+struct FetchArguments {
+ std::string object_id{};
+ std::optional<std::filesystem::path> output_path{};
+};
+
+/// \brief Arguments required for running from graph file.
+struct GraphArguments {
+ nlohmann::json artifacts{};
+ std::filesystem::path graph_file{};
+ std::optional<std::filesystem::path> git_cas{};
+};
+
+static inline auto SetupCommonArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommonArguments*> const& clargs) {
+ app->add_option("-C,--repository_config",
+ clargs->repository_config,
+ "Path to configuration file for multi-repository builds.")
+ ->type_name("PATH");
+ app->add_option(
+ "--main", clargs->main, "The repository to take the target from.")
+ ->type_name("NAME");
+ app->add_option_function<std::string>(
+ "-w,--workspace_root",
+ [clargs](auto const& workspace_root_raw) {
+ clargs->workspace_root = std::filesystem::canonical(
+ std::filesystem::absolute(workspace_root_raw));
+ },
+ "Path of the workspace's root directory.")
+ ->type_name("PATH");
+ app->add_option("-j,--jobs",
+ clargs->jobs,
+ "Number of jobs to run (Default: Number of cores).")
+ ->type_name("NUM");
+ app->add_option(
+ "-f,--log-file", clargs->log_file, "Path to local log file.")
+ ->type_name("PATH");
+ app->add_option_function<std::underlying_type_t<LogLevel>>(
+ "--log-limit",
+ [clargs](auto const& limit) {
+ clargs->log_limit = ToLogLevel(limit);
+ },
+ fmt::format("Log limit in interval [{},{}] (Default: {}).",
+ kFirstLogLevel,
+ kLastLogLevel,
+ kDefaultLogLevel))
+ ->type_name("NUM");
+}
+
+static inline auto SetupAnalysisArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<AnalysisArguments*> const& clargs,
+ bool with_graph = true) {
+ app->add_option(
+ "-c,--config", clargs->config_file, "Path to configuration file.")
+ ->type_name("PATH");
+ app->add_option_function<std::vector<std::string>>(
+ "target",
+ [clargs](auto const& target_raw) {
+ if (target_raw.size() > 1) {
+ clargs->target =
+ nlohmann::json{target_raw[0], target_raw[1]};
+ }
+ else {
+ clargs->target = nlohmann::json{target_raw[0]}[0];
+ }
+ },
+ "Module and target name to build.\n"
+ "Assumes current module if module name is omitted.")
+ ->expected(2);
+ app->add_option("--target_root",
+ clargs->target_root,
+ "Path of the target files' root directory.\n"
+ "Default: Same as --workspace_root")
+ ->type_name("PATH");
+ app->add_option("--rule_root",
+ clargs->rule_root,
+ "Path of the rule files' root directory.\n"
+ "Default: Same as --target_root")
+ ->type_name("PATH");
+ app->add_option("--expression_root",
+ clargs->expression_root,
+ "Path of the expression files' root directory.\n"
+ "Default: Same as --rule_root")
+ ->type_name("PATH");
+ app->add_option("--target_file_name",
+ clargs->target_file_name,
+ "Name of the targets file.");
+ app->add_option(
+ "--rule_file_name", clargs->rule_file_name, "Name of the rules file.");
+ app->add_option("--expression_file_name",
+ clargs->expression_file_name,
+ "Name of the expressions file.");
+ if (with_graph) {
+ app->add_option(
+ "--dump_graph",
+ clargs->graph_file,
+ "File path for writing the action graph description to.")
+ ->type_name("PATH");
+ app->add_option("--dump_artifacts_to_build",
+ clargs->artifacts_to_build_file,
+ "File path for writing the artifacts to build to.")
+ ->type_name("PATH");
+ }
+}
+
+static inline auto SetupDiagnosticArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<DiagnosticArguments*> const& clargs) {
+ app->add_option("--dump_actions",
+ clargs->dump_actions,
+ "Dump actions to file (use - for stdout).")
+ ->type_name("PATH");
+ app->add_option("--dump_trees",
+ clargs->dump_trees,
+ "Dump trees to file (use - for stdout).")
+ ->type_name("PATH");
+ app->add_option("--dump_blobs",
+ clargs->dump_blobs,
+ "Dump blobs to file (use - for stdout).")
+ ->type_name("PATH");
+ app->add_option("--dump_targets",
+ clargs->dump_targets,
+ "Dump targets to file (use - for stdout).")
+ ->type_name("PATH");
+ app->add_option("--dump_anonymous",
+ clargs->dump_anonymous,
+ "Dump anonymous targets to file (use - for stdout).")
+ ->type_name("PATH");
+ app->add_option("--dump_nodes",
+ clargs->dump_nodes,
+ "Dump nodes of target to file (use - for stdout).")
+ ->type_name("PATH");
+}
+
+static inline auto SetupEndpointArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<EndpointArguments*> const& clargs) {
+ app->add_option_function<std::string>(
+ "--local_build_root",
+ [clargs](auto const& build_root_raw) {
+ clargs->local_root = std::filesystem::weakly_canonical(
+ std::filesystem::absolute(build_root_raw));
+ },
+ "Root for local CAS, cache, and build directories.")
+ ->type_name("PATH");
+
+ app->add_option("-r,--remote_execution_address",
+ clargs->remote_execution_address,
+ "Address of the remote execution service.")
+ ->type_name("NAME:PORT");
+}
+
+static inline auto SetupBuildArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<BuildArguments*> const& clargs) {
+ app->add_flag("-p,--persistent",
+ clargs->persistent_build_dir,
+ "Do not clean build directory after execution.");
+
+ app->add_option_function<std::string>(
+ "-L,--local_launcher",
+ [clargs](auto const& launcher_raw) {
+ clargs->local_launcher =
+ nlohmann::json::parse(launcher_raw)
+ .template get<std::vector<std::string>>();
+ },
+ "JSON array with the list of strings representing the launcher to "
+ "prepend actions' commands before being executed locally.")
+ ->type_name("JSON")
+ ->default_val(nlohmann::json{"env", "--"}.dump());
+
+ app->add_option_function<std::string>(
+ "--remote_execution_property",
+ [clargs](auto const& property) {
+ std::istringstream pss(property);
+ std::string key;
+ std::string val;
+ if (not std::getline(pss, key, ':') or
+ not std::getline(pss, val, ':')) {
+ throw CLI::ConversionError{property,
+ "--remote_execution_property"};
+ }
+ clargs->platform_properties.emplace(std::move(key),
+ std::move(val));
+ },
+ "Property for remote execution as key-value pair.")
+ ->type_name("KEY:VAL")
+ ->allow_extra_args(false)
+ ->expected(1, 1);
+
+ app->add_option(
+ "-J,--build_jobs",
+ clargs->build_jobs,
+ "Number of jobs to run during build phase (Default: same as jobs).")
+ ->type_name("NUM");
+ app->add_option("--dump_artifacts",
+ clargs->dump_artifacts,
+ "Dump artifacts to file (use - for stdout).")
+ ->type_name("PATH");
+
+ app->add_flag("-s,--show_runfiles",
+ clargs->show_runfiles,
+ "Do not omit runfiles in build report.");
+
+ app->add_option("-P,--print_to_stdout",
+ clargs->print_to_stdout,
+ "After building, print the specified artifact to stdout.")
+ ->type_name("LOGICAL_PATH");
+}
+
+static inline auto SetupStageArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<StageArguments*> const& clargs) {
+ app->add_option_function<std::string>(
+ "-o,--output_dir",
+ [clargs](auto const& output_dir_raw) {
+ clargs->output_dir = std::filesystem::weakly_canonical(
+ std::filesystem::absolute(output_dir_raw));
+ },
+ "Path of the directory where outputs will be copied.")
+ ->type_name("PATH")
+ ->required();
+}
+
+static inline auto SetupRebuildArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<RebuildArguments*> const& clargs) {
+ app->add_option_function<std::string>(
+ "--vs",
+ [clargs](auto const& cache_endpoint) {
+ clargs->cache_endpoint = cache_endpoint;
+ },
+ "Cache endpoint to compare against (use \"local\" for local cache).")
+ ->type_name("NAME:PORT|\"local\"");
+
+ app->add_option(
+ "--dump_flaky", clargs->dump_flaky, "Dump flaky actions to file.")
+ ->type_name("PATH");
+}
+
+static inline auto SetupFetchArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<FetchArguments*> const& clargs) {
+ app->add_option(
+ "object_id",
+ clargs->object_id,
+ "Object identifier with the format '[<hash>:<size>:<type>]'.")
+ ->required();
+
+ app->add_option_function<std::string>(
+ "-o,--output_path",
+ [clargs](auto const& output_path_raw) {
+ clargs->output_path = std::filesystem::weakly_canonical(
+ std::filesystem::absolute(output_path_raw));
+ },
+ "Install path for the artifact. (omit to dump to stdout)")
+ ->type_name("PATH");
+}
+
+static inline auto SetupGraphArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<GraphArguments*> const& clargs) {
+ app->add_option_function<std::string>(
+ "-a,--artifacts",
+ [clargs](auto const& artifact_map_raw) {
+ clargs->artifacts = nlohmann::json::parse(artifact_map_raw);
+ },
+ "Json object with key/value pairs formed by the relative path in which "
+ "artifact is to be copied and the description of the artifact as json "
+ "object as well.");
+
+ app->add_option("-g,--graph_file",
+ clargs->graph_file,
+ "Path of the file containing the description of the "
+ "actions.")
+ ->required();
+
+ app->add_option("--git_cas",
+ clargs->git_cas,
+ "Path to a Git repository, containing blobs of potentially "
+ "missing KNOWN artifacts.");
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_CLI_HPP
diff --git a/src/buildtool/common/identifier.hpp b/src/buildtool/common/identifier.hpp
new file mode 100644
index 00000000..1d9875fd
--- /dev/null
+++ b/src/buildtool/common/identifier.hpp
@@ -0,0 +1,25 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_IDENTIFIER_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_IDENTIFIER_HPP
+
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+// Global artifact identifier (not the CAS hash)
+using ArtifactIdentifier = std::string;
+
+// Global action identifier
+using ActionIdentifier = std::string;
+
+static inline auto IdentifierToString(const std::string& id) -> std::string {
+ std::ostringstream encoded{};
+ encoded << std::hex << std::setfill('0');
+ for (auto const& b : id) {
+ encoded << std::setw(2)
+ << static_cast<int>(
+ static_cast<std::make_unsigned_t<
+ std::remove_reference_t<decltype(b)>>>(b));
+ }
+ return encoded.str();
+}
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_IDENTIFIER_HPP
diff --git a/src/buildtool/common/repository_config.hpp b/src/buildtool/common/repository_config.hpp
new file mode 100644
index 00000000..62c2c4ff
--- /dev/null
+++ b/src/buildtool/common/repository_config.hpp
@@ -0,0 +1,133 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_REPOSITORY_CONFIG_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_REPOSITORY_CONFIG_HPP
+
+#include <filesystem>
+#include <string>
+#include <unordered_map>
+
+#include "src/buildtool/file_system/file_root.hpp"
+#include "src/buildtool/file_system/git_cas.hpp"
+
+class RepositoryConfig {
+ public:
+ struct RepositoryInfo {
+ FileRoot workspace_root;
+ FileRoot target_root{workspace_root};
+ FileRoot rule_root{target_root};
+ FileRoot expression_root{rule_root};
+ std::unordered_map<std::string, std::string> name_mapping{};
+ std::string target_file_name{"TARGETS"};
+ std::string rule_file_name{"RULES"};
+ std::string expression_file_name{"EXPRESSIONS"};
+ };
+
+ [[nodiscard]] static auto Instance() noexcept -> RepositoryConfig& {
+ static RepositoryConfig instance{};
+ return instance;
+ }
+
+ void SetInfo(std::string const& repo, RepositoryInfo&& info) {
+ infos_.emplace(repo, std::move(info));
+ }
+
+ [[nodiscard]] auto SetGitCAS(
+ std::filesystem::path const& repo_path) noexcept {
+ git_cas_ = GitCAS::Open(repo_path);
+ return static_cast<bool>(git_cas_);
+ }
+
+ [[nodiscard]] auto Info(std::string const& repo) const noexcept
+ -> RepositoryInfo const* {
+ auto it = infos_.find(repo);
+ if (it != infos_.end()) {
+ return &it->second;
+ }
+ return nullptr;
+ }
+
+ [[nodiscard]] auto ReadBlobFromGitCAS(std::string const& hex_id)
+ const noexcept -> std::optional<std::string> {
+ return git_cas_ ? git_cas_->ReadObject(hex_id, /*is_hex_id=*/true)
+ : std::nullopt;
+ }
+
+ [[nodiscard]] auto WorkspaceRoot(std::string const& repo) const noexcept
+ -> FileRoot const* {
+ return Get<FileRoot>(
+ repo, [](auto const& info) { return &info.workspace_root; });
+ }
+
+ [[nodiscard]] auto TargetRoot(std::string const& repo) const noexcept
+ -> FileRoot const* {
+ return Get<FileRoot>(
+ repo, [](auto const& info) { return &info.target_root; });
+ }
+
+ [[nodiscard]] auto RuleRoot(std::string const& repo) const
+ -> FileRoot const* {
+ return Get<FileRoot>(repo,
+ [](auto const& info) { return &info.rule_root; });
+ }
+
+ [[nodiscard]] auto ExpressionRoot(std::string const& repo) const noexcept
+ -> FileRoot const* {
+ return Get<FileRoot>(
+ repo, [](auto const& info) { return &info.expression_root; });
+ }
+
+ [[nodiscard]] auto GlobalName(std::string const& repo,
+ std::string const& local_name) const noexcept
+ -> std::string const* {
+ return Get<std::string>(
+ repo, [&local_name](auto const& info) -> std::string const* {
+ auto it = info.name_mapping.find(local_name);
+ if (it != info.name_mapping.end()) {
+ return &it->second;
+ }
+ return nullptr;
+ });
+ }
+
+ [[nodiscard]] auto TargetFileName(std::string const& repo) const noexcept
+ -> std::string const* {
+ return Get<std::string>(
+ repo, [](auto const& info) { return &info.target_file_name; });
+ }
+
+ [[nodiscard]] auto RuleFileName(std::string const& repo) const noexcept
+ -> std::string const* {
+ return Get<std::string>(
+ repo, [](auto const& info) { return &info.rule_file_name; });
+ }
+
+ [[nodiscard]] auto ExpressionFileName(
+ std::string const& repo) const noexcept -> std::string const* {
+ return Get<std::string>(
+ repo, [](auto const& info) { return &info.expression_file_name; });
+ }
+
+ // used for testing
+ void Reset() {
+ infos_.clear();
+ git_cas_.reset();
+ }
+
+ private:
+ std::unordered_map<std::string, RepositoryInfo> infos_;
+ GitCASPtr git_cas_;
+
+ template <class T>
+ [[nodiscard]] auto Get(std::string const& repo,
+ std::function<T const*(RepositoryInfo const&)> const&
+ getter) const noexcept -> T const* {
+ if (auto const* info = Info(repo)) {
+ try { // satisfy clang-tidy's bugprone-exception-escape
+ return getter(*info);
+ } catch (...) {
+ }
+ }
+ return nullptr;
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_REPOSITORY_CONFIG_HPP
diff --git a/src/buildtool/common/statistics.hpp b/src/buildtool/common/statistics.hpp
new file mode 100644
index 00000000..a7b89791
--- /dev/null
+++ b/src/buildtool/common/statistics.hpp
@@ -0,0 +1,61 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_STATISTICS_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_STATISTICS_HPP
+
+#include <atomic>
+
+class Statistics {
+ public:
+ [[nodiscard]] static auto Instance() noexcept -> Statistics& {
+ static Statistics instance{};
+ return instance;
+ }
+
+ void Reset() noexcept {
+ num_actions_queued_ = 0;
+ num_actions_cached_ = 0;
+ num_actions_flaky_ = 0;
+ num_actions_flaky_tainted_ = 0;
+ num_rebuilt_actions_compared_ = 0;
+ num_rebuilt_actions_missing_ = 0;
+ }
+ void IncrementActionsQueuedCounter() noexcept { ++num_actions_queued_; }
+ void IncrementActionsCachedCounter() noexcept { ++num_actions_cached_; }
+ void IncrementActionsFlakyCounter() noexcept { ++num_actions_flaky_; }
+ void IncrementActionsFlakyTaintedCounter() noexcept {
+ ++num_actions_flaky_tainted_;
+ }
+ void IncrementRebuiltActionMissingCounter() noexcept {
+ ++num_rebuilt_actions_missing_;
+ }
+ void IncrementRebuiltActionComparedCounter() noexcept {
+ ++num_rebuilt_actions_compared_;
+ }
+ [[nodiscard]] auto ActionsQueuedCounter() const noexcept -> int {
+ return num_actions_queued_;
+ }
+ [[nodiscard]] auto ActionsCachedCounter() const noexcept -> int {
+ return num_actions_cached_;
+ }
+ [[nodiscard]] auto ActionsFlakyCounter() const noexcept -> int {
+ return num_actions_flaky_;
+ }
+ [[nodiscard]] auto ActionsFlakyTaintedCounter() const noexcept -> int {
+ return num_actions_flaky_tainted_;
+ }
+ [[nodiscard]] auto RebuiltActionMissingCounter() const noexcept -> int {
+ return num_rebuilt_actions_missing_;
+ }
+ [[nodiscard]] auto RebuiltActionComparedCounter() const noexcept -> int {
+ return num_rebuilt_actions_compared_;
+ }
+
+ private:
+ std::atomic<int> num_actions_queued_{};
+ std::atomic<int> num_actions_cached_{};
+ std::atomic<int> num_actions_flaky_{};
+ std::atomic<int> num_actions_flaky_tainted_{};
+ std::atomic<int> num_rebuilt_actions_missing_{};
+ std::atomic<int> num_rebuilt_actions_compared_{};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_STATISTICS_HPP
diff --git a/src/buildtool/common/tree.hpp b/src/buildtool/common/tree.hpp
new file mode 100644
index 00000000..512eda86
--- /dev/null
+++ b/src/buildtool/common/tree.hpp
@@ -0,0 +1,72 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_COMMON_TREE_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMMON_TREE_HPP
+
+#include <string>
+#include <unordered_map>
+
+#include "nlohmann/json.hpp"
+#include "src/buildtool/common/action_description.hpp"
+#include "src/buildtool/common/artifact_description.hpp"
+
+// Describes tree, its inputs, output (tree artifact), and action (tree action).
+class Tree {
+ using inputs_t = ActionDescription::inputs_t;
+
+ public:
+ explicit Tree(inputs_t&& inputs)
+ : id_{ComputeId(inputs)}, inputs_{std::move(inputs)} {}
+
+ [[nodiscard]] auto Id() const& -> std::string const& { return id_; }
+ [[nodiscard]] auto Id() && -> std::string { return std::move(id_); }
+
+ [[nodiscard]] auto ToJson() const -> nlohmann::json {
+ return ComputeDescription(inputs_);
+ }
+
+ [[nodiscard]] auto Inputs() const -> inputs_t { return inputs_; }
+
+ [[nodiscard]] auto Action() const -> ActionDescription {
+ return {
+ {/*unused*/}, {/*unused*/}, Action::CreateTreeAction(id_), inputs_};
+ }
+
+ [[nodiscard]] auto Output() const -> ArtifactDescription {
+ return ArtifactDescription{id_};
+ }
+
+ [[nodiscard]] static auto FromJson(std::string const& id,
+ nlohmann::json const& json)
+ -> std::optional<Tree> {
+ auto inputs = inputs_t{};
+ inputs.reserve(json.size());
+ for (auto const& [path, artifact] : json.items()) {
+ auto artifact_desc = ArtifactDescription::FromJson(artifact);
+ if (not artifact_desc) {
+ return std::nullopt;
+ }
+ inputs.emplace(path, std::move(*artifact_desc));
+ }
+ return Tree{id, std::move(inputs)};
+ }
+
+ private:
+ std::string id_;
+ inputs_t inputs_;
+
+ Tree(std::string id, inputs_t&& inputs)
+ : id_{std::move(id)}, inputs_{std::move(inputs)} {}
+
+ static auto ComputeDescription(inputs_t const& inputs) -> nlohmann::json {
+ auto json = nlohmann::json::object();
+ for (auto const& [path, artifact] : inputs) {
+ json[path] = artifact.ToJson();
+ }
+ return json;
+ }
+
+ static auto ComputeId(inputs_t const& inputs) -> std::string {
+ return ComputeHash(ComputeDescription(inputs).dump());
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_COMMON_TREE_HPP
diff --git a/src/buildtool/crypto/TARGETS b/src/buildtool/crypto/TARGETS
new file mode 100644
index 00000000..d68758ed
--- /dev/null
+++ b/src/buildtool/crypto/TARGETS
@@ -0,0 +1,31 @@
+{ "hash_impl":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["hash_impl"]
+ , "hdrs":
+ [ "hash_impl.hpp"
+ , "hash_impl_md5.hpp"
+ , "hash_impl_sha1.hpp"
+ , "hash_impl_sha256.hpp"
+ , "hash_impl_git.hpp"
+ ]
+ , "srcs":
+ [ "hash_impl_md5.cpp"
+ , "hash_impl_sha1.cpp"
+ , "hash_impl_sha256.cpp"
+ , "hash_impl_git.cpp"
+ ]
+ , "deps":
+ [ ["src/buildtool/logging", "logging"]
+ , ["src/utils/cpp", "hex_string"]
+ , ["@", "ssl", "", "crypto"]
+ ]
+ , "stage": ["src", "buildtool", "crypto"]
+ }
+, "hash_generator":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["hash_generator"]
+ , "hdrs": ["hash_generator.hpp"]
+ , "deps": ["hash_impl"]
+ , "stage": ["src", "buildtool", "crypto"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/crypto/hash_generator.hpp b/src/buildtool/crypto/hash_generator.hpp
new file mode 100644
index 00000000..453a9c06
--- /dev/null
+++ b/src/buildtool/crypto/hash_generator.hpp
@@ -0,0 +1,130 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_GENERATOR_HPP
+#define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_GENERATOR_HPP
+
+#include <iomanip>
+#include <memory>
+#include <optional>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "src/buildtool/crypto/hash_impl_git.hpp"
+#include "src/buildtool/crypto/hash_impl_md5.hpp"
+#include "src/buildtool/crypto/hash_impl_sha1.hpp"
+#include "src/buildtool/crypto/hash_impl_sha256.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+/// \brief Hash generator, supports multiple types \ref HashType.
+class HashGenerator {
+ public:
+ /// \brief Types of hash implementations supported by generator.
+ enum class HashType { MD5, SHA1, SHA256, GIT };
+
+ /// \brief The universal hash digest.
+ /// The type of hash and the digest length depends on the hash
+ /// implementation used to generated this digest.
+ class HashDigest {
+ friend HashGenerator;
+
+ public:
+ /// \brief Get pointer to raw bytes of digest.
+ /// Length can be obtained using \ref Length.
+ [[nodiscard]] auto Bytes() const -> std::string const& {
+ return bytes_;
+ }
+
+ /// \brief Get hexadecimal string of digest.
+ /// Length is twice the length of raw bytes (\ref Length).
+ [[nodiscard]] auto HexString() const -> std::string {
+ return ToHexString(bytes_);
+ }
+
+ /// \brief Get digest length in raw bytes.
+ [[nodiscard]] auto Length() const -> std::size_t {
+ return bytes_.size();
+ }
+
+ private:
+ std::string bytes_{};
+
+ explicit HashDigest(std::string bytes) : bytes_{std::move(bytes)} {}
+ };
+
+ /// \brief Incremental hasher.
+ class Hasher {
+ friend HashGenerator;
+
+ public:
+ /// \brief Feed data to the hasher.
+ auto Update(std::string const& data) noexcept -> bool {
+ return impl_->Update(data);
+ }
+
+ /// \brief Finalize hash.
+ [[nodiscard]] auto Finalize() && noexcept -> std::optional<HashDigest> {
+ auto hash = std::move(*impl_).Finalize();
+ if (hash) {
+ return HashDigest{*hash};
+ }
+ return std::nullopt;
+ }
+
+ private:
+ std::unique_ptr<IHashImpl> impl_;
+
+ explicit Hasher(std::unique_ptr<IHashImpl> impl)
+ : impl_{std::move(impl)} {}
+ };
+
+ /// \brief Create hash generator for specific type.
+ explicit HashGenerator(HashType type)
+ : type_{type}, digest_length_{create_impl()->DigestLength()} {}
+ HashGenerator(HashGenerator const&) = delete;
+ HashGenerator(HashGenerator&&) = delete;
+ auto operator=(HashGenerator const&) -> HashGenerator& = delete;
+ auto operator=(HashGenerator &&) -> HashGenerator& = delete;
+ ~HashGenerator() noexcept = default;
+
+ /// \brief Run hash function on data.
+ [[nodiscard]] auto Run(std::string const& data) const noexcept
+ -> HashDigest {
+ auto impl = create_impl();
+ return HashDigest{std::move(*impl).Compute(data)};
+ }
+
+ [[nodiscard]] auto IncrementalHasher() const noexcept -> Hasher {
+ return Hasher(create_impl());
+ }
+
+ [[nodiscard]] auto DigestLength() const noexcept -> std::size_t {
+ return digest_length_;
+ }
+
+ private:
+ HashType type_{};
+ std::size_t digest_length_{};
+
+ /// \brief Dispatch for creating the actual implementation
+ [[nodiscard]] auto create_impl() const noexcept
+ -> std::unique_ptr<IHashImpl> {
+ switch (type_) {
+ case HashType::MD5:
+ return CreateHashImplMd5();
+ case HashType::SHA1:
+ return CreateHashImplSha1();
+ case HashType::SHA256:
+ return CreateHashImplSha256();
+ case HashType::GIT:
+ return CreateHashImplGit();
+ }
+ }
+};
+
+/// \brief Hash function used for the entire buildtool
+[[maybe_unused]] [[nodiscard]] static inline auto ComputeHash(
+ std::string const& data) noexcept -> std::string {
+ static HashGenerator gen{HashGenerator::HashType::GIT};
+ return gen.Run(data).HexString();
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_GENERATOR_HPP
diff --git a/src/buildtool/crypto/hash_impl.hpp b/src/buildtool/crypto/hash_impl.hpp
new file mode 100644
index 00000000..aa5a9559
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl.hpp
@@ -0,0 +1,40 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_HPP
+#define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_HPP
+
+#include <optional>
+#include <string>
+
+#include "src/buildtool/logging/logger.hpp"
+
+/// \brief Interface for hash implementations
+class IHashImpl {
+ public:
+ IHashImpl() noexcept = default;
+ IHashImpl(IHashImpl const&) = default;
+ IHashImpl(IHashImpl&&) = default;
+ auto operator=(IHashImpl const&) -> IHashImpl& = default;
+ auto operator=(IHashImpl &&) -> IHashImpl& = default;
+ virtual ~IHashImpl() = default;
+
+ /// \brief Feed data to the incremental hashing.
+ [[nodiscard]] virtual auto Update(std::string const& data) noexcept
+ -> bool = 0;
+
+ /// \brief Finalize the hashing and return hash as string of raw bytes.
+ [[nodiscard]] virtual auto Finalize() && noexcept
+ -> std::optional<std::string> = 0;
+
+ /// \brief Compute the hash of data and return it as string of raw bytes.
+ [[nodiscard]] virtual auto Compute(std::string const& data) && noexcept
+ -> std::string = 0;
+
+ /// \brief Get length of the hash in raw bytes.
+ [[nodiscard]] virtual auto DigestLength() const noexcept -> std::size_t = 0;
+
+ static auto FatalError() noexcept -> void {
+ Logger::Log(LogLevel::Error, "Failed to compute hash.");
+ std::terminate();
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_HPP
diff --git a/src/buildtool/crypto/hash_impl_git.cpp b/src/buildtool/crypto/hash_impl_git.cpp
new file mode 100644
index 00000000..9cb2a761
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl_git.cpp
@@ -0,0 +1,42 @@
+#include <array>
+#include <cstdint>
+
+#include "openssl/sha.h"
+#include "src/buildtool/crypto/hash_impl.hpp"
+
+/// \brief Hash implementation for Git blob ids.
+/// Does not support incremental hashing.
+class HashImplGit final : public IHashImpl {
+ public:
+ auto Update(std::string const& /*data*/) noexcept -> bool final {
+ return false;
+ }
+
+ auto Finalize() && noexcept -> std::optional<std::string> final {
+ return std::nullopt;
+ }
+
+ auto Compute(std::string const& data) && noexcept -> std::string final {
+ SHA_CTX ctx;
+ std::string const header{"blob " + std::to_string(data.size()) + '\0'};
+ if (SHA1_Init(&ctx) == 1 &&
+ SHA1_Update(&ctx, header.data(), header.size()) == 1 &&
+ SHA1_Update(&ctx, data.data(), data.size()) == 1) {
+ auto out = std::array<std::uint8_t, SHA_DIGEST_LENGTH>{};
+ if (SHA1_Final(out.data(), &ctx) == 1) {
+ return std::string{out.begin(), out.end()};
+ }
+ }
+ FatalError();
+ return {};
+ }
+
+ [[nodiscard]] auto DigestLength() const noexcept -> std::size_t final {
+ return SHA_DIGEST_LENGTH;
+ }
+};
+
+/// \brief Factory for Git implementation
+auto CreateHashImplGit() -> std::unique_ptr<IHashImpl> {
+ return std::make_unique<HashImplGit>();
+}
diff --git a/src/buildtool/crypto/hash_impl_git.hpp b/src/buildtool/crypto/hash_impl_git.hpp
new file mode 100644
index 00000000..be0738da
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl_git.hpp
@@ -0,0 +1,10 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_GIT_HPP
+#define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_GIT_HPP
+
+#include <memory>
+
+#include "src/buildtool/crypto/hash_impl.hpp"
+
+[[nodiscard]] extern auto CreateHashImplGit() -> std::unique_ptr<IHashImpl>;
+
+#endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_GIT_HPP
diff --git a/src/buildtool/crypto/hash_impl_md5.cpp b/src/buildtool/crypto/hash_impl_md5.cpp
new file mode 100644
index 00000000..106dc984
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl_md5.cpp
@@ -0,0 +1,50 @@
+#include <array>
+#include <cstdint>
+
+#include "openssl/md5.h"
+#include "src/buildtool/crypto/hash_impl.hpp"
+
+/// \brief Hash implementation for MD5
+class HashImplMd5 final : public IHashImpl {
+ public:
+ HashImplMd5() { initialized_ = MD5_Init(&ctx_) == 1; }
+
+ auto Update(std::string const& data) noexcept -> bool final {
+ return initialized_ and
+ MD5_Update(&ctx_, data.data(), data.size()) == 1;
+ }
+
+ auto Finalize() && noexcept -> std::optional<std::string> final {
+ if (initialized_) {
+ auto out = std::array<std::uint8_t, MD5_DIGEST_LENGTH>{};
+ if (MD5_Final(out.data(), &ctx_) == 1) {
+ return std::string{out.begin(), out.end()};
+ }
+ }
+ return std::nullopt;
+ }
+
+ auto Compute(std::string const& data) && noexcept -> std::string final {
+ if (Update(data)) {
+ auto digest = std::move(*this).Finalize();
+ if (digest) {
+ return *digest;
+ }
+ }
+ FatalError();
+ return {};
+ }
+
+ [[nodiscard]] auto DigestLength() const noexcept -> std::size_t final {
+ return MD5_DIGEST_LENGTH;
+ }
+
+ private:
+ MD5_CTX ctx_{};
+ bool initialized_{};
+};
+
+/// \brief Factory for MD5 implementation
+auto CreateHashImplMd5() -> std::unique_ptr<IHashImpl> {
+ return std::make_unique<HashImplMd5>();
+}
diff --git a/src/buildtool/crypto/hash_impl_md5.hpp b/src/buildtool/crypto/hash_impl_md5.hpp
new file mode 100644
index 00000000..95411570
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl_md5.hpp
@@ -0,0 +1,10 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_MD5_HPP
+#define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_MD5_HPP
+
+#include <memory>
+
+#include "src/buildtool/crypto/hash_impl.hpp"
+
+[[nodiscard]] extern auto CreateHashImplMd5() -> std::unique_ptr<IHashImpl>;
+
+#endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_MD5_HPP
diff --git a/src/buildtool/crypto/hash_impl_sha1.cpp b/src/buildtool/crypto/hash_impl_sha1.cpp
new file mode 100644
index 00000000..e0bee0fb
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl_sha1.cpp
@@ -0,0 +1,50 @@
+#include <array>
+#include <cstdint>
+
+#include "openssl/sha.h"
+#include "src/buildtool/crypto/hash_impl.hpp"
+
+/// \brief Hash implementation for SHA-1
+class HashImplSha1 final : public IHashImpl {
+ public:
+ HashImplSha1() { initialized_ = SHA1_Init(&ctx_) == 1; }
+
+ auto Update(std::string const& data) noexcept -> bool final {
+ return initialized_ and
+ SHA1_Update(&ctx_, data.data(), data.size()) == 1;
+ }
+
+ auto Finalize() && noexcept -> std::optional<std::string> final {
+ if (initialized_) {
+ auto out = std::array<std::uint8_t, SHA_DIGEST_LENGTH>{};
+ if (SHA1_Final(out.data(), &ctx_) == 1) {
+ return std::string{out.begin(), out.end()};
+ }
+ }
+ return std::nullopt;
+ }
+
+ auto Compute(std::string const& data) && noexcept -> std::string final {
+ if (Update(data)) {
+ auto digest = std::move(*this).Finalize();
+ if (digest) {
+ return *digest;
+ }
+ }
+ FatalError();
+ return {};
+ }
+
+ [[nodiscard]] auto DigestLength() const noexcept -> std::size_t final {
+ return SHA_DIGEST_LENGTH;
+ }
+
+ private:
+ SHA_CTX ctx_{};
+ bool initialized_{};
+};
+
+/// \brief Factory for SHA-1 implementation
+auto CreateHashImplSha1() -> std::unique_ptr<IHashImpl> {
+ return std::make_unique<HashImplSha1>();
+}
diff --git a/src/buildtool/crypto/hash_impl_sha1.hpp b/src/buildtool/crypto/hash_impl_sha1.hpp
new file mode 100644
index 00000000..7b8196b5
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl_sha1.hpp
@@ -0,0 +1,10 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_SHA1_HPP
+#define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_SHA1_HPP
+
+#include <memory>
+
+#include "src/buildtool/crypto/hash_impl.hpp"
+
+[[nodiscard]] extern auto CreateHashImplSha1() -> std::unique_ptr<IHashImpl>;
+
+#endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_SHA1_HPP
diff --git a/src/buildtool/crypto/hash_impl_sha256.cpp b/src/buildtool/crypto/hash_impl_sha256.cpp
new file mode 100644
index 00000000..bbea10d3
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl_sha256.cpp
@@ -0,0 +1,50 @@
+#include <array>
+#include <cstdint>
+
+#include "openssl/sha.h"
+#include "src/buildtool/crypto/hash_impl.hpp"
+
+/// \brief Hash implementation for SHA-256
+class HashImplSha256 final : public IHashImpl {
+ public:
+ HashImplSha256() { initialized_ = SHA256_Init(&ctx_) == 1; }
+
+ auto Update(std::string const& data) noexcept -> bool final {
+ return initialized_ and
+ SHA256_Update(&ctx_, data.data(), data.size()) == 1;
+ }
+
+ auto Finalize() && noexcept -> std::optional<std::string> final {
+ if (initialized_) {
+ auto out = std::array<std::uint8_t, SHA256_DIGEST_LENGTH>{};
+ if (SHA256_Final(out.data(), &ctx_) == 1) {
+ return std::string{out.begin(), out.end()};
+ }
+ }
+ return std::nullopt;
+ }
+
+ auto Compute(std::string const& data) && noexcept -> std::string final {
+ if (Update(data)) {
+ auto digest = std::move(*this).Finalize();
+ if (digest) {
+ return *digest;
+ }
+ }
+ FatalError();
+ return {};
+ }
+
+ [[nodiscard]] auto DigestLength() const noexcept -> std::size_t final {
+ return SHA256_DIGEST_LENGTH;
+ }
+
+ private:
+ SHA256_CTX ctx_{};
+ bool initialized_{};
+};
+
+/// \brief Factory for SHA-256 implementation
+auto CreateHashImplSha256() -> std::unique_ptr<IHashImpl> {
+ return std::make_unique<HashImplSha256>();
+}
diff --git a/src/buildtool/crypto/hash_impl_sha256.hpp b/src/buildtool/crypto/hash_impl_sha256.hpp
new file mode 100644
index 00000000..d74c1492
--- /dev/null
+++ b/src/buildtool/crypto/hash_impl_sha256.hpp
@@ -0,0 +1,10 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_SHA256_HPP
+#define INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_SHA256_HPP
+
+#include <memory>
+
+#include "src/buildtool/crypto/hash_impl.hpp"
+
+[[nodiscard]] extern auto CreateHashImplSha256() -> std::unique_ptr<IHashImpl>;
+
+#endif // INCLUDED_SRC_BUILDTOOL_CRYPTO_HASH_IMPL_SHA256_HPP
diff --git a/src/buildtool/execution_api/TARGETS b/src/buildtool/execution_api/TARGETS
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/buildtool/execution_api/TARGETS
@@ -0,0 +1 @@
+{} \ No newline at end of file
diff --git a/src/buildtool/execution_api/bazel_msg/TARGETS b/src/buildtool/execution_api/bazel_msg/TARGETS
new file mode 100644
index 00000000..19fe1277
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/TARGETS
@@ -0,0 +1,31 @@
+{ "bazel_msg":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["bazel_msg"]
+ , "hdrs": ["bazel_blob.hpp", "bazel_blob_container.hpp", "bazel_common.hpp"]
+ , "deps":
+ [ ["src/buildtool/crypto", "hash_generator"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/utils/cpp", "concepts"]
+ , ["src/utils/cpp", "type_safe_arithmetic"]
+ , ["@", "grpc", "", "grpc++"]
+ ]
+ , "proto": [["@", "bazel_remote_apis", "", "remote_execution_proto"]]
+ , "stage": ["src", "buildtool", "execution_api", "bazel_msg"]
+ }
+, "bazel_msg_factory":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["bazel_msg_factory"]
+ , "hdrs": ["bazel_msg_factory.hpp"]
+ , "srcs": ["bazel_msg_factory.cpp"]
+ , "deps":
+ [ "bazel_msg"
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/execution_engine/dag", "dag"]
+ , ["@", "grpc", "", "grpc++"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "proto": [["@", "bazel_remote_apis", "", "remote_execution_proto"]]
+ , "stage": ["src", "buildtool", "execution_api", "bazel_msg"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp b/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp
new file mode 100644
index 00000000..115c4018
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/bazel_blob.hpp
@@ -0,0 +1,31 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_BLOB_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_BLOB_HPP
+
+#include <filesystem>
+#include <memory>
+#include <optional>
+#include <string>
+
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+
+struct BazelBlob {
+ BazelBlob(bazel_re::Digest mydigest, std::string mydata)
+ : digest{std::move(mydigest)}, data{std::move(mydata)} {}
+
+ bazel_re::Digest digest{};
+ std::string data{};
+};
+
+[[nodiscard]] static inline auto CreateBlobFromFile(
+ std::filesystem::path const& file_path) noexcept
+ -> std::optional<BazelBlob> {
+ auto const content = FileSystemManager::ReadFile(file_path);
+ if (not content.has_value()) {
+ return std::nullopt;
+ }
+ return BazelBlob{ArtifactDigest::Create(*content), *content};
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_BLOB_HPP
diff --git a/src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp b/src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp
new file mode 100644
index 00000000..7005f129
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp
@@ -0,0 +1,264 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_BLOB_CONTAINER_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_BLOB_CONTAINER_HPP
+
+#include <string>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
+#include "src/utils/cpp/concepts.hpp"
+
+namespace detail {
+
+// Interface for transforming iteratee for wrapped_iterators
+template <class T_Iteratee, class T_Iterator>
+struct wrapped_iterator_transform {
+ [[nodiscard]] virtual auto operator()(T_Iterator const&) const& noexcept
+ -> T_Iteratee const& = 0;
+ [[nodiscard]] auto operator()(T_Iterator const&) && noexcept = delete;
+};
+
+// Wrap iterator from read-only container with custom transform. This class
+// represents a read-only iterable view with an implicit transform operation.
+template <class T_Iteratee,
+ class T_Iterator,
+ derived_from<wrapped_iterator_transform<T_Iteratee, T_Iterator>>
+ T_Transform>
+class wrapped_iterator {
+ public:
+ wrapped_iterator(T_Iterator it, T_Transform&& transform) noexcept
+ : it_{std::move(it)}, transform_{std::move(transform)} {}
+ wrapped_iterator(wrapped_iterator const& other) noexcept = default;
+ wrapped_iterator(wrapped_iterator&& other) noexcept = default;
+ ~wrapped_iterator() noexcept = default;
+
+ auto operator=(wrapped_iterator const& other) noexcept
+ -> wrapped_iterator& = default;
+ auto operator=(wrapped_iterator&& other) noexcept
+ -> wrapped_iterator& = default;
+
+ auto operator++() noexcept -> wrapped_iterator& {
+ ++it_;
+ return *this;
+ }
+
+ auto operator++(int) noexcept -> wrapped_iterator {
+ wrapped_iterator r = *this;
+ ++(*this);
+ return r;
+ }
+
+ [[nodiscard]] auto operator==(wrapped_iterator other) const noexcept
+ -> bool {
+ return it_ == other.it_;
+ }
+ [[nodiscard]] auto operator!=(wrapped_iterator other) const noexcept
+ -> bool {
+ return not(*this == other);
+ }
+ [[nodiscard]] auto operator*() const noexcept -> T_Iteratee const& {
+ return transform_(it_);
+ }
+ using difference_type = typename T_Iterator::difference_type;
+ using value_type = T_Iteratee;
+ using pointer = T_Iteratee const*;
+ using reference = T_Iteratee const&;
+ using iterator_category = std::forward_iterator_tag;
+
+ private:
+ T_Iterator it_;
+ T_Transform transform_;
+};
+
+} // namespace detail
+
+/// \brief Container for Blobs
+/// Can be used to iterate over digests or subset of blobs with certain digest.
+class BlobContainer {
+ using underlaying_map_t = std::unordered_map<bazel_re::Digest, BazelBlob>;
+ using item_iterator = underlaying_map_t::const_iterator;
+
+ // transform underlaying_map_t::value_type to BazelBlob
+ struct item_to_blob
+ : public detail::wrapped_iterator_transform<BazelBlob, item_iterator> {
+ public:
+ auto operator()(item_iterator const& it) const& noexcept
+ -> BazelBlob const& final {
+ return it->second;
+ }
+ };
+
+ public:
+ class iterator : public detail::wrapped_iterator<BazelBlob,
+ item_iterator,
+ item_to_blob> {
+ friend class BlobContainer;
+ explicit iterator(item_iterator const& it) noexcept
+ : wrapped_iterator{it, item_to_blob{}} {}
+ };
+
+ /// \brief Iterable read-only list for Digests
+ class DigestList {
+ friend class BlobContainer;
+
+ // transform underlaying_map_t::value_type to Digest
+ struct item_to_digest
+ : public detail::wrapped_iterator_transform<bazel_re::Digest,
+ item_iterator> {
+ public:
+ auto operator()(item_iterator const& it) const& noexcept
+ -> bazel_re::Digest const& final {
+ return it->first;
+ }
+ };
+
+ public:
+ /// \brief Read-only iterator for DigestList
+ class iterator : public detail::wrapped_iterator<bazel_re::Digest,
+ item_iterator,
+ item_to_digest> {
+ public:
+ explicit iterator(item_iterator const& it) noexcept
+ : wrapped_iterator{it, item_to_digest{}} {}
+ };
+
+ /// \brief Obtain start iterator for DigestList
+ [[nodiscard]] auto begin() const noexcept -> iterator {
+ return iterator(blobs_->cbegin());
+ }
+
+ /// \brief Obtain end iterator for DigestList
+ [[nodiscard]] auto end() const noexcept -> iterator {
+ return iterator(blobs_->cend());
+ }
+
+ private:
+ gsl::not_null<underlaying_map_t const*> blobs_;
+
+ explicit DigestList(underlaying_map_t const& blobs) noexcept
+ : blobs_{&blobs} {}
+ };
+
+ /// \brief Iterable read-only list for Blobs related to given Digests
+ class RelatedBlobList {
+ friend class BlobContainer;
+ using digest_iterator = std::vector<bazel_re::Digest>::const_iterator;
+
+ // transform Digest to BazelBlob
+ struct digest_to_blob
+ : public detail::wrapped_iterator_transform<BazelBlob,
+ digest_iterator> {
+ public:
+ explicit digest_to_blob(
+ gsl::not_null<underlaying_map_t const*> blobs) noexcept
+ : blobs_{std::move(blobs)} {}
+ digest_to_blob(digest_to_blob const& other) noexcept = default;
+ digest_to_blob(digest_to_blob&& other) noexcept = default;
+ ~digest_to_blob() noexcept = default;
+
+ auto operator=(digest_to_blob const& other) noexcept
+ -> digest_to_blob& = default;
+ auto operator=(digest_to_blob&& other) noexcept
+ -> digest_to_blob& = default;
+
+ auto operator()(digest_iterator const& it) const& noexcept
+ -> BazelBlob const& final {
+ try {
+ return blobs_->at(*it);
+ } catch (std::exception const&) {
+ return kEmpty;
+ }
+ }
+
+ private:
+ static inline BazelBlob kEmpty{bazel_re::Digest{}, std::string{}};
+ gsl::not_null<underlaying_map_t const*> blobs_;
+ };
+
+ public:
+ /// \brief Read-only iterator for RelatedBlobList
+ class iterator : public detail::wrapped_iterator<BazelBlob,
+ digest_iterator,
+ digest_to_blob> {
+ public:
+ iterator(
+ digest_iterator const& it,
+ gsl::not_null<underlaying_map_t const*> const& blobs) noexcept
+ : wrapped_iterator{it, digest_to_blob{blobs}} {}
+ };
+
+ /// \brief Obtain start iterator for RelatedBlobList
+ [[nodiscard]] auto begin() const noexcept -> iterator {
+ return iterator(digests_.cbegin(), blobs_);
+ }
+
+ /// \brief Obtain end iterator for RelatedBlobList
+ [[nodiscard]] auto end() const noexcept -> iterator {
+ return iterator(digests_.cend(), blobs_);
+ }
+
+ private:
+ std::vector<bazel_re::Digest> digests_;
+ gsl::not_null<underlaying_map_t const*> blobs_;
+
+ RelatedBlobList(underlaying_map_t const& blobs,
+ std::vector<bazel_re::Digest> digests) noexcept
+ : digests_{std::move(digests)}, blobs_{&blobs} {}
+ };
+
+ BlobContainer() noexcept = default;
+ explicit BlobContainer(std::vector<BazelBlob> blobs) {
+ blobs_.reserve(blobs.size());
+ for (auto& blob : blobs) {
+ blobs_.emplace(blob.digest, std::move(blob));
+ }
+ }
+
+ /// \brief Emplace new BazelBlob to container.
+ void Emplace(BazelBlob&& blob) {
+ blobs_.emplace(blob.digest, std::move(blob));
+ }
+
+ /// \brief Clear all BazelBlobs from container.
+ void Clear() noexcept { return blobs_.clear(); }
+
+ /// \brief Number of BazelBlobs in container.
+ [[nodiscard]] auto Size() const noexcept -> std::size_t {
+ return blobs_.size();
+ }
+
+ /// \brief Is equivalent BazelBlob (with same Digest) in container.
+ /// \param[in] blob BazelBlob to search equivalent BazelBlob for
+ [[nodiscard]] auto Contains(BazelBlob const& blob) const noexcept -> bool {
+ return blobs_.contains(blob.digest);
+ }
+
+ /// \brief Obtain iterable list of Digests from container.
+ [[nodiscard]] auto Digests() const noexcept -> DigestList {
+ return DigestList{blobs_};
+ }
+
+ /// \brief Obtain iterable list of BazelBlobs related to Digests.
+ /// \param[in] related Related Digests
+ [[nodiscard]] auto RelatedBlobs(
+ std::vector<bazel_re::Digest> const& related) const noexcept
+ -> RelatedBlobList {
+ return RelatedBlobList{blobs_, related};
+ };
+
+ [[nodiscard]] auto begin() const noexcept -> iterator {
+ return iterator{blobs_.begin()};
+ }
+
+ [[nodiscard]] auto end() const noexcept -> iterator {
+ return iterator{blobs_.end()};
+ }
+
+ private:
+ underlaying_map_t blobs_{};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_BLOB_CONTAINER_HPP
diff --git a/src/buildtool/execution_api/bazel_msg/bazel_common.hpp b/src/buildtool/execution_api/bazel_msg/bazel_common.hpp
new file mode 100644
index 00000000..cc76541c
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/bazel_common.hpp
@@ -0,0 +1,21 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_COMMON_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_COMMON_HPP
+
+/// \file bazel_common.hpp
+/// \brief Common types and functions required by Bazel API.
+
+#include <cstdint>
+
+#include "src/utils/cpp/type_safe_arithmetic.hpp"
+
+// Port
+struct PortTag : type_safe_arithmetic_tag<std::uint16_t> {};
+using Port = type_safe_arithmetic<PortTag>;
+
+struct ExecutionConfiguration {
+ int execution_priority{};
+ int results_cache_priority{};
+ bool skip_cache_lookup{};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_COMMON_HPP
diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp
new file mode 100644
index 00000000..8fad8cae
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.cpp
@@ -0,0 +1,590 @@
+#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
+
+#include <algorithm>
+#include <exception>
+#include <filesystem>
+#include <functional>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <variant>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+
+namespace {
+
+/// \brief Abstract interface for bundle (message, content, and digest).
+/// Provides getters for content, corresponding digest, and creating a blob.
+class IBundle {
+ public:
+ using Ptr = std::unique_ptr<IBundle>;
+ using ContentCreateFunc = std::function<std::optional<std::string>()>;
+ using DigestCreateFunc =
+ std::function<bazel_re::Digest(std::string const&)>;
+
+ IBundle() = default;
+ IBundle(IBundle const&) = delete;
+ IBundle(IBundle&&) = delete;
+ auto operator=(IBundle const&) -> IBundle& = delete;
+ auto operator=(IBundle &&) -> IBundle& = delete;
+ virtual ~IBundle() noexcept = default;
+
+ [[nodiscard]] virtual auto Content() const& noexcept
+ -> std::string const& = 0;
+ [[nodiscard]] virtual auto Digest() const& noexcept
+ -> bazel_re::Digest const& = 0;
+ [[nodiscard]] auto MakeBlob() const noexcept -> BazelBlob {
+ return BazelBlob{Digest(), Content()};
+ }
+};
+
+/// \brief Sparse Bundle implementation for protobuf messages.
+/// It is called "Sparse" as it does not contain its own Digest. Instead, the
+/// protobuf message's Digest is used.
+/// \tparam T The actual protobuf message type.
+template <typename T>
+class SparseBundle final : public IBundle {
+ public:
+ using Ptr = std::unique_ptr<SparseBundle<T>>;
+
+ [[nodiscard]] auto Message() const noexcept -> T const& { return msg_; }
+
+ [[nodiscard]] auto Content() const& noexcept -> std::string const& final {
+ return content_;
+ }
+
+ [[nodiscard]] auto Digest() const& noexcept
+ -> bazel_re::Digest const& final {
+ return msg_.digest();
+ }
+
+ [[nodiscard]] static auto Create(T const& msg,
+ ContentCreateFunc const& content_creator,
+ DigestCreateFunc const& digest_creator)
+ -> Ptr {
+ auto content = content_creator();
+ if (content) {
+ // create bundle with message and content
+ Ptr bundle{new SparseBundle<T>{msg, std::move(*content)}};
+
+ // create digest
+ bundle->msg_.set_allocated_digest(gsl::owner<bazel_re::Digest*>{
+ new bazel_re::Digest{digest_creator(bundle->content_)}});
+ return bundle;
+ }
+ return Ptr{};
+ }
+
+ SparseBundle(SparseBundle const&) = delete;
+ SparseBundle(SparseBundle&&) = delete;
+ auto operator=(SparseBundle const&) -> SparseBundle& = delete;
+ auto operator=(SparseBundle &&) -> SparseBundle& = delete;
+ ~SparseBundle() noexcept final = default;
+
+ private:
+ T msg_{}; /**< Protobuf message */
+ std::string content_{}; /**< Content the message's digest refers to */
+
+ explicit SparseBundle(T msg, std::string&& content)
+ : msg_{std::move(msg)}, content_{std::move(content)} {}
+};
+
+/// \brief Full Bundle implementation for protobuf messages.
+/// Contains its own Digest memory, as the protobuf message does not contain
+/// one itself.
+/// \tparam T The actual protobuf message type.
+template <typename T>
+class FullBundle final : public IBundle {
+ public:
+ using Ptr = std::unique_ptr<FullBundle<T>>;
+
+ [[nodiscard]] auto Message() const noexcept -> T const& { return msg_; }
+
+ auto Content() const& noexcept -> std::string const& final {
+ return content_;
+ }
+
+ auto Digest() const& noexcept -> bazel_re::Digest const& final {
+ return digest_;
+ }
+
+ [[nodiscard]] static auto Create(T const& msg,
+ ContentCreateFunc const& content_creator,
+ DigestCreateFunc const& digest_creator)
+ -> Ptr {
+ auto content = content_creator();
+ if (content) {
+ // create bundle with message and content
+ Ptr bundle{new FullBundle<T>{msg, std::move(*content)}};
+
+ // create digest
+ bundle->digest_ = digest_creator(bundle->content_);
+ return bundle;
+ }
+ return Ptr{};
+ }
+
+ FullBundle(FullBundle const&) = delete;
+ FullBundle(FullBundle&&) = delete;
+ auto operator=(FullBundle const&) -> FullBundle& = delete;
+ auto operator=(FullBundle &&) -> FullBundle& = delete;
+ ~FullBundle() noexcept final = default;
+
+ private:
+ T msg_{}; /**< Protobuf message */
+ bazel_re::Digest digest_{}; /**< Digest of content */
+ std::string content_{}; /**< Content the digest refers to */
+
+ explicit FullBundle(T msg, std::string&& content)
+ : msg_{std::move(msg)}, content_{std::move(content)} {}
+};
+
+using DirectoryNodeBundle = SparseBundle<bazel_re::DirectoryNode>;
+using SymlinkNodeBundle = FullBundle<bazel_re::SymlinkNode>;
+using ActionBundle = FullBundle<bazel_re::Action>;
+using CommandBundle = FullBundle<bazel_re::Command>;
+
+/// \brief Serialize protobuf message to string.
+template <class T>
+[[nodiscard]] auto SerializeMessage(T const& message) noexcept
+ -> std::optional<std::string> {
+ try {
+ std::string content(message.ByteSizeLong(), '\0');
+ message.SerializeToArray(content.data(), content.size());
+ return content;
+ } catch (...) {
+ }
+ return std::nullopt;
+}
+
+/// \brief Create protobuf message 'Platform'.
+[[nodiscard]] auto CreatePlatform(
+ std::vector<bazel_re::Platform_Property> const& props) noexcept
+ -> std::unique_ptr<bazel_re::Platform> {
+ auto platform = std::make_unique<bazel_re::Platform>();
+ std::copy(props.cbegin(),
+ props.cend(),
+ pb::back_inserter(platform->mutable_properties()));
+ return platform;
+}
+
+/// \brief Create protobuf message 'Directory'.
+[[nodiscard]] auto CreateDirectory(
+ std::vector<bazel_re::FileNode> const& files,
+ std::vector<bazel_re::DirectoryNode> const& dirs,
+ std::vector<bazel_re::SymlinkNode> const& links,
+ std::vector<bazel_re::NodeProperty> const& props) noexcept
+ -> bazel_re::Directory {
+ bazel_re::Directory dir{};
+
+ auto copy_nodes = [](auto* pb_container, auto const& nodes) {
+ pb_container->Reserve(nodes.size());
+ std::copy(nodes.begin(), nodes.end(), pb::back_inserter(pb_container));
+ std::sort(
+ pb_container->begin(),
+ pb_container->end(),
+ [](auto const& l, auto const& r) { return l.name() < r.name(); });
+ };
+
+ copy_nodes(dir.mutable_files(), files);
+ copy_nodes(dir.mutable_directories(), dirs);
+ copy_nodes(dir.mutable_symlinks(), links);
+
+ std::copy(props.cbegin(),
+ props.cend(),
+ pb::back_inserter(dir.mutable_node_properties()));
+
+ return dir;
+}
+
+/// \brief Create protobuf message 'FileNode' without digest.
+[[nodiscard]] auto CreateFileNode(
+ std::string const& file_name,
+ ObjectType type,
+ std::vector<bazel_re::NodeProperty> const& props) noexcept
+ -> bazel_re::FileNode {
+ bazel_re::FileNode node;
+ node.set_name(file_name);
+ node.set_is_executable(IsExecutableObject(type));
+ std::copy(props.cbegin(),
+ props.cend(),
+ pb::back_inserter(node.mutable_node_properties()));
+ return node;
+}
+
+/// \brief Create protobuf message 'DirectoryNode' without digest.
+[[nodiscard]] auto CreateDirectoryNode(std::string const& dir_name) noexcept
+ -> bazel_re::DirectoryNode {
+ bazel_re::DirectoryNode node;
+ node.set_name(dir_name);
+ return node;
+}
+
+/// \brief Create profobuf message FileNode from Artifact::ObjectInfo
+[[nodiscard]] auto CreateFileNodeFromObjectInfo(
+ std::string const& name,
+ Artifact::ObjectInfo const& object_info) noexcept -> bazel_re::FileNode {
+ auto file_node = CreateFileNode(name, object_info.type, {});
+
+ file_node.set_allocated_digest(gsl::owner<bazel_re::Digest*>{
+ new bazel_re::Digest{object_info.digest}});
+
+ return file_node;
+}
+
+/// \brief Create profobuf message DirectoryNode from Artifact::ObjectInfo
+[[nodiscard]] auto CreateDirectoryNodeFromObjectInfo(
+ std::string const& name,
+ Artifact::ObjectInfo const& object_info) noexcept
+ -> bazel_re::DirectoryNode {
+ auto dir_node = CreateDirectoryNode(name);
+
+ dir_node.set_allocated_digest(gsl::owner<bazel_re::Digest*>{
+ new bazel_re::Digest{object_info.digest}});
+
+ return dir_node;
+}
+
+/// \brief Create bundle for profobuf message DirectoryNode from Directory.
+[[nodiscard]] auto CreateDirectoryNodeBundle(std::string const& dir_name,
+ bazel_re::Directory const& dir)
+ -> DirectoryNodeBundle::Ptr {
+ // setup protobuf message except digest
+ auto msg = CreateDirectoryNode(dir_name);
+ auto content_creator = [&dir] { return SerializeMessage(dir); };
+ auto digest_creator = [](std::string const& content) -> bazel_re::Digest {
+ return ArtifactDigest::Create(content);
+ };
+ return DirectoryNodeBundle::Create(msg, content_creator, digest_creator);
+}
+
+/// \brief Create bundle for profobuf message Command from args strings.
+[[nodiscard]] auto CreateCommandBundle(
+ std::vector<std::string> const& args,
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::vector<bazel_re::Command_EnvironmentVariable> const& env_vars,
+ std::vector<bazel_re::Platform_Property> const& platform_properties)
+ -> CommandBundle::Ptr {
+ bazel_re::Command msg;
+ msg.set_allocated_platform(CreatePlatform(platform_properties).release());
+ std::copy(std::cbegin(args),
+ std::cend(args),
+ pb::back_inserter(msg.mutable_arguments()));
+ std::copy(std::cbegin(output_files),
+ std::cend(output_files),
+ pb::back_inserter(msg.mutable_output_files()));
+ std::copy(std::cbegin(output_dirs),
+ std::cend(output_dirs),
+ pb::back_inserter(msg.mutable_output_directories()));
+ std::copy(std::cbegin(env_vars),
+ std::cend(env_vars),
+ pb::back_inserter(msg.mutable_environment_variables()));
+
+ auto content_creator = [&msg] { return SerializeMessage(msg); };
+
+ auto digest_creator = [](std::string const& content) -> bazel_re::Digest {
+ return ArtifactDigest::Create(content);
+ };
+
+ return CommandBundle::Create(msg, content_creator, digest_creator);
+}
+
+/// \brief Create bundle for profobuf message Action from Command.
+[[nodiscard]] auto CreateActionBundle(
+ bazel_re::Digest const& command,
+ bazel_re::Digest const& root_dir,
+ std::vector<std::string> const& output_node_properties,
+ bool do_not_cache,
+ std::chrono::milliseconds const& timeout) -> ActionBundle::Ptr {
+ using seconds = std::chrono::seconds;
+ using nanoseconds = std::chrono::nanoseconds;
+ auto sec = std::chrono::duration_cast<seconds>(timeout);
+ auto nanos = std::chrono::duration_cast<nanoseconds>(timeout - sec);
+
+ auto duration = std::make_unique<google::protobuf::Duration>();
+ duration->set_seconds(sec.count());
+ duration->set_nanos(nanos.count());
+
+ bazel_re::Action msg;
+ msg.set_do_not_cache(do_not_cache);
+ msg.set_allocated_timeout(duration.release());
+ msg.set_allocated_command_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{command}});
+ msg.set_allocated_input_root_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{root_dir}});
+ std::copy(output_node_properties.cbegin(),
+ output_node_properties.cend(),
+ pb::back_inserter(msg.mutable_output_node_properties()));
+
+ auto content_creator = [&msg] { return SerializeMessage(msg); };
+
+ auto digest_creator = [](std::string const& content) -> bazel_re::Digest {
+ return ArtifactDigest::Create(content);
+ };
+
+ return ActionBundle::Create(msg, content_creator, digest_creator);
+}
+
+[[nodiscard]] auto CreateObjectInfo(bazel_re::DirectoryNode const& node)
+ -> Artifact::ObjectInfo {
+ return Artifact::ObjectInfo{ArtifactDigest{node.digest()},
+ ObjectType::Tree};
+}
+
+[[nodiscard]] auto CreateObjectInfo(bazel_re::FileNode const& node)
+ -> Artifact::ObjectInfo {
+ return Artifact::ObjectInfo{
+ ArtifactDigest{node.digest()},
+ node.is_executable() ? ObjectType::Executable : ObjectType::File};
+}
+
+class DirectoryTree;
+using DirectoryTreePtr = std::unique_ptr<DirectoryTree>;
+
+/// \brief Tree of `Artifact*` that can be converted to `DirectoryNodeBundle`.
+class DirectoryTree {
+ public:
+ /// \brief Add `Artifact*` to tree.
+ [[nodiscard]] auto AddArtifact(std::filesystem::path const& path,
+ Artifact const* artifact) -> bool {
+ auto const norm_path = path.lexically_normal();
+ if (norm_path.empty() or
+ not FileSystemManager::IsRelativePath(norm_path)) {
+ return false;
+ }
+ auto it = norm_path.begin();
+ return AddArtifact(&it, norm_path.end(), artifact);
+ }
+
+ /// \brief Convert tree to `DirectoryNodeBundle`.
+ [[nodiscard]] auto ToBundle(
+ std::string const& root_name,
+ std::optional<BazelMsgFactory::BlobStoreFunc> const& store_blob,
+ std::optional<BazelMsgFactory::InfoStoreFunc> const& store_info,
+ std::filesystem::path const& parent = "") const
+ -> DirectoryNodeBundle::Ptr {
+ std::vector<bazel_re::FileNode> file_nodes;
+ std::vector<bazel_re::DirectoryNode> dir_nodes;
+ for (auto const& [name, node] : nodes) {
+ if (std::holds_alternative<DirectoryTreePtr>(node)) {
+ auto const& dir = std::get<DirectoryTreePtr>(node);
+ auto const dir_bundle =
+ dir->ToBundle(name, store_blob, store_info, parent / name);
+ if (not dir_bundle) {
+ return nullptr;
+ }
+ dir_nodes.push_back(dir_bundle->Message());
+ if (store_blob) {
+ (*store_blob)(dir_bundle->MakeBlob());
+ }
+ }
+ else {
+ auto const& artifact = std::get<Artifact const*>(node);
+ auto const& object_info = artifact->Info();
+ if (not object_info) {
+ return nullptr;
+ }
+ if (IsTreeObject(object_info->type)) {
+ dir_nodes.push_back(
+ CreateDirectoryNodeFromObjectInfo(name, *object_info));
+ }
+ else {
+ file_nodes.push_back(
+ CreateFileNodeFromObjectInfo(name, *object_info));
+ }
+ if (store_info and
+ not(*store_info)(parent / name, *object_info)) {
+ return nullptr;
+ }
+ }
+ }
+ return CreateDirectoryNodeBundle(
+ root_name, CreateDirectory(file_nodes, dir_nodes, {}, {}));
+ }
+
+ private:
+ using Node = std::variant<DirectoryTreePtr, Artifact const*>;
+ std::unordered_map<std::string, Node> nodes;
+
+ [[nodiscard]] auto AddArtifact(std::filesystem::path::iterator* begin,
+ std::filesystem::path::iterator const& end,
+ Artifact const* artifact) -> bool {
+ auto segment = *((*begin)++);
+ if (segment == "." or segment == "..") { // fail on "." and ".."
+ return false;
+ }
+ if (*begin == end) {
+ return nodes.emplace(segment, artifact).second;
+ }
+ auto const [it, success] =
+ nodes.emplace(segment, std::make_unique<DirectoryTree>());
+ return (success or
+ std::holds_alternative<DirectoryTreePtr>(it->second)) and
+ std::get<DirectoryTreePtr>(it->second)
+ ->AddArtifact(begin, end, artifact);
+ }
+};
+
+} // namespace
+
+auto BazelMsgFactory::ReadObjectInfosFromDirectory(
+ bazel_re::Directory const& dir,
+ InfoStoreFunc const& store_info) noexcept -> bool {
+ try {
+ for (auto const& f : dir.files()) {
+ if (not store_info(f.name(), CreateObjectInfo(f))) {
+ return false;
+ }
+ }
+ for (auto const& d : dir.directories()) {
+ if (not store_info(d.name(), CreateObjectInfo(d))) {
+ return false;
+ }
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "reading object infos from Directory failed with:\n{}",
+ ex.what());
+ return false;
+ }
+ return true;
+}
+
+auto BazelMsgFactory::CreateDirectoryDigestFromTree(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const& artifacts,
+ std::optional<BlobStoreFunc> const& store_blob,
+ std::optional<InfoStoreFunc> const& store_info)
+ -> std::optional<bazel_re::Digest> {
+ DirectoryTree build_root{};
+ for (auto const& [local_path, node] : artifacts) {
+ auto const* artifact = &node->Content();
+ if (not build_root.AddArtifact(local_path, artifact)) {
+ Logger::Log(LogLevel::Error,
+ "failed to add artifact {} ({}) to build root",
+ local_path,
+ artifact->Digest().value_or(ArtifactDigest{}).hash());
+ return std::nullopt;
+ }
+ }
+
+ auto bundle = build_root.ToBundle("", store_blob, store_info);
+ if (not bundle) {
+ return std::nullopt;
+ }
+ if (store_blob) {
+ (*store_blob)(bundle->MakeBlob());
+ }
+ return bundle->Digest();
+}
+
+auto BazelMsgFactory::CreateDirectoryDigestFromLocalTree(
+ std::filesystem::path const& root,
+ FileStoreFunc const& store_file,
+ DirStoreFunc const& store_dir) noexcept -> std::optional<bazel_re::Digest> {
+ std::vector<bazel_re::FileNode> files{};
+ std::vector<bazel_re::DirectoryNode> dirs{};
+
+ auto dir_reader = [&files, &dirs, &root, &store_file, &store_dir](
+ auto name, auto type) {
+ if (IsTreeObject(type)) {
+ // create and store sub directory
+ auto digest = CreateDirectoryDigestFromLocalTree(
+ root / name, store_file, store_dir);
+ if (not digest) {
+ return false;
+ }
+
+ auto dir = CreateDirectoryNode(name.string());
+ dir.set_allocated_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{*digest}});
+ dirs.emplace_back(std::move(dir));
+ return true;
+ }
+
+ // create and store file
+ try {
+ if (auto digest =
+ store_file(root / name, type == ObjectType::Executable)) {
+ auto file = CreateFileNode(name.string(), type, {});
+ file.set_allocated_digest(gsl::owner<bazel_re::Digest*>{
+ new bazel_re::Digest{std::move(*digest)}});
+ files.emplace_back(std::move(file));
+ return true;
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(
+ LogLevel::Error, "storing file failed with:\n{}", ex.what());
+ }
+ return false;
+ };
+
+ if (FileSystemManager::ReadDirectory(root, dir_reader)) {
+ auto dir = CreateDirectory(files, dirs, {}, {});
+ if (auto bytes = SerializeMessage(dir)) {
+ try {
+ if (auto digest = store_dir(*bytes, dir)) {
+ return *digest;
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "storing directory failed with:\n{}",
+ ex.what());
+ }
+ return std::nullopt;
+ }
+ }
+ return std::nullopt;
+}
+
+auto BazelMsgFactory::CreateActionDigestFromCommandLine(
+ std::vector<std::string> const& cmdline,
+ bazel_re::Digest const& exec_dir,
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::vector<std::string> const& output_node_properties,
+ std::vector<bazel_re::Command_EnvironmentVariable> const& env_vars,
+ std::vector<bazel_re::Platform_Property> const& properties,
+ bool do_not_cache,
+ std::chrono::milliseconds const& timeout,
+ std::optional<BlobStoreFunc> const& store_blob) -> bazel_re::Digest {
+ // create command
+ auto cmd = CreateCommandBundle(
+ cmdline, output_files, output_dirs, env_vars, properties);
+
+ // create action
+ auto action = CreateActionBundle(
+ cmd->Digest(), exec_dir, output_node_properties, do_not_cache, timeout);
+
+ if (store_blob) {
+ (*store_blob)(cmd->MakeBlob());
+ (*store_blob)(action->MakeBlob());
+ }
+
+ return action->Digest();
+}
+
+auto BazelMsgFactory::DirectoryToString(bazel_re::Directory const& dir) noexcept
+ -> std::optional<std::string> {
+ auto json = nlohmann::json::object();
+ try {
+ if (not BazelMsgFactory::ReadObjectInfosFromDirectory(
+ dir, [&json](auto path, auto info) {
+ json[path.string()] = info.ToString();
+ return true;
+ })) {
+ Logger::Log(LogLevel::Error,
+ "reading object infos from Directory failed");
+ return std::nullopt;
+ }
+ return json.dump(2);
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "dumping Directory to string failed with:\n{}",
+ ex.what());
+ return std::nullopt;
+ }
+}
diff --git a/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp
new file mode 100644
index 00000000..d850e6c6
--- /dev/null
+++ b/src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp
@@ -0,0 +1,128 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_MSG_FACTORY_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_MSG_FACTORY_HPP
+
+#include <chrono>
+#include <filesystem>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_engine/dag/dag.hpp"
+
+/// \brief Factory for creating Bazel API protobuf messages.
+/// Responsible for creating protobuf messages necessary for Bazel API server
+/// communication.
+class BazelMsgFactory {
+ public:
+ using BlobStoreFunc = std::function<void(BazelBlob&&)>;
+ using InfoStoreFunc = std::function<bool(std::filesystem::path const&,
+ Artifact::ObjectInfo const&)>;
+ using FileStoreFunc = std::function<
+ std::optional<bazel_re::Digest>(std::filesystem::path const&, bool)>;
+ using DirStoreFunc = std::function<std::optional<bazel_re::Digest>(
+ std::string const&,
+ bazel_re::Directory const&)>;
+
+ /// \brief Read object infos from directory.
+ /// \returns true on success.
+ [[nodiscard]] static auto ReadObjectInfosFromDirectory(
+ bazel_re::Directory const& dir,
+ InfoStoreFunc const& store_info) noexcept -> bool;
+
+ /// \brief Create Directory digest from artifact tree structure.
+ /// Recursively traverse entire tree and create blobs for sub-directories.
+ /// \param artifacts Artifact tree structure.
+ /// \param store_blob Function for storing Directory blobs.
+ /// \param store_info Function for storing object infos.
+ /// \returns Digest representing the entire tree.
+ [[nodiscard]] static auto CreateDirectoryDigestFromTree(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const& artifacts,
+ std::optional<BlobStoreFunc> const& store_blob = std::nullopt,
+ std::optional<InfoStoreFunc> const& store_info = std::nullopt)
+ -> std::optional<bazel_re::Digest>;
+
+ /// \brief Create Directory digest from local file root.
+ /// Recursively traverse entire root and store files and directories.
+ /// \param root Path to local file root.
+ /// \param store_file Function for storing local file via path.
+ /// \param store_dir Function for storing Directory blobs.
+ /// \returns Digest representing the entire file root.
+ [[nodiscard]] static auto CreateDirectoryDigestFromLocalTree(
+ std::filesystem::path const& root,
+ FileStoreFunc const& store_file,
+ DirStoreFunc const& store_dir) noexcept
+ -> std::optional<bazel_re::Digest>;
+
+ /// \brief Creates Action digest from command line.
+ /// As part of the internal process, it creates an ActionBundle and
+ /// CommandBundle that can be captured via BlobStoreFunc.
+ /// \param[in] cmdline The command line.
+ /// \param[in] exec_dir The Digest of the execution directory.
+ /// \param[in] output_files The paths of output files.
+ /// \param[in] output_dirs The paths of output directories.
+ /// \param[in] output_node. The output node's properties.
+ /// \param[in] env_vars The environment variables set.
+ /// \param[in] properties The target platform's properties.
+ /// \param[in] do_not_cache Skip action cache.
+ /// \param[in] timeout The command execution timeout.
+ /// \param[in] store_blob Function for storing action and cmd bundles.
+ /// \returns Digest representing the action.
+ [[nodiscard]] static auto CreateActionDigestFromCommandLine(
+ std::vector<std::string> const& cmdline,
+ bazel_re::Digest const& exec_dir,
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::vector<std::string> const& output_node_properties,
+ std::vector<bazel_re::Command_EnvironmentVariable> const& env_vars,
+ std::vector<bazel_re::Platform_Property> const& properties,
+ bool do_not_cache,
+ std::chrono::milliseconds const& timeout,
+ std::optional<BlobStoreFunc> const& store_blob = std::nullopt)
+ -> bazel_re::Digest;
+
+ /// \brief Create descriptive string from Directory protobuf message.
+ [[nodiscard]] static auto DirectoryToString(
+ bazel_re::Directory const& dir) noexcept -> std::optional<std::string>;
+
+ /// \brief Create message vector from std::map.
+ /// \param[in] input map
+ /// \tparam T protobuf message type. It must be a name-value
+ /// message (i.e. class methods T::set_name(std::string) and
+ /// T::set_value(std::string) must exist)
+ template <class T>
+ [[nodiscard]] static auto CreateMessageVectorFromMap(
+ std::map<std::string, std::string> const& input) noexcept
+ -> std::vector<T> {
+ std::vector<T> output{};
+ std::transform(std::begin(input),
+ std::end(input),
+ std::back_inserter(output),
+ [](auto const& key_val) {
+ T msg;
+ msg.set_name(key_val.first);
+ msg.set_value(key_val.second);
+ return msg;
+ });
+ return output;
+ }
+
+ template <class T>
+ [[nodiscard]] static auto MessageFromString(std::string const& blob)
+ -> std::optional<T> {
+ T msg{};
+ if (msg.ParseFromString(blob)) {
+ return msg;
+ }
+ Logger::Log(LogLevel::Error, "failed to parse message from string");
+ return std::nullopt;
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_BAZEL_MSG_BAZEL_MSG_FACTORY_HPP
diff --git a/src/buildtool/execution_api/common/TARGETS b/src/buildtool/execution_api/common/TARGETS
new file mode 100644
index 00000000..aa3ad0bd
--- /dev/null
+++ b/src/buildtool/execution_api/common/TARGETS
@@ -0,0 +1,22 @@
+{ "common":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["common"]
+ , "hdrs":
+ [ "execution_common.hpp"
+ , "execution_api.hpp"
+ , "execution_action.hpp"
+ , "execution_response.hpp"
+ , "local_tree_map.hpp"
+ ]
+ , "deps":
+ [ ["@", "gsl-lite", "", "gsl-lite"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/crypto", "hash_generator"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
+ , ["src/utils/cpp", "hex_string"]
+ ]
+ , "stage": ["src", "buildtool", "execution_api", "common"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/execution_api/common/execution_action.hpp b/src/buildtool/execution_api/common/execution_action.hpp
new file mode 100644
index 00000000..58176bda
--- /dev/null
+++ b/src/buildtool/execution_api/common/execution_action.hpp
@@ -0,0 +1,58 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP
+
+#include <chrono>
+#include <memory>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/execution_api/common/execution_response.hpp"
+
+class Logger;
+class ExecutionArtifactContainer;
+
+/// \brief Abstract action.
+/// Can execute multiple commands. Commands are executed in arbitrary order and
+/// cannot depend on each other.
+class IExecutionAction {
+ public:
+ using Ptr = std::unique_ptr<IExecutionAction>;
+
+ enum class CacheFlag {
+ CacheOutput, ///< run and cache, or serve from cache
+ DoNotCacheOutput, ///< run and do not cache, never served from cached
+ FromCacheOnly, ///< do not run, only serve from cache
+ PretendCached ///< always run, respond same action id as if cached
+ };
+
+ static constexpr std::chrono::milliseconds kDefaultTimeout{1000};
+
+ [[nodiscard]] static constexpr auto CacheEnabled(CacheFlag f) -> bool {
+ return f == CacheFlag::CacheOutput or f == CacheFlag::FromCacheOnly;
+ }
+
+ [[nodiscard]] static constexpr auto ExecutionEnabled(CacheFlag f) -> bool {
+ return f == CacheFlag::CacheOutput or
+ f == CacheFlag::DoNotCacheOutput or
+ f == CacheFlag::PretendCached;
+ }
+
+ IExecutionAction() = default;
+ IExecutionAction(IExecutionAction const&) = delete;
+ IExecutionAction(IExecutionAction&&) = delete;
+ auto operator=(IExecutionAction const&) -> IExecutionAction& = delete;
+ auto operator=(IExecutionAction &&) -> IExecutionAction& = delete;
+ virtual ~IExecutionAction() = default;
+
+ /// \brief Execute the action.
+ /// \returns Execution response, with commands' outputs and artifacts.
+ /// \returns nullptr if execution failed.
+ // NOLINTNEXTLINE(google-default-arguments)
+ [[nodiscard]] virtual auto Execute(Logger const* logger = nullptr) noexcept
+ -> IExecutionResponse::Ptr = 0;
+
+ virtual void SetCacheFlag(CacheFlag flag) noexcept = 0;
+
+ virtual void SetTimeout(std::chrono::milliseconds timeout) noexcept = 0;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_ACTION_HPP
diff --git a/src/buildtool/execution_api/common/execution_api.hpp b/src/buildtool/execution_api/common/execution_api.hpp
new file mode 100644
index 00000000..92002d48
--- /dev/null
+++ b/src/buildtool/execution_api/common/execution_api.hpp
@@ -0,0 +1,78 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact.hpp" // Artifact::ObjectInfo
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
+#include "src/buildtool/execution_api/common/execution_action.hpp"
+
+/// \brief Abstract remote execution API
+/// Can be used to create actions.
+class IExecutionApi {
+ public:
+ using Ptr = std::unique_ptr<IExecutionApi>;
+
+ IExecutionApi() = default;
+ IExecutionApi(IExecutionApi const&) = delete;
+ IExecutionApi(IExecutionApi&&) = default;
+ auto operator=(IExecutionApi const&) -> IExecutionApi& = delete;
+ auto operator=(IExecutionApi &&) -> IExecutionApi& = default;
+ virtual ~IExecutionApi() = default;
+
+ /// \brief Create a new action.
+ /// \param[in] root_digest Digest of the build root.
+ /// \param[in] command Command as argv vector
+ /// \param[in] output_files List of paths to output files.
+ /// \param[in] output_dirs List of paths to output directories.
+ /// \param[in] env_vars The environment variables to set.
+ /// \param[in] properties Platform properties to set.
+ /// \returns The new action.
+ [[nodiscard]] virtual auto CreateAction(
+ ArtifactDigest const& root_digest,
+ std::vector<std::string> const& command,
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) noexcept
+ -> IExecutionAction::Ptr = 0;
+
+ /// \brief Retrieve artifacts from CAS and store to specified paths.
+ /// Tree artifacts are resolved its containing file artifacts are
+ /// recursively retrieved.
+ [[nodiscard]] virtual auto RetrieveToPaths(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<std::filesystem::path> const& output_paths) noexcept
+ -> bool = 0;
+
+ /// \brief Retrieve artifacts from CAS and write to file descriptors.
+ /// Tree artifacts are not resolved and instead the raw protobuf message
+ /// will be written to fd.
+ [[nodiscard]] virtual auto RetrieveToFds(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<int> const& fds) noexcept -> bool = 0;
+
+ /// \brief Upload blobs to CAS. Uploads only the blobs that are not yet
+ /// available in CAS, unless `skip_find_missing` is specified.
+ /// \param blobs Container of blobs to upload.
+ /// \param skip_find_missing Skip finding missing blobs, just upload all.
+ /// NOLINTNEXTLINE(google-default-arguments)
+ [[nodiscard]] virtual auto Upload(BlobContainer const& blobs,
+ bool skip_find_missing = false) noexcept
+ -> bool = 0;
+
+ [[nodiscard]] virtual auto UploadTree(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const&
+ artifacts) noexcept -> std::optional<ArtifactDigest> = 0;
+
+ [[nodiscard]] virtual auto IsAvailable(
+ ArtifactDigest const& digest) const noexcept -> bool = 0;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_APIHPP
diff --git a/src/buildtool/execution_api/common/execution_common.hpp b/src/buildtool/execution_api/common/execution_common.hpp
new file mode 100644
index 00000000..8b6aea40
--- /dev/null
+++ b/src/buildtool/execution_api/common/execution_common.hpp
@@ -0,0 +1,109 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_COMMON_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_COMMON_HPP
+
+#ifdef __unix__
+#include <sys/types.h>
+#include <unistd.h>
+#else
+#error "Non-unix is not supported yet"
+#endif
+
+#include <array>
+#include <filesystem>
+#include <optional>
+#include <random>
+#include <sstream>
+#include <string>
+#include <thread>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/crypto/hash_generator.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+/// \brief Create unique ID for current process and thread.
+[[nodiscard]] static inline auto CreateProcessUniqueId() noexcept
+ -> std::optional<std::string> {
+#ifdef __unix__
+ pid_t pid{};
+ try {
+ pid = getpid();
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return std::nullopt;
+ }
+#endif
+ auto tid = std::this_thread::get_id();
+ std::ostringstream id{};
+ id << pid << "-" << tid;
+ return id.str();
+}
+
+/// \brief Create unique path based on file_path.
+[[nodiscard]] static inline auto CreateUniquePath(
+ std::filesystem::path file_path) noexcept
+ -> std::optional<std::filesystem::path> {
+ auto id = CreateProcessUniqueId();
+ if (id) {
+ return file_path.concat("." + *id);
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] static auto GetNonDeterministicRandomNumber() -> unsigned int {
+ std::uniform_int_distribution<unsigned int> dist{};
+ std::random_device urandom{
+#ifdef __unix__
+ "/dev/urandom"
+#endif
+ };
+ return dist(urandom);
+}
+
+static auto kRandomConstant = GetNonDeterministicRandomNumber();
+
+static void EncodeUUIDVersion4(std::string* uuid) {
+ constexpr auto kVersionByte = 6UL;
+ constexpr auto kVersionBits = 0x40U; // version 4: 0100 xxxx
+ constexpr auto kClearMask = 0x0fU;
+ gsl_Expects(uuid->size() >= kVersionByte);
+ auto& byte = uuid->at(kVersionByte);
+ byte = static_cast<char>(kVersionBits |
+ (kClearMask & static_cast<std::uint8_t>(byte)));
+}
+
+static void EncodeUUIDVariant1(std::string* uuid) {
+ constexpr auto kVariantByte = 8UL;
+ constexpr auto kVariantBits = 0x80U; // variant 1: 10xx xxxx
+ constexpr auto kClearMask = 0x3fU;
+ gsl_Expects(uuid->size() >= kVariantByte);
+ auto& byte = uuid->at(kVariantByte);
+ byte = static_cast<char>(kVariantBits |
+ (kClearMask & static_cast<std::uint8_t>(byte)));
+}
+
+/// \brief Create UUID version 4 from seed.
+[[nodiscard]] static inline auto CreateUUIDVersion4(std::string const& seed)
+ -> std::string {
+ constexpr auto kRawLength = 16UL;
+ constexpr auto kHexDashPos = std::array{8UL, 12UL, 16UL, 20UL};
+
+ auto value = fmt::format("{}-{}", std::to_string(kRandomConstant), seed);
+ auto uuid = HashGenerator{HashGenerator::HashType::SHA1}.Run(value).Bytes();
+ EncodeUUIDVersion4(&uuid);
+ EncodeUUIDVariant1(&uuid);
+ gsl_Expects(uuid.size() >= kRawLength);
+
+ std::size_t cur{};
+ std::ostringstream ss{};
+ auto uuid_hex = ToHexString(uuid.substr(0, kRawLength));
+ for (auto pos : kHexDashPos) {
+ ss << uuid_hex.substr(cur, pos - cur) << '-';
+ cur = pos;
+ }
+ ss << uuid_hex.substr(cur);
+ gsl_EnsuresAudit(ss.str().size() == (2 * kRawLength) + kHexDashPos.size());
+ return ss.str();
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_EXECUTION_COMMON_HPP
diff --git a/src/buildtool/execution_api/common/execution_response.hpp b/src/buildtool/execution_api/common/execution_response.hpp
new file mode 100644
index 00000000..76349018
--- /dev/null
+++ b/src/buildtool/execution_api/common/execution_response.hpp
@@ -0,0 +1,48 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact.hpp"
+
+/// \brief Abstract response.
+/// Response of an action execution. Contains outputs from multiple commands and
+/// a single container with artifacts.
+class IExecutionResponse {
+ public:
+ using Ptr = std::unique_ptr<IExecutionResponse>;
+ using ArtifactInfos = std::unordered_map<std::string, Artifact::ObjectInfo>;
+
+ enum class StatusCode { Failed, Success };
+
+ IExecutionResponse() = default;
+ IExecutionResponse(IExecutionResponse const&) = delete;
+ IExecutionResponse(IExecutionResponse&&) = delete;
+ auto operator=(IExecutionResponse const&) -> IExecutionResponse& = delete;
+ auto operator=(IExecutionResponse &&) -> IExecutionResponse& = delete;
+ virtual ~IExecutionResponse() = default;
+
+ [[nodiscard]] virtual auto Status() const noexcept -> StatusCode = 0;
+
+ [[nodiscard]] virtual auto ExitCode() const noexcept -> int = 0;
+
+ [[nodiscard]] virtual auto IsCached() const noexcept -> bool = 0;
+
+ [[nodiscard]] virtual auto HasStdErr() const noexcept -> bool = 0;
+
+ [[nodiscard]] virtual auto HasStdOut() const noexcept -> bool = 0;
+
+ [[nodiscard]] virtual auto StdErr() noexcept -> std::string = 0;
+
+ [[nodiscard]] virtual auto StdOut() noexcept -> std::string = 0;
+
+ [[nodiscard]] virtual auto ActionDigest() const noexcept -> std::string = 0;
+
+ [[nodiscard]] virtual auto Artifacts() const noexcept -> ArtifactInfos = 0;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_REMOTE_EXECUTION_RESPONSE_HPP
diff --git a/src/buildtool/execution_api/common/local_tree_map.hpp b/src/buildtool/execution_api/common/local_tree_map.hpp
new file mode 100644
index 00000000..77de2d53
--- /dev/null
+++ b/src/buildtool/execution_api/common/local_tree_map.hpp
@@ -0,0 +1,140 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP
+
+#include <filesystem>
+#include <shared_mutex>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// \brief Maps digest of `bazel_re::Directory` to `LocalTree`.
+class LocalTreeMap {
+ /// \brief Thread-safe pool of unique object infos.
+ class ObjectInfoPool {
+ public:
+ /// Get pointer to stored info, or a add new one and return its pointer.
+ [[nodiscard]] auto GetOrAdd(Artifact::ObjectInfo const& info)
+ -> Artifact::ObjectInfo const* {
+ { // get
+ std::shared_lock lock{mutex_};
+ auto it = infos_.find(info);
+ if (it != infos_.end()) {
+ return &(*it);
+ }
+ }
+ { // or add
+ std::unique_lock lock{mutex_};
+ return &(*infos_.emplace(info).first);
+ }
+ }
+
+ private:
+ std::unordered_set<Artifact::ObjectInfo> infos_;
+ mutable std::shared_mutex mutex_;
+ };
+
+ public:
+ /// \brief Maps blob locations to object infos.
+ class LocalTree {
+ friend class LocalTreeMap;
+
+ public:
+ /// \brief Add a new path and info pair to the tree.
+ /// Path must not be absolute, empty, or contain dot-segments.
+ /// \param path The location to add the object info.
+ /// \param info The object info to add.
+ /// \returns true if successfully inserted or info existed before.
+ [[nodiscard]] auto AddInfo(std::filesystem::path const& path,
+ Artifact::ObjectInfo const& info) noexcept
+ -> bool {
+ auto norm_path = path.lexically_normal();
+ if (norm_path.is_absolute() or norm_path.empty() or
+ *norm_path.begin() == "..") {
+ Logger::Log(LogLevel::Error,
+ "cannot add malformed path to local tree: {}",
+ path.string());
+ return false;
+ }
+ try {
+ if (entries_.contains(norm_path.string())) {
+ return true;
+ }
+ if (auto const* info_ptr = infos_->GetOrAdd(info)) {
+ entries_.emplace(norm_path.string(), info_ptr);
+ return true;
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "adding object info to tree failed with:\n{}",
+ ex.what());
+ }
+ return false;
+ }
+
+ [[nodiscard]] auto size() const noexcept { return entries_.size(); }
+ [[nodiscard]] auto begin() const noexcept { return entries_.begin(); }
+ [[nodiscard]] auto end() const noexcept { return entries_.end(); }
+
+ private:
+ gsl::not_null<ObjectInfoPool*> infos_;
+ std::unordered_map<std::string,
+ gsl::not_null<Artifact::ObjectInfo const*>>
+ entries_{};
+
+ explicit LocalTree(gsl::not_null<ObjectInfoPool*> infos) noexcept
+ : infos_{std::move(infos)} {}
+ };
+
+ /// \brief Create a new `LocalTree` object.
+ [[nodiscard]] auto CreateTree() noexcept -> LocalTree {
+ return LocalTree{&infos_};
+ }
+
+ /// \brief Get pointer to existing `LocalTree` object.
+ /// \param root_digest The root digest of the tree to lookup.
+ /// \returns nullptr if no tree was found for given root digest.
+ [[nodiscard]] auto GetTree(bazel_re::Digest const& root_digest)
+ const noexcept -> LocalTree const* {
+ std::shared_lock lock{mutex_};
+ auto it = trees_.find(root_digest);
+ return (it != trees_.end()) ? &(it->second) : nullptr;
+ }
+
+ /// \brief Checks if entry for root digest exists.
+ [[nodiscard]] auto HasTree(
+ bazel_re::Digest const& root_digest) const noexcept -> bool {
+ return GetTree(root_digest) != nullptr;
+ }
+
+ /// \brief Add new `LocalTree` for given root digest. Does not overwrite if
+ /// a tree for the given root digest already exists.
+ /// \param root_digest The root digest to add the new tree for.
+ /// \param tree The new tree to add.
+ /// \returns true if the tree was successfully added or existed before.
+ [[nodiscard]] auto AddTree(bazel_re::Digest const& root_digest,
+ LocalTree&& tree) noexcept -> bool {
+ if (not HasTree(root_digest)) {
+ try {
+ std::unique_lock lock{mutex_};
+ trees_.emplace(root_digest, std::move(tree));
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "adding local tree to tree map failed with:\n{}",
+ ex.what());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private:
+ ObjectInfoPool infos_; // pool to store each solid object info exactly once
+ std::unordered_map<bazel_re::Digest, LocalTree> trees_;
+ mutable std::shared_mutex mutex_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_COMMON_LOCAL_TREE_MAP_HPP
diff --git a/src/buildtool/execution_api/local/TARGETS b/src/buildtool/execution_api/local/TARGETS
new file mode 100644
index 00000000..b3e54597
--- /dev/null
+++ b/src/buildtool/execution_api/local/TARGETS
@@ -0,0 +1,36 @@
+{ "config":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["config"]
+ , "hdrs": ["config.hpp"]
+ , "deps":
+ [ ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ ]
+ , "stage": ["src", "buildtool", "execution_api", "local"]
+ }
+, "local":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["local"]
+ , "hdrs":
+ [ "file_storage.hpp"
+ , "local_api.hpp"
+ , "local_action.hpp"
+ , "local_response.hpp"
+ , "local_storage.hpp"
+ , "local_cas.hpp"
+ , "local_ac.hpp"
+ ]
+ , "srcs": ["local_action.cpp", "local_storage.cpp"]
+ , "deps":
+ [ ["@", "gsl-lite", "", "gsl-lite"]
+ , "config"
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/file_system", "system_command"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/logging", "logging"]
+ ]
+ , "stage": ["src", "buildtool", "execution_api", "local"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/execution_api/local/config.hpp b/src/buildtool/execution_api/local/config.hpp
new file mode 100644
index 00000000..5f3a2a80
--- /dev/null
+++ b/src/buildtool/execution_api/local/config.hpp
@@ -0,0 +1,137 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP
+
+#ifdef __unix__
+#include <pwd.h>
+#include <sys/types.h>
+#include <unistd.h>
+#else
+#error "Non-unix is not supported yet"
+#endif
+
+#include <filesystem>
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// \brief Store global build system configuration.
+class LocalExecutionConfig {
+ public:
+ [[nodiscard]] static auto SetBuildRoot(
+ std::filesystem::path const& dir) noexcept -> bool {
+ if (FileSystemManager::IsRelativePath(dir)) {
+ Logger::Log(LogLevel::Error,
+ "Build root must be absolute path but got '{}'.",
+ dir.string());
+ return false;
+ }
+ build_root_ = dir;
+ return true;
+ }
+
+ [[nodiscard]] static auto SetDiskCache(
+ std::filesystem::path const& dir) noexcept -> bool {
+ if (FileSystemManager::IsRelativePath(dir)) {
+ Logger::Log(LogLevel::Error,
+ "Disk cache must be absolute path but got '{}'.",
+ dir.string());
+ return false;
+ }
+ disk_cache_ = dir;
+ return true;
+ }
+
+ [[nodiscard]] static auto SetLauncher(
+ std::vector<std::string> const& launcher) noexcept -> bool {
+ try {
+ launcher_ = launcher;
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "when setting the local launcher\n{}",
+ e.what());
+ return false;
+ }
+ return true;
+ }
+
+ [[nodiscard]] static auto SetKeepBuildDir(bool is_persistent) noexcept
+ -> bool {
+ keep_build_dir_ = is_persistent;
+ return true;
+ }
+
+ /// \brief User directory.
+ [[nodiscard]] static auto GetUserDir() noexcept -> std::filesystem::path {
+ if (user_root_.empty()) {
+ user_root_ = GetUserRoot() / ".cache" / "just";
+ }
+ return user_root_;
+ }
+
+ /// \brief Build directory, defaults to user directory if not set
+ [[nodiscard]] static auto GetBuildDir() noexcept -> std::filesystem::path {
+ if (build_root_.empty()) {
+ return GetUserDir();
+ }
+ return build_root_;
+ }
+
+ /// \brief Cache directory, defaults to user directory if not set
+ [[nodiscard]] static auto GetCacheDir() noexcept -> std::filesystem::path {
+ if (disk_cache_.empty()) {
+ return GetBuildDir();
+ }
+ return disk_cache_;
+ }
+
+ [[nodiscard]] static auto GetLauncher() noexcept
+ -> std::vector<std::string> {
+ return launcher_;
+ }
+
+ [[nodiscard]] static auto KeepBuildDir() noexcept -> bool {
+ return keep_build_dir_;
+ }
+
+ private:
+ // User root directory (Unix default: /home/${USER})
+ static inline std::filesystem::path user_root_{};
+
+ // Build root directory (default: empty)
+ static inline std::filesystem::path build_root_{};
+
+ // Disk cache directory (default: empty)
+ static inline std::filesystem::path disk_cache_{};
+
+ // Launcher to be prepended to action's command before executed.
+ // Default: ["env", "--"]
+ static inline std::vector<std::string> launcher_{"env", "--"};
+
+ // Persistent build directory option
+ static inline bool keep_build_dir_{false};
+
+ /// \brief Determine user root directory
+ [[nodiscard]] static inline auto GetUserRoot() noexcept
+ -> std::filesystem::path {
+ char const* root{nullptr};
+
+#ifdef __unix__
+ root = std::getenv("HOME");
+ if (root == nullptr) {
+ root = getpwuid(getuid())->pw_dir;
+ }
+#endif
+
+ if (root == nullptr) {
+ Logger::Log(LogLevel::Error, "Cannot determine user directory.");
+ std::exit(EXIT_FAILURE);
+ }
+
+ return root;
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_CONFIG_HPP
diff --git a/src/buildtool/execution_api/local/file_storage.hpp b/src/buildtool/execution_api/local/file_storage.hpp
new file mode 100644
index 00000000..07ac1204
--- /dev/null
+++ b/src/buildtool/execution_api/local/file_storage.hpp
@@ -0,0 +1,107 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_FILE_STORAGE_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_FILE_STORAGE_HPP
+
+#include <filesystem>
+#include <string>
+
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+
+enum class StoreMode {
+ // First thread to write conflicting file wins.
+ FirstWins,
+ // Last thread to write conflicting file wins, effectively overwriting
+ // existing entries. NOTE: This might cause races if hard linking from
+ // stored files due to an issue with the interaction of rename(2) and
+ // link(2) (see: https://stackoverflow.com/q/69076026/1107763).
+ LastWins
+};
+
+template <ObjectType kType = ObjectType::File,
+ StoreMode kMode = StoreMode::FirstWins>
+class FileStorage {
+ public:
+ explicit FileStorage(std::filesystem::path storage_root) noexcept
+ : storage_root_{std::move(storage_root)} {}
+
+ /// \brief Add file to storage.
+ /// \returns true if file exists afterward.
+ [[nodiscard]] auto AddFromFile(
+ std::string const& id,
+ std::filesystem::path const& source_path) const noexcept -> bool {
+ return AtomicAdd(id, source_path);
+ }
+
+ /// \brief Add bytes to storage.
+ /// \returns true if file exists afterward.
+ [[nodiscard]] auto AddFromBytes(std::string const& id,
+ std::string const& bytes) const noexcept
+ -> bool {
+ return AtomicAdd(id, bytes);
+ }
+
+ [[nodiscard]] auto GetPath(std::string const& name) const noexcept
+ -> std::filesystem::path {
+ return storage_root_ / name;
+ }
+
+ private:
+ std::filesystem::path const storage_root_{};
+
+ /// \brief Add file to storage via copy and atomic rename.
+ /// If a race-condition occurs, the winning thread will be the one
+ /// performing the rename operation first or last, depending on kMode being
+ /// set to FirstWins or LastWins, respectively. All threads will signal
+ /// success.
+ /// \returns true if file exists afterward.
+ template <class T>
+ [[nodiscard]] auto AtomicAdd(std::string const& id,
+ T const& data) const noexcept -> bool {
+ auto file_path = storage_root_ / id;
+ if (kMode == StoreMode::LastWins or
+ not FileSystemManager::Exists(file_path)) {
+ auto unique_path = CreateUniquePath(file_path);
+ if (unique_path and
+ FileSystemManager::CreateDirectory(file_path.parent_path()) and
+ CreateFileFromData(*unique_path, data) and
+ StageFile(*unique_path, file_path)) {
+ Logger::Log(
+ LogLevel::Trace, "created entry {}.", file_path.string());
+ return true;
+ }
+ }
+ return FileSystemManager::IsFile(file_path);
+ }
+
+ /// \brief Create file from file path.
+ [[nodiscard]] static auto CreateFileFromData(
+ std::filesystem::path const& file_path,
+ std::filesystem::path const& other_path) noexcept -> bool {
+ return FileSystemManager::CopyFileAs<kType>(other_path, file_path);
+ }
+
+ /// \brief Create file from bytes.
+ [[nodiscard]] static auto CreateFileFromData(
+ std::filesystem::path const& file_path,
+ std::string const& bytes) noexcept -> bool {
+ return FileSystemManager::WriteFileAs<kType>(bytes, file_path);
+ }
+
+ /// \brief Stage file from source path to target path.
+ [[nodiscard]] static auto StageFile(
+ std::filesystem::path const& src_path,
+ std::filesystem::path const& dst_path) noexcept -> bool {
+ switch (kMode) {
+ case StoreMode::FirstWins:
+ // try rename source or delete it if the target already exists
+ return FileSystemManager::Rename(
+ src_path, dst_path, /*no_clobber=*/true) or
+ (FileSystemManager::IsFile(dst_path) and
+ FileSystemManager::RemoveFile(src_path));
+ case StoreMode::LastWins:
+ return FileSystemManager::Rename(src_path, dst_path);
+ }
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_FILE_STORAGE_HPP
diff --git a/src/buildtool/execution_api/local/local_ac.hpp b/src/buildtool/execution_api/local/local_ac.hpp
new file mode 100644
index 00000000..f319940a
--- /dev/null
+++ b/src/buildtool/execution_api/local/local_ac.hpp
@@ -0,0 +1,82 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/execution_api/local/file_storage.hpp"
+#include "src/buildtool/execution_api/local/local_cas.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+class LocalAC {
+ public:
+ explicit LocalAC(gsl::not_null<LocalCAS<ObjectType::File>*> cas) noexcept
+ : cas_{std::move(cas)} {};
+
+ LocalAC(gsl::not_null<LocalCAS<ObjectType::File>*> cas,
+ std::filesystem::path cache_root) noexcept
+ : cas_{std::move(cas)}, cache_root_{std::move(cache_root)} {}
+
+ LocalAC(LocalAC const&) = delete;
+ LocalAC(LocalAC&&) = delete;
+ auto operator=(LocalAC const&) -> LocalAC& = delete;
+ auto operator=(LocalAC &&) -> LocalAC& = delete;
+ ~LocalAC() noexcept = default;
+
+ [[nodiscard]] auto StoreResult(
+ bazel_re::Digest const& action_id,
+ bazel_re::ActionResult const& result) const noexcept -> bool {
+ auto bytes = result.SerializeAsString();
+ auto digest = cas_->StoreBlobFromBytes(bytes);
+ return (digest and file_store_.AddFromBytes(
+ action_id.hash(), digest->SerializeAsString()));
+ }
+
+ [[nodiscard]] auto CachedResult(bazel_re::Digest const& action_id)
+ const noexcept -> std::optional<bazel_re::ActionResult> {
+ auto entry_path = file_store_.GetPath(action_id.hash());
+ bazel_re::Digest digest{};
+ auto const entry =
+ FileSystemManager::ReadFile(entry_path, ObjectType::File);
+ if (not entry.has_value()) {
+ logger_.Emit(LogLevel::Debug,
+ "Cache miss, entry not found {}",
+ entry_path.string());
+ return std::nullopt;
+ }
+ if (not digest.ParseFromString(*entry)) {
+ logger_.Emit(LogLevel::Warning,
+ "Parsing cache entry failed failed for action {}",
+ action_id.hash());
+ return std::nullopt;
+ }
+ auto src_path = cas_->BlobPath(digest);
+ bazel_re::ActionResult result{};
+ if (src_path) {
+ auto const bytes = FileSystemManager::ReadFile(*src_path);
+ if (bytes.has_value() and result.ParseFromString(*bytes)) {
+ return result;
+ }
+ }
+ logger_.Emit(LogLevel::Warning,
+ "Parsing action result failed for action {}",
+ action_id.hash());
+ return std::nullopt;
+ }
+
+ private:
+ // The action cache stores the results of failed actions. For those to be
+ // overwritable by subsequent runs we need to choose the store mode "last
+ // wins" for the underlying file storage.
+ static constexpr auto kStoreMode = StoreMode::LastWins;
+
+ Logger logger_{"LocalAC"};
+ gsl::not_null<LocalCAS<ObjectType::File>*> cas_;
+ std::filesystem::path const cache_root_{
+ LocalExecutionConfig::GetCacheDir()};
+ FileStorage<ObjectType::File, kStoreMode> file_store_{cache_root_ / "ac"};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_AC_HPP
diff --git a/src/buildtool/execution_api/local/local_action.cpp b/src/buildtool/execution_api/local/local_action.cpp
new file mode 100644
index 00000000..eac6ede8
--- /dev/null
+++ b/src/buildtool/execution_api/local/local_action.cpp
@@ -0,0 +1,295 @@
+#include "src/buildtool/execution_api/local/local_action.hpp"
+
+#include <algorithm>
+#include <filesystem>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/local/local_response.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/file_system/system_command.hpp"
+
+namespace {
+
+/// \brief Removes specified directory if KeepBuildDir() is not set.
+class BuildCleanupAnchor {
+ public:
+ explicit BuildCleanupAnchor(std::filesystem::path build_path) noexcept
+ : build_path{std::move(build_path)} {}
+ BuildCleanupAnchor(BuildCleanupAnchor const&) = delete;
+ BuildCleanupAnchor(BuildCleanupAnchor&&) = delete;
+ auto operator=(BuildCleanupAnchor const&) -> BuildCleanupAnchor& = delete;
+ auto operator=(BuildCleanupAnchor &&) -> BuildCleanupAnchor& = delete;
+ ~BuildCleanupAnchor() {
+ if (not LocalExecutionConfig::KeepBuildDir() and
+ not FileSystemManager::RemoveDirectory(build_path, true)) {
+ Logger::Log(LogLevel::Error,
+ "Could not cleanup build directory {}",
+ build_path.string());
+ }
+ }
+
+ private:
+ std::filesystem::path const build_path{};
+};
+
+} // namespace
+
+auto LocalAction::Execute(Logger const* logger) noexcept
+ -> IExecutionResponse::Ptr {
+ auto do_cache = CacheEnabled(cache_flag_);
+ auto action = CreateActionDigest(root_digest_, not do_cache);
+
+ if (logger != nullptr) {
+ logger->Emit(LogLevel::Trace,
+ "start execution\n"
+ " - exec_dir digest: {}\n"
+ " - action digest: {}",
+ root_digest_.hash(),
+ action.hash());
+ }
+
+ if (do_cache) {
+ if (auto result = storage_->CachedActionResult(action)) {
+ if (result->exit_code() == 0) {
+ return IExecutionResponse::Ptr{
+ new LocalResponse{action.hash(),
+ {std::move(*result), /*is_cached=*/true},
+ storage_}};
+ }
+ }
+ }
+
+ if (ExecutionEnabled(cache_flag_)) {
+ if (auto output = Run(action)) {
+ if (cache_flag_ == CacheFlag::PretendCached) {
+ // ensure the same id is created as if caching were enabled
+ auto action_id = CreateActionDigest(root_digest_, false).hash();
+ output->is_cached = true;
+ return IExecutionResponse::Ptr{new LocalResponse{
+ std::move(action_id), std::move(*output), storage_}};
+ }
+ return IExecutionResponse::Ptr{
+ new LocalResponse{action.hash(), std::move(*output), storage_}};
+ }
+ }
+
+ return nullptr;
+}
+
+auto LocalAction::Run(bazel_re::Digest const& action_id) const noexcept
+ -> std::optional<Output> {
+ auto exec_path = CreateUniquePath(LocalExecutionConfig::GetBuildDir() /
+ "exec_root" / action_id.hash());
+
+ if (not exec_path) {
+ return std::nullopt;
+ }
+
+ // anchor for cleaning up build directory at end of function (using RAII)
+ auto anchor = BuildCleanupAnchor(*exec_path);
+
+ auto const build_root = *exec_path / "build_root";
+ if (not CreateDirectoryStructure(build_root)) {
+ return std::nullopt;
+ }
+
+ if (cmdline_.empty()) {
+ logger_.Emit(LogLevel::Error, "malformed command line");
+ return std::nullopt;
+ }
+
+ auto cmdline = LocalExecutionConfig::GetLauncher();
+ std::copy(cmdline_.begin(), cmdline_.end(), std::back_inserter(cmdline));
+
+ SystemCommand system{"LocalExecution"};
+ auto const command_output =
+ system.Execute(cmdline, env_vars_, build_root, *exec_path);
+ if (command_output.has_value()) {
+ Output result{};
+ result.action.set_exit_code(command_output->return_value);
+ if (gsl::owner<bazel_re::Digest*> digest_ptr =
+ DigestFromFile(command_output->stdout_file)) {
+ result.action.set_allocated_stdout_digest(digest_ptr);
+ }
+ if (gsl::owner<bazel_re::Digest*> digest_ptr =
+ DigestFromFile(command_output->stderr_file)) {
+ result.action.set_allocated_stderr_digest(digest_ptr);
+ }
+
+ if (CollectAndStoreOutputs(&result.action, build_root)) {
+ if (cache_flag_ == CacheFlag::CacheOutput) {
+ if (not storage_->StoreActionResult(action_id, result.action)) {
+ logger_.Emit(LogLevel::Warning,
+ "failed to store action results");
+ }
+ }
+ }
+ return result;
+ }
+
+ logger_.Emit(LogLevel::Error, "failed to execute commands");
+
+ return std::nullopt;
+}
+
+auto LocalAction::StageFile(std::filesystem::path const& target_path,
+ Artifact::ObjectInfo const& info) const -> bool {
+ auto blob_path =
+ storage_->BlobPath(info.digest, IsExecutableObject(info.type));
+
+ return blob_path and
+ FileSystemManager::CreateDirectory(target_path.parent_path()) and
+ FileSystemManager::CreateFileHardlink(*blob_path, target_path);
+}
+
+auto LocalAction::StageInputFiles(
+ std::filesystem::path const& exec_path) const noexcept -> bool {
+ if (FileSystemManager::IsRelativePath(exec_path)) {
+ return false;
+ }
+
+ auto infos = storage_->ReadTreeInfos(root_digest_, exec_path);
+ if (not infos) {
+ return false;
+ }
+ for (std::size_t i{}; i < infos->first.size(); ++i) {
+ if (not StageFile(infos->first.at(i), infos->second.at(i))) {
+ return false;
+ }
+ }
+ return true;
+}
+
+auto LocalAction::CreateDirectoryStructure(
+ std::filesystem::path const& exec_path) const noexcept -> bool {
+ // clean execution directory
+ if (not FileSystemManager::RemoveDirectory(exec_path, true)) {
+ logger_.Emit(LogLevel::Error, "failed to clean exec_path");
+ return false;
+ }
+
+ // create process-exclusive execution directory
+ if (not FileSystemManager::CreateDirectoryExclusive(exec_path)) {
+ logger_.Emit(LogLevel::Error, "failed to exclusively create exec_path");
+ return false;
+ }
+
+ // stage input files to execution directory
+ if (not StageInputFiles(exec_path)) {
+ logger_.Emit(LogLevel::Error,
+ "failed to stage input files to exec_path");
+ return false;
+ }
+
+ // create output paths
+ for (auto const& local_path : output_files_) {
+ if (not FileSystemManager::CreateDirectory(
+ (exec_path / local_path).parent_path())) {
+ logger_.Emit(LogLevel::Error, "failed to create output directory");
+ return false;
+ }
+ }
+
+ return true;
+}
+
+auto LocalAction::CollectOutputFile(std::filesystem::path const& exec_path,
+ std::string const& local_path)
+ const noexcept -> std::optional<bazel_re::OutputFile> {
+ auto file_path = exec_path / local_path;
+ auto type = FileSystemManager::Type(file_path);
+ if (not type or not IsFileObject(*type)) {
+ Logger::Log(LogLevel::Error, "expected file at {}", local_path);
+ return std::nullopt;
+ }
+ bool is_executable = IsExecutableObject(*type);
+ auto digest = storage_->StoreBlob(file_path, is_executable);
+ if (digest) {
+ auto out_file = bazel_re::OutputFile{};
+ out_file.set_path(local_path);
+ out_file.set_allocated_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{*digest}});
+ out_file.set_is_executable(is_executable);
+ return out_file;
+ }
+ return std::nullopt;
+}
+
+auto LocalAction::CollectOutputDir(std::filesystem::path const& exec_path,
+ std::string const& local_path) const noexcept
+ -> std::optional<bazel_re::OutputDirectory> {
+ auto dir_path = exec_path / local_path;
+ auto type = FileSystemManager::Type(dir_path);
+ if (not type or not IsTreeObject(*type)) {
+ Logger::Log(LogLevel::Error, "expected directory at {}", local_path);
+ return std::nullopt;
+ }
+ auto digest = BazelMsgFactory::CreateDirectoryDigestFromLocalTree(
+ dir_path,
+ [this](auto path, auto is_exec) {
+ return storage_->StoreBlob(path, is_exec);
+ },
+ [this](auto bytes, auto dir) -> std::optional<bazel_re::Digest> {
+ auto digest = storage_->StoreBlob(bytes);
+ if (digest and not tree_map_->HasTree(*digest)) {
+ auto tree = tree_map_->CreateTree();
+ if (not BazelMsgFactory::ReadObjectInfosFromDirectory(
+ dir,
+ [&tree](auto path, auto info) {
+ return tree.AddInfo(path, info);
+ }) or
+ not tree_map_->AddTree(*digest, std::move(tree))) {
+ return std::nullopt;
+ }
+ }
+ return digest;
+ });
+ if (digest) {
+ auto out_dir = bazel_re::OutputDirectory{};
+ out_dir.set_path(local_path);
+ out_dir.set_allocated_tree_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{*digest}});
+ return out_dir;
+ }
+ return std::nullopt;
+}
+
+auto LocalAction::CollectAndStoreOutputs(
+ bazel_re::ActionResult* result,
+ std::filesystem::path const& exec_path) const noexcept -> bool {
+ logger_.Emit(LogLevel::Trace, "collecting outputs:");
+ for (auto const& path : output_files_) {
+ auto out_file = CollectOutputFile(exec_path, path);
+ if (not out_file) {
+ logger_.Emit(
+ LogLevel::Error, "could not collect output file {}", path);
+ return false;
+ }
+ auto const& digest = out_file->digest().hash();
+ logger_.Emit(LogLevel::Trace, " - file {}: {}", path, digest);
+ result->mutable_output_files()->Add(std::move(*out_file));
+ }
+ for (auto const& path : output_dirs_) {
+ auto out_dir = CollectOutputDir(exec_path, path);
+ if (not out_dir) {
+ logger_.Emit(
+ LogLevel::Error, "could not collect output dir {}", path);
+ return false;
+ }
+ auto const& digest = out_dir->tree_digest().hash();
+ logger_.Emit(LogLevel::Trace, " - dir {}: {}", path, digest);
+ result->mutable_output_directories()->Add(std::move(*out_dir));
+ }
+
+ return true;
+}
+
+auto LocalAction::DigestFromFile(std::filesystem::path const& file_path)
+ const noexcept -> gsl::owner<bazel_re::Digest*> {
+ if (auto digest = storage_->StoreBlob(file_path)) {
+ return new bazel_re::Digest{std::move(*digest)};
+ }
+ return nullptr;
+}
diff --git a/src/buildtool/execution_api/local/local_action.hpp b/src/buildtool/execution_api/local/local_action.hpp
new file mode 100644
index 00000000..3bf49fd2
--- /dev/null
+++ b/src/buildtool/execution_api/local/local_action.hpp
@@ -0,0 +1,122 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_ACTION_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_ACTION_HPP
+
+#include <chrono>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
+#include "src/buildtool/execution_api/common/execution_action.hpp"
+#include "src/buildtool/execution_api/common/execution_response.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/execution_api/local/local_storage.hpp"
+
+class LocalApi;
+
+/// \brief Action for local execution.
+class LocalAction final : public IExecutionAction {
+ friend class LocalApi;
+
+ public:
+ struct Output {
+ bazel_re::ActionResult action{};
+ bool is_cached{};
+ };
+
+ auto Execute(Logger const* logger) noexcept
+ -> IExecutionResponse::Ptr final;
+
+ void SetCacheFlag(CacheFlag flag) noexcept final { cache_flag_ = flag; }
+
+ void SetTimeout(std::chrono::milliseconds timeout) noexcept final {
+ timeout_ = timeout;
+ }
+
+ private:
+ Logger logger_{"LocalExecution"};
+ std::shared_ptr<LocalStorage> storage_;
+ std::shared_ptr<LocalTreeMap> tree_map_;
+ ArtifactDigest root_digest_{};
+ std::vector<std::string> cmdline_{};
+ std::vector<std::string> output_files_{};
+ std::vector<std::string> output_dirs_{};
+ std::map<std::string, std::string> env_vars_{};
+ std::vector<bazel_re::Platform_Property> properties_;
+ std::chrono::milliseconds timeout_{kDefaultTimeout};
+ CacheFlag cache_flag_{CacheFlag::CacheOutput};
+
+ LocalAction(std::shared_ptr<LocalStorage> storage,
+ std::shared_ptr<LocalTreeMap> tree_map,
+ ArtifactDigest root_digest,
+ std::vector<std::string> command,
+ std::vector<std::string> output_files,
+ std::vector<std::string> output_dirs,
+ std::map<std::string, std::string> env_vars,
+ std::map<std::string, std::string> const& properties) noexcept
+ : storage_{std::move(storage)},
+ tree_map_{std::move(tree_map)},
+ root_digest_{std::move(root_digest)},
+ cmdline_{std::move(command)},
+ output_files_{std::move(output_files)},
+ output_dirs_{std::move(output_dirs)},
+ env_vars_{std::move(env_vars)},
+ properties_{BazelMsgFactory::CreateMessageVectorFromMap<
+ bazel_re::Platform_Property>(properties)} {
+ std::sort(output_files_.begin(), output_files_.end());
+ std::sort(output_dirs_.begin(), output_dirs_.end());
+ }
+
+ [[nodiscard]] auto CreateActionDigest(bazel_re::Digest const& exec_dir,
+ bool do_not_cache)
+ -> bazel_re::Digest {
+ return BazelMsgFactory::CreateActionDigestFromCommandLine(
+ cmdline_,
+ exec_dir,
+ output_files_,
+ output_dirs_,
+ {} /*FIXME output node properties*/,
+ BazelMsgFactory::CreateMessageVectorFromMap<
+ bazel_re::Command_EnvironmentVariable>(env_vars_),
+ properties_,
+ do_not_cache,
+ timeout_);
+ }
+
+ [[nodiscard]] auto Run(bazel_re::Digest const& action_id) const noexcept
+ -> std::optional<Output>;
+
+ [[nodiscard]] auto StageFile(std::filesystem::path const& target_path,
+ Artifact::ObjectInfo const& info) const
+ -> bool;
+
+ /// \brief Stage input artifacts to the execution directory.
+ /// Stage artifacts and their parent directory structure from CAS to the
+ /// specified execution directory. The execution directory may no exist.
+ /// \param[in] exec_path Absolute path to the execution directory.
+ /// \returns Success indicator.
+ [[nodiscard]] auto StageInputFiles(
+ std::filesystem::path const& exec_path) const noexcept -> bool;
+
+ [[nodiscard]] auto CreateDirectoryStructure(
+ std::filesystem::path const& exec_path) const noexcept -> bool;
+
+ [[nodiscard]] auto CollectOutputFile(std::filesystem::path const& exec_path,
+ std::string const& local_path)
+ const noexcept -> std::optional<bazel_re::OutputFile>;
+
+ [[nodiscard]] auto CollectOutputDir(std::filesystem::path const& exec_path,
+ std::string const& local_path)
+ const noexcept -> std::optional<bazel_re::OutputDirectory>;
+
+ [[nodiscard]] auto CollectAndStoreOutputs(
+ bazel_re::ActionResult* result,
+ std::filesystem::path const& exec_path) const noexcept -> bool;
+
+ /// \brief Store file from path in file CAS and return pointer to digest.
+ [[nodiscard]] auto DigestFromFile(std::filesystem::path const& file_path)
+ const noexcept -> gsl::owner<bazel_re::Digest*>;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_ACTION_HPP
diff --git a/src/buildtool/execution_api/local/local_api.hpp b/src/buildtool/execution_api/local/local_api.hpp
new file mode 100644
index 00000000..96b96416
--- /dev/null
+++ b/src/buildtool/execution_api/local/local_api.hpp
@@ -0,0 +1,157 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/common/local_tree_map.hpp"
+#include "src/buildtool/execution_api/local/local_action.hpp"
+#include "src/buildtool/execution_api/local/local_storage.hpp"
+
+/// \brief API for local execution.
+class LocalApi final : public IExecutionApi {
+ public:
+ auto CreateAction(
+ ArtifactDigest const& root_digest,
+ std::vector<std::string> const& command,
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) noexcept
+ -> IExecutionAction::Ptr final {
+ return IExecutionAction::Ptr{new LocalAction{storage_,
+ tree_map_,
+ root_digest,
+ command,
+ output_files,
+ output_dirs,
+ env_vars,
+ properties}};
+ }
+
+ [[nodiscard]] auto RetrieveToPaths(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<std::filesystem::path> const& output_paths) noexcept
+ -> bool final {
+ if (artifacts_info.size() != output_paths.size()) {
+ Logger::Log(LogLevel::Error,
+ "different number of digests and output paths.");
+ return false;
+ }
+
+ for (std::size_t i{}; i < artifacts_info.size(); ++i) {
+ auto const& info = artifacts_info[i];
+ if (IsTreeObject(info.type)) {
+ // read object infos from sub tree and call retrieve recursively
+ auto const infos =
+ storage_->ReadTreeInfos(info.digest, output_paths[i]);
+ if (not infos or
+ not RetrieveToPaths(infos->second, infos->first)) {
+ return false;
+ }
+ }
+ else {
+ auto const blob_path = storage_->BlobPath(
+ info.digest, IsExecutableObject(info.type));
+ if (not blob_path or
+ not FileSystemManager::CreateDirectory(
+ output_paths[i].parent_path()) or
+ not FileSystemManager::CopyFileAs(
+ *blob_path, output_paths[i], info.type)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ [[nodiscard]] auto RetrieveToFds(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<int> const& fds) noexcept -> bool final {
+ if (artifacts_info.size() != fds.size()) {
+ Logger::Log(LogLevel::Error,
+ "different number of digests and file descriptors.");
+ return false;
+ }
+
+ for (std::size_t i{}; i < artifacts_info.size(); ++i) {
+ auto fd = fds[i];
+ auto const& info = artifacts_info[i];
+
+ if (gsl::owner<FILE*> out = fdopen(fd, "wb")) { // NOLINT
+ auto const success = storage_->DumpToStream(info, out);
+ std::fclose(out);
+ if (not success) {
+ Logger::Log(LogLevel::Error,
+ "dumping {} {} to file descriptor {} failed.",
+ IsTreeObject(info.type) ? "tree" : "blob",
+ info.ToString(),
+ fd);
+ return false;
+ }
+ }
+ else {
+ Logger::Log(LogLevel::Error,
+ "dumping to file descriptor {} failed.",
+ fd);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ [[nodiscard]] auto Upload(BlobContainer const& blobs,
+ bool /*skip_find_missing*/) noexcept
+ -> bool final {
+ for (auto const& blob : blobs) {
+ auto cas_digest = storage_->StoreBlob(blob.data);
+ if (not cas_digest or not std::equal_to<bazel_re::Digest>{}(
+ *cas_digest, blob.digest)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ [[nodiscard]] auto UploadTree(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const&
+ artifacts) noexcept -> std::optional<ArtifactDigest> final {
+ BlobContainer blobs{};
+ auto tree = tree_map_->CreateTree();
+ auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree(
+ artifacts,
+ [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); },
+ [&tree](auto path, auto info) { return tree.AddInfo(path, info); });
+ if (not digest) {
+ Logger::Log(LogLevel::Debug, "failed to create digest for tree.");
+ return std::nullopt;
+ }
+
+ if (not Upload(blobs, /*skip_find_missing=*/false)) {
+ Logger::Log(LogLevel::Debug, "failed to upload blobs for tree.");
+ return std::nullopt;
+ }
+
+ if (tree_map_->AddTree(*digest, std::move(tree))) {
+ return ArtifactDigest{std::move(*digest)};
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept
+ -> bool final {
+ return storage_->BlobPath(digest, false).has_value();
+ }
+
+ private:
+ std::shared_ptr<LocalTreeMap> tree_map_{std::make_shared<LocalTreeMap>()};
+ std::shared_ptr<LocalStorage> storage_{
+ std::make_shared<LocalStorage>(tree_map_)};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_API_HPP
diff --git a/src/buildtool/execution_api/local/local_cas.hpp b/src/buildtool/execution_api/local/local_cas.hpp
new file mode 100644
index 00000000..4a28e796
--- /dev/null
+++ b/src/buildtool/execution_api/local/local_cas.hpp
@@ -0,0 +1,103 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP
+
+#include <sstream>
+#include <thread>
+
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/execution_api/local/config.hpp"
+#include "src/buildtool/execution_api/local/file_storage.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+template <ObjectType kType = ObjectType::File>
+class LocalCAS {
+ public:
+ LocalCAS() noexcept = default;
+
+ explicit LocalCAS(std::filesystem::path cache_root) noexcept
+ : cache_root_{std::move(cache_root)} {}
+
+ LocalCAS(LocalCAS const&) = delete;
+ LocalCAS(LocalCAS&&) = delete;
+ auto operator=(LocalCAS const&) -> LocalCAS& = delete;
+ auto operator=(LocalCAS &&) -> LocalCAS& = delete;
+ ~LocalCAS() noexcept = default;
+
+ [[nodiscard]] auto StoreBlobFromBytes(std::string const& bytes)
+ const noexcept -> std::optional<bazel_re::Digest> {
+ return StoreBlob(bytes);
+ }
+
+ [[nodiscard]] auto StoreBlobFromFile(std::filesystem::path const& file_path)
+ const noexcept -> std::optional<bazel_re::Digest> {
+ return StoreBlob(file_path);
+ }
+
+ [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest) const noexcept
+ -> std::optional<std::filesystem::path> {
+ auto blob_path = file_store_.GetPath(digest.hash());
+ if (FileSystemManager::IsFile(blob_path)) {
+ return blob_path;
+ }
+ logger_.Emit(LogLevel::Debug, "Blob not found {}", digest.hash());
+ return std::nullopt;
+ }
+
+ private:
+ static constexpr char kSuffix = ToChar(kType);
+ Logger logger_{std::string{"LocalCAS"} + kSuffix};
+ std::filesystem::path const cache_root_{
+ LocalExecutionConfig::GetCacheDir()};
+ FileStorage<kType> file_store_{cache_root_ /
+ (std::string{"cas"} + kSuffix)};
+
+ [[nodiscard]] static auto CreateDigest(std::string const& bytes) noexcept
+ -> std::optional<bazel_re::Digest> {
+ return ArtifactDigest::Create(bytes);
+ }
+
+ [[nodiscard]] static auto CreateDigest(
+ std::filesystem::path const& file_path) noexcept
+ -> std::optional<bazel_re::Digest> {
+ auto const bytes = FileSystemManager::ReadFile(file_path);
+ if (bytes.has_value()) {
+ return ArtifactDigest::Create(*bytes);
+ }
+ return std::nullopt;
+ }
+
+ /// \brief Store blob from bytes to storage.
+ [[nodiscard]] auto StoreBlobData(std::string const& blob_id,
+ std::string const& bytes) const noexcept
+ -> bool {
+ return file_store_.AddFromBytes(blob_id, bytes);
+ }
+
+ /// \brief Store blob from file path to storage.
+ [[nodiscard]] auto StoreBlobData(
+ std::string const& blob_id,
+ std::filesystem::path const& file_path) const noexcept -> bool {
+ return file_store_.AddFromFile(blob_id, file_path);
+ }
+
+ /// \brief Store blob from unspecified data to storage.
+ template <class T>
+ [[nodiscard]] auto StoreBlob(T const& data) const noexcept
+ -> std::optional<bazel_re::Digest> {
+ auto digest = CreateDigest(data);
+ if (digest) {
+ if (StoreBlobData(digest->hash(), data)) {
+ return digest;
+ }
+ logger_.Emit(
+ LogLevel::Debug, "Failed to store blob {}.", digest->hash());
+ }
+ logger_.Emit(LogLevel::Debug, "Failed to create digest.");
+ return std::nullopt;
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_CAS_HPP
diff --git a/src/buildtool/execution_api/local/local_response.hpp b/src/buildtool/execution_api/local/local_response.hpp
new file mode 100644
index 00000000..9084be0b
--- /dev/null
+++ b/src/buildtool/execution_api/local/local_response.hpp
@@ -0,0 +1,101 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_RESPONSE_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_RESPONSE_HPP
+
+#include "src/buildtool/execution_api/common/execution_response.hpp"
+#include "src/buildtool/execution_api/local/local_action.hpp"
+#include "src/buildtool/execution_api/local/local_storage.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+
+/// \brief Response of a LocalAction.
+class LocalResponse final : public IExecutionResponse {
+ friend class LocalAction;
+
+ public:
+ auto Status() const noexcept -> StatusCode final {
+ return StatusCode::Success; // unused
+ }
+ auto HasStdErr() const noexcept -> bool final {
+ return (output_.action.stderr_digest().size_bytes() != 0);
+ }
+ auto HasStdOut() const noexcept -> bool final {
+ return (output_.action.stdout_digest().size_bytes() != 0);
+ }
+ auto StdErr() noexcept -> std::string final {
+ if (auto path = storage_->BlobPath(output_.action.stderr_digest(),
+ /*is_executable=*/false)) {
+ if (auto content = FileSystemManager::ReadFile(*path)) {
+ return std::move(*content);
+ }
+ }
+ Logger::Log(LogLevel::Debug, "reading stderr failed");
+ return {};
+ }
+ auto StdOut() noexcept -> std::string final {
+ if (auto path = storage_->BlobPath(output_.action.stdout_digest(),
+ /*is_executable=*/false)) {
+ if (auto content = FileSystemManager::ReadFile(*path)) {
+ return std::move(*content);
+ }
+ }
+ Logger::Log(LogLevel::Debug, "reading stdout failed");
+ return {};
+ }
+ auto ExitCode() const noexcept -> int final {
+ return output_.action.exit_code();
+ }
+ auto IsCached() const noexcept -> bool final { return output_.is_cached; };
+
+ auto ActionDigest() const noexcept -> std::string final {
+ return action_id_;
+ }
+
+ auto Artifacts() const noexcept -> ArtifactInfos final {
+ ArtifactInfos artifacts{};
+ auto const& action_result = output_.action;
+ artifacts.reserve(
+ static_cast<std::size_t>(action_result.output_files().size()));
+
+ // collect files and store them
+ for (auto const& file : action_result.output_files()) {
+ try {
+ artifacts.emplace(
+ file.path(),
+ Artifact::ObjectInfo{ArtifactDigest{file.digest()},
+ file.is_executable()
+ ? ObjectType::Executable
+ : ObjectType::File});
+ } catch (...) {
+ return {};
+ }
+ }
+
+ // collect directories and store them
+ for (auto const& dir : action_result.output_directories()) {
+ try {
+ artifacts.emplace(
+ dir.path(),
+ Artifact::ObjectInfo{ArtifactDigest{dir.tree_digest()},
+ ObjectType::Tree});
+ } catch (...) {
+ return {};
+ }
+ }
+
+ return artifacts;
+ };
+
+ private:
+ std::string action_id_{};
+ LocalAction::Output output_{};
+ gsl::not_null<std::shared_ptr<LocalStorage>> storage_;
+
+ explicit LocalResponse(
+ std::string action_id,
+ LocalAction::Output output,
+ gsl::not_null<std::shared_ptr<LocalStorage>> storage) noexcept
+ : action_id_{std::move(action_id)},
+ output_{std::move(output)},
+ storage_{std::move(storage)} {}
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_RESPONSE_HPP
diff --git a/src/buildtool/execution_api/local/local_storage.cpp b/src/buildtool/execution_api/local/local_storage.cpp
new file mode 100644
index 00000000..b4fe1658
--- /dev/null
+++ b/src/buildtool/execution_api/local/local_storage.cpp
@@ -0,0 +1,125 @@
+#include "src/buildtool/execution_api/local/local_api.hpp"
+
+namespace {
+
+[[nodiscard]] auto ReadDirectory(
+ gsl::not_null<LocalStorage const*> const& storage,
+ bazel_re::Digest const& digest) noexcept
+ -> std::optional<bazel_re::Directory> {
+ if (auto const path = storage->BlobPath(digest, /*is_executable=*/false)) {
+ if (auto const content = FileSystemManager::ReadFile(*path)) {
+ return BazelMsgFactory::MessageFromString<bazel_re::Directory>(
+ *content);
+ }
+ }
+ Logger::Log(
+ LogLevel::Error, "Directory {} not found in CAS", digest.hash());
+ return std::nullopt;
+}
+
+[[nodiscard]] auto TreeToStream(
+ gsl::not_null<LocalStorage const*> const& storage,
+ bazel_re::Digest const& tree_digest,
+ gsl::not_null<FILE*> const& stream) noexcept -> bool {
+ if (auto dir = ReadDirectory(storage, tree_digest)) {
+ if (auto data = BazelMsgFactory::DirectoryToString(*dir)) {
+ std::fwrite(data->data(), 1, data->size(), stream);
+ return true;
+ }
+ }
+ return false;
+}
+
+[[nodiscard]] auto BlobToStream(
+ gsl::not_null<LocalStorage const*> const& storage,
+ Artifact::ObjectInfo const& blob_info,
+ gsl::not_null<FILE*> const& stream) noexcept -> bool {
+ constexpr std::size_t kChunkSize{512};
+ if (auto const path = storage->BlobPath(
+ blob_info.digest, IsExecutableObject(blob_info.type))) {
+ std::string data(kChunkSize, '\0');
+ if (gsl::owner<FILE*> in = std::fopen(path->c_str(), "rb")) {
+ while (auto size = std::fread(data.data(), 1, kChunkSize, in)) {
+ std::fwrite(data.data(), 1, size, stream);
+ }
+ std::fclose(in);
+ return true;
+ }
+ }
+ return false;
+}
+
+} // namespace
+
+auto LocalStorage::ReadTreeInfos(
+ bazel_re::Digest const& tree_digest,
+ std::filesystem::path const& parent) const noexcept
+ -> std::optional<std::pair<std::vector<std::filesystem::path>,
+ std::vector<Artifact::ObjectInfo>>> {
+ std::vector<std::filesystem::path> paths{};
+ std::vector<Artifact::ObjectInfo> infos{};
+
+ auto store_info = [&paths, &infos](auto path, auto info) {
+ paths.emplace_back(path);
+ infos.emplace_back(info);
+ return true;
+ };
+
+ if (ReadObjectInfosRecursively(store_info, parent, tree_digest)) {
+ return std::make_pair(std::move(paths), std::move(infos));
+ }
+ return std::nullopt;
+}
+
+auto LocalStorage::ReadObjectInfosRecursively(
+ BazelMsgFactory::InfoStoreFunc const& store_info,
+ std::filesystem::path const& parent,
+ bazel_re::Digest const& digest) const noexcept -> bool {
+ // read from in-memory tree map
+ if (tree_map_) {
+ auto const* tree = tree_map_->GetTree(digest);
+ if (tree != nullptr) {
+ for (auto const& [path, info] : *tree) {
+ try {
+ if (IsTreeObject(info->type)
+ ? not ReadObjectInfosRecursively(
+ store_info, parent / path, info->digest)
+ : not store_info(parent / path, *info)) {
+ return false;
+ }
+ } catch (...) { // satisfy clang-tidy, store_info() could throw
+ return false;
+ }
+ }
+ return true;
+ }
+ Logger::Log(
+ LogLevel::Debug, "tree {} not found in tree map", digest.hash());
+ }
+
+ // fallback read from CAS and cache it in in-memory tree map
+ if (auto dir = ReadDirectory(this, digest)) {
+ auto tree = tree_map_ ? std::make_optional(tree_map_->CreateTree())
+ : std::nullopt;
+ return BazelMsgFactory::ReadObjectInfosFromDirectory(
+ *dir,
+ [this, &store_info, &parent, &tree](auto path, auto info) {
+ return IsTreeObject(info.type)
+ ? (not tree or tree->AddInfo(path, info)) and
+ ReadObjectInfosRecursively(
+ store_info,
+ parent / path,
+ info.digest)
+ : store_info(parent / path, info);
+ }) and
+ (not tree_map_ or tree_map_->AddTree(digest, std::move(*tree)));
+ }
+ return false;
+}
+
+auto LocalStorage::DumpToStream(
+ Artifact::ObjectInfo const& info,
+ gsl::not_null<FILE*> const& stream) const noexcept -> bool {
+ return IsTreeObject(info.type) ? TreeToStream(this, info.digest, stream)
+ : BlobToStream(this, info, stream);
+}
diff --git a/src/buildtool/execution_api/local/local_storage.hpp b/src/buildtool/execution_api/local/local_storage.hpp
new file mode 100644
index 00000000..0b90cf63
--- /dev/null
+++ b/src/buildtool/execution_api/local/local_storage.hpp
@@ -0,0 +1,109 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP
+
+#include <optional>
+
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/execution_api/common/local_tree_map.hpp"
+#include "src/buildtool/execution_api/local/local_ac.hpp"
+#include "src/buildtool/execution_api/local/local_cas.hpp"
+
+class LocalStorage {
+ public:
+ explicit LocalStorage(
+ std::shared_ptr<LocalTreeMap> tree_map = nullptr) noexcept
+ : tree_map_{std::move(tree_map)} {}
+
+ explicit LocalStorage(
+ std::filesystem::path const& cache_root,
+ std::shared_ptr<LocalTreeMap> tree_map = nullptr) noexcept
+ : cas_file_{cache_root},
+ cas_exec_{cache_root},
+ ac_{&cas_file_, cache_root},
+ tree_map_{std::move(tree_map)} {}
+
+ /// \brief Store blob from file path with x-bit determined from file system.
+ [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path)
+ const noexcept -> std::optional<bazel_re::Digest> {
+ return StoreBlob(file_path, FileSystemManager::IsExecutable(file_path));
+ }
+
+ /// \brief Store blob from file path with x-bit.
+ [[nodiscard]] auto StoreBlob(std::filesystem::path const& file_path,
+ bool is_executable) const noexcept
+ -> std::optional<bazel_re::Digest> {
+ if (is_executable) {
+ return cas_exec_.StoreBlobFromFile(file_path);
+ }
+ return cas_file_.StoreBlobFromFile(file_path);
+ }
+
+ /// \brief Store blob from bytes with x-bit (default: non-executable).
+ [[nodiscard]] auto StoreBlob(std::string const& bytes,
+ bool is_executable = false) const noexcept
+ -> std::optional<bazel_re::Digest> {
+ return is_executable ? cas_exec_.StoreBlobFromBytes(bytes)
+ : cas_file_.StoreBlobFromBytes(bytes);
+ }
+
+ /// \brief Obtain blob path from digest with x-bit.
+ [[nodiscard]] auto BlobPath(bazel_re::Digest const& digest,
+ bool is_executable) const noexcept
+ -> std::optional<std::filesystem::path> {
+ auto const path = is_executable ? cas_exec_.BlobPath(digest)
+ : cas_file_.BlobPath(digest);
+ return path ? path : TrySyncBlob(digest, is_executable);
+ }
+
+ [[nodiscard]] auto StoreActionResult(
+ bazel_re::Digest const& action_id,
+ bazel_re::ActionResult const& result) const noexcept -> bool {
+ return ac_.StoreResult(action_id, result);
+ }
+
+ [[nodiscard]] auto CachedActionResult(bazel_re::Digest const& action_id)
+ const noexcept -> std::optional<bazel_re::ActionResult> {
+ return ac_.CachedResult(action_id);
+ }
+
+ [[nodiscard]] auto ReadTreeInfos(
+ bazel_re::Digest const& tree_digest,
+ std::filesystem::path const& parent) const noexcept
+ -> std::optional<std::pair<std::vector<std::filesystem::path>,
+ std::vector<Artifact::ObjectInfo>>>;
+
+ [[nodiscard]] auto DumpToStream(
+ Artifact::ObjectInfo const& info,
+ gsl::not_null<FILE*> const& stream) const noexcept -> bool;
+
+ private:
+ LocalCAS<ObjectType::File> cas_file_{};
+ LocalCAS<ObjectType::Executable> cas_exec_{};
+ LocalAC ac_{&cas_file_};
+ std::shared_ptr<LocalTreeMap> tree_map_;
+
+ /// \brief Try to sync blob between file CAS and executable CAS.
+ /// \param digest Blob digest.
+ /// \param to_executable Sync direction.
+ /// \returns Path to blob in target CAS.
+ [[nodiscard]] auto TrySyncBlob(bazel_re::Digest const& digest,
+ bool to_executable) const noexcept
+ -> std::optional<std::filesystem::path> {
+ std::optional<std::filesystem::path> const src_blob{
+ to_executable ? cas_file_.BlobPath(digest)
+ : cas_exec_.BlobPath(digest)};
+ if (src_blob and StoreBlob(*src_blob, to_executable)) {
+ return BlobPath(digest, to_executable);
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto ReadObjectInfosRecursively(
+ BazelMsgFactory::InfoStoreFunc const& store_info,
+ std::filesystem::path const& parent,
+ bazel_re::Digest const& digest) const noexcept -> bool;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_LOCAL_LOCAL_STORAGE_HPP
diff --git a/src/buildtool/execution_api/remote/TARGETS b/src/buildtool/execution_api/remote/TARGETS
new file mode 100644
index 00000000..6f9fc004
--- /dev/null
+++ b/src/buildtool/execution_api/remote/TARGETS
@@ -0,0 +1,59 @@
+{ "bazel_network":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["bazel_network"]
+ , "hdrs":
+ [ "bazel/bytestream_client.hpp"
+ , "bazel/bazel_client_common.hpp"
+ , "bazel/bazel_action.hpp"
+ , "bazel/bazel_response.hpp"
+ , "bazel/bazel_network.hpp"
+ , "bazel/bazel_ac_client.hpp"
+ , "bazel/bazel_cas_client.hpp"
+ , "bazel/bazel_execution_client.hpp"
+ ]
+ , "srcs":
+ [ "bazel/bazel_action.cpp"
+ , "bazel/bazel_response.cpp"
+ , "bazel/bazel_network.cpp"
+ , "bazel/bazel_ac_client.cpp"
+ , "bazel/bazel_cas_client.cpp"
+ , "bazel/bazel_execution_client.cpp"
+ ]
+ , "deps":
+ [ ["src/buildtool/common", "common"]
+ , ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg_factory"]
+ , ["@", "grpc", "", "grpc++"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "proto":
+ [ ["@", "bazel_remote_apis", "", "remote_execution_proto"]
+ , ["@", "googleapis", "", "google_bytestream_proto"]
+ ]
+ , "stage": ["src", "buildtool", "execution_api", "remote"]
+ }
+, "bazel":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["bazel"]
+ , "hdrs": ["bazel/bazel_api.hpp"]
+ , "srcs": ["bazel/bazel_api.cpp"]
+ , "deps":
+ [ "bazel_network"
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "execution_api", "remote"]
+ }
+, "config":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["config"]
+ , "hdrs": ["config.hpp"]
+ , "deps":
+ [["src/buildtool/logging", "logging"], ["@", "gsl-lite", "", "gsl-lite"]]
+ , "stage": ["src", "buildtool", "execution_api", "remote"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_ac_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_ac_client.cpp
new file mode 100644
index 00000000..d4c5095d
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_ac_client.cpp
@@ -0,0 +1,75 @@
+#include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp"
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp"
+
+BazelAcClient::BazelAcClient(std::string const& server,
+ Port port,
+ std::string const& user,
+ std::string const& pwd) noexcept {
+ stub_ = bazel_re::ActionCache::NewStub(
+ CreateChannelWithCredentials(server, port, user, pwd));
+}
+
+auto BazelAcClient::GetActionResult(
+ std::string const& instance_name,
+ bazel_re::Digest const& action_digest,
+ bool inline_stdout,
+ bool inline_stderr,
+ std::vector<std::string> const& inline_output_files) noexcept
+ -> std::optional<bazel_re::ActionResult> {
+ bazel_re::GetActionResultRequest request{};
+ request.set_instance_name(instance_name);
+ request.set_allocated_action_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{action_digest}});
+ request.set_inline_stdout(inline_stdout);
+ request.set_inline_stderr(inline_stderr);
+ std::copy(inline_output_files.begin(),
+ inline_output_files.end(),
+ pb::back_inserter(request.mutable_inline_output_files()));
+
+ grpc::ClientContext context;
+ bazel_re::ActionResult response;
+ grpc::Status status = stub_->GetActionResult(&context, request, &response);
+
+ if (not status.ok()) {
+ if (status.error_code() == grpc::StatusCode::NOT_FOUND) {
+ logger_.Emit(
+ LogLevel::Debug, "cache miss '{}'", status.error_message());
+ }
+ else {
+ LogStatus(&logger_, LogLevel::Debug, status);
+ }
+ return std::nullopt;
+ }
+ return response;
+}
+
+auto BazelAcClient::UpdateActionResult(std::string const& instance_name,
+ bazel_re::Digest const& action_digest,
+ bazel_re::ActionResult const& result,
+ int priority) noexcept
+ -> std::optional<bazel_re::ActionResult> {
+ auto policy = std::make_unique<bazel_re::ResultsCachePolicy>();
+ policy->set_priority(priority);
+
+ bazel_re::UpdateActionResultRequest request{};
+ request.set_instance_name(instance_name);
+ request.set_allocated_action_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{action_digest}});
+ request.set_allocated_action_result(gsl::owner<bazel_re::ActionResult*>{
+ new bazel_re::ActionResult{result}});
+ request.set_allocated_results_cache_policy(policy.release());
+
+ grpc::ClientContext context;
+ bazel_re::ActionResult response;
+ grpc::Status status =
+ stub_->UpdateActionResult(&context, request, &response);
+
+ if (not status.ok()) {
+ LogStatus(&logger_, LogLevel::Debug, status);
+ return std::nullopt;
+ }
+ return response;
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp
new file mode 100644
index 00000000..b9514d8a
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp
@@ -0,0 +1,41 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_AC_CLIENT_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_AC_CLIENT_HPP
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// Implements client side for serivce defined here:
+/// https://github.com/bazelbuild/bazel/blob/4b6ad34dbba15dacebfb6cbf76fa741649cdb007/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto#L137
+class BazelAcClient {
+ public:
+ BazelAcClient(std::string const& server,
+ Port port,
+ std::string const& user = "",
+ std::string const& pwd = "") noexcept;
+
+ [[nodiscard]] auto GetActionResult(
+ std::string const& instance_name,
+ bazel_re::Digest const& action_digest,
+ bool inline_stdout,
+ bool inline_stderr,
+ std::vector<std::string> const& inline_output_files) noexcept
+ -> std::optional<bazel_re::ActionResult>;
+
+ [[nodiscard]] auto UpdateActionResult(std::string const& instance_name,
+ bazel_re::Digest const& digest,
+ bazel_re::ActionResult const& result,
+ int priority) noexcept
+ -> std::optional<bazel_re::ActionResult>;
+
+ private:
+ std::unique_ptr<bazel_re::ActionCache::Stub> stub_;
+ Logger logger_{"BazelAcClient"};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_AC_CLIENT_HPP
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_action.cpp b/src/buildtool/execution_api/remote/bazel/bazel_action.cpp
new file mode 100644
index 00000000..34fc5380
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_action.cpp
@@ -0,0 +1,94 @@
+#include "src/buildtool/execution_api/remote/bazel/bazel_action.hpp"
+
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_response.hpp"
+
+BazelAction::BazelAction(
+ std::shared_ptr<BazelNetwork> network,
+ std::shared_ptr<LocalTreeMap> tree_map,
+ bazel_re::Digest root_digest,
+ std::vector<std::string> command,
+ std::vector<std::string> output_files,
+ std::vector<std::string> output_dirs,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) noexcept
+ : network_{std::move(network)},
+ tree_map_{std::move(tree_map)},
+ root_digest_{std::move(root_digest)},
+ cmdline_{std::move(command)},
+ output_files_{std::move(output_files)},
+ output_dirs_{std::move(output_dirs)},
+ env_vars_{BazelMsgFactory::CreateMessageVectorFromMap<
+ bazel_re::Command_EnvironmentVariable>(env_vars)},
+ properties_{BazelMsgFactory::CreateMessageVectorFromMap<
+ bazel_re::Platform_Property>(properties)} {
+ std::sort(output_files_.begin(), output_files_.end());
+ std::sort(output_dirs_.begin(), output_dirs_.end());
+}
+
+auto BazelAction::Execute(Logger const* logger) noexcept
+ -> IExecutionResponse::Ptr {
+ BlobContainer blobs{};
+ auto do_cache = CacheEnabled(cache_flag_);
+ auto action = CreateBundlesForAction(&blobs, root_digest_, not do_cache);
+
+ if (logger != nullptr) {
+ logger->Emit(LogLevel::Trace,
+ "start execution\n"
+ " - exec_dir digest: {}\n"
+ " - action digest: {}",
+ root_digest_.hash(),
+ action.hash());
+ }
+
+ if (do_cache) {
+ if (auto result =
+ network_->GetCachedActionResult(action, output_files_)) {
+ if (result->exit_code() == 0) {
+ return IExecutionResponse::Ptr{new BazelResponse{
+ action.hash(), network_, tree_map_, {*result, true}}};
+ }
+ }
+ }
+
+ if (ExecutionEnabled(cache_flag_) and network_->UploadBlobs(blobs)) {
+ if (auto output = network_->ExecuteBazelActionSync(action)) {
+ if (cache_flag_ == CacheFlag::PretendCached) {
+ // ensure the same id is created as if caching were enabled
+ auto action_id =
+ CreateBundlesForAction(nullptr, root_digest_, false).hash();
+ output->cached_result = true;
+ return IExecutionResponse::Ptr{
+ new BazelResponse{std::move(action_id),
+ network_,
+ tree_map_,
+ std::move(*output)}};
+ }
+ return IExecutionResponse::Ptr{new BazelResponse{
+ action.hash(), network_, tree_map_, std::move(*output)}};
+ }
+ }
+
+ return nullptr;
+}
+
+auto BazelAction::CreateBundlesForAction(BlobContainer* blobs,
+ bazel_re::Digest const& exec_dir,
+ bool do_not_cache) const noexcept
+ -> bazel_re::Digest {
+ return BazelMsgFactory::CreateActionDigestFromCommandLine(
+ cmdline_,
+ exec_dir,
+ output_files_,
+ output_dirs_,
+ {} /*FIXME output node properties*/,
+ env_vars_,
+ properties_,
+ do_not_cache,
+ timeout_,
+ blobs == nullptr ? std::nullopt
+ : std::make_optional([&blobs](BazelBlob&& blob) {
+ blobs->Emplace(std::move(blob));
+ }));
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_action.hpp b/src/buildtool/execution_api/remote/bazel/bazel_action.hpp
new file mode 100644
index 00000000..7eb9a9e0
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_action.hpp
@@ -0,0 +1,54 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_ACTION_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_ACTION_HPP
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp"
+
+class BazelApi;
+
+/// \brief Bazel implementation of the abstract Execution Action.
+/// Uploads all dependencies, creates a Bazel Action and executes it.
+class BazelAction final : public IExecutionAction {
+ friend class BazelApi;
+
+ public:
+ auto Execute(Logger const* logger) noexcept
+ -> IExecutionResponse::Ptr final;
+ void SetCacheFlag(CacheFlag flag) noexcept final { cache_flag_ = flag; }
+ void SetTimeout(std::chrono::milliseconds timeout) noexcept final {
+ timeout_ = timeout;
+ }
+
+ private:
+ std::shared_ptr<BazelNetwork> network_;
+ std::shared_ptr<LocalTreeMap> tree_map_;
+ bazel_re::Digest const root_digest_;
+ std::vector<std::string> const cmdline_;
+ std::vector<std::string> output_files_;
+ std::vector<std::string> output_dirs_;
+ std::vector<bazel_re::Command_EnvironmentVariable> env_vars_;
+ std::vector<bazel_re::Platform_Property> properties_;
+ CacheFlag cache_flag_{CacheFlag::CacheOutput};
+ std::chrono::milliseconds timeout_{kDefaultTimeout};
+
+ BazelAction(std::shared_ptr<BazelNetwork> network,
+ std::shared_ptr<LocalTreeMap> tree_map,
+ bazel_re::Digest root_digest,
+ std::vector<std::string> command,
+ std::vector<std::string> output_files,
+ std::vector<std::string> output_dirs,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) noexcept;
+
+ [[nodiscard]] auto CreateBundlesForAction(BlobContainer* blobs,
+ bazel_re::Digest const& exec_dir,
+ bool do_not_cache) const noexcept
+ -> bazel_re::Digest;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_ACTION_HPP
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.cpp b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
new file mode 100644
index 00000000..990d6067
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.cpp
@@ -0,0 +1,177 @@
+#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_msg_factory.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_action.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_response.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+BazelApi::BazelApi(std::string const& instance_name,
+ std::string const& host,
+ Port port,
+ ExecutionConfiguration const& exec_config) noexcept {
+ tree_map_ = std::make_shared<LocalTreeMap>();
+ network_ = std::make_shared<BazelNetwork>(
+ instance_name, host, port, exec_config, tree_map_);
+}
+
+// implement move constructor in cpp, where all members are complete types
+BazelApi::BazelApi(BazelApi&& other) noexcept = default;
+
+// implement destructor in cpp, where all members are complete types
+BazelApi::~BazelApi() = default;
+
+auto BazelApi::CreateAction(
+ ArtifactDigest const& root_digest,
+ std::vector<std::string> const& command,
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) noexcept
+ -> IExecutionAction::Ptr {
+ return std::unique_ptr<BazelAction>{new BazelAction{network_,
+ tree_map_,
+ root_digest,
+ command,
+ output_files,
+ output_dirs,
+ env_vars,
+ properties}};
+}
+
+[[nodiscard]] auto BazelApi::RetrieveToPaths(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<std::filesystem::path> const& output_paths) noexcept -> bool {
+ if (artifacts_info.size() != output_paths.size()) {
+ Logger::Log(LogLevel::Error,
+ "different number of digests and output paths.");
+ return false;
+ }
+
+ // Obtain file digests from artifact infos
+ std::vector<bazel_re::Digest> file_digests{};
+ for (std::size_t i{}; i < artifacts_info.size(); ++i) {
+ auto const& info = artifacts_info[i];
+ if (IsTreeObject(info.type)) {
+ // read object infos from sub tree and call retrieve recursively
+ auto const infos =
+ network_->ReadTreeInfos(info.digest, output_paths[i]);
+ if (not infos or not RetrieveToPaths(infos->second, infos->first)) {
+ return false;
+ }
+ }
+ else {
+ file_digests.emplace_back(info.digest);
+ }
+ }
+
+ // Request file blobs
+ auto size = file_digests.size();
+ auto reader = network_->ReadBlobs(std::move(file_digests));
+ auto blobs = reader.Next();
+ std::size_t count{};
+ while (not blobs.empty()) {
+ if (count + blobs.size() > size) {
+ Logger::Log(LogLevel::Error, "received more blobs than requested.");
+ return false;
+ }
+ for (std::size_t pos = 0; pos < blobs.size(); ++pos) {
+ auto gpos = count + pos;
+ auto const& type = artifacts_info[gpos].type;
+ if (not FileSystemManager::WriteFileAs(
+ blobs[pos].data, output_paths[gpos], type)) {
+ return false;
+ }
+ }
+ count += blobs.size();
+ blobs = reader.Next();
+ }
+
+ if (count != size) {
+ Logger::Log(LogLevel::Error, "could not retrieve all requested blobs.");
+ return false;
+ }
+
+ return true;
+}
+
+[[nodiscard]] auto BazelApi::RetrieveToFds(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<int> const& fds) noexcept -> bool {
+ if (artifacts_info.size() != fds.size()) {
+ Logger::Log(LogLevel::Error,
+ "different number of digests and file descriptors.");
+ return false;
+ }
+
+ for (std::size_t i{}; i < artifacts_info.size(); ++i) {
+ auto fd = fds[i];
+ auto const& info = artifacts_info[i];
+
+ if (gsl::owner<FILE*> out = fdopen(fd, "wb")) { // NOLINT
+ auto const success = network_->DumpToStream(info, out);
+ std::fclose(out);
+ if (not success) {
+ Logger::Log(LogLevel::Error,
+ "dumping {} {} to file descriptor {} failed.",
+ IsTreeObject(info.type) ? "tree" : "blob",
+ info.ToString(),
+ fd);
+ return false;
+ }
+ }
+ else {
+ Logger::Log(
+ LogLevel::Error, "opening file descriptor {} failed.", fd);
+ return false;
+ }
+ }
+ return true;
+}
+
+[[nodiscard]] auto BazelApi::Upload(BlobContainer const& blobs,
+ bool skip_find_missing) noexcept -> bool {
+ return network_->UploadBlobs(blobs, skip_find_missing);
+}
+
+[[nodiscard]] auto BazelApi::UploadTree(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const&
+ artifacts) noexcept -> std::optional<ArtifactDigest> {
+ BlobContainer blobs{};
+ auto tree = tree_map_->CreateTree();
+ auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree(
+ artifacts,
+ [&blobs](BazelBlob&& blob) { blobs.Emplace(std::move(blob)); },
+ [&tree](auto path, auto info) { return tree.AddInfo(path, info); });
+ if (not digest) {
+ Logger::Log(LogLevel::Debug, "failed to create digest for tree.");
+ return std::nullopt;
+ }
+ if (not Upload(blobs, /*skip_find_missing=*/false)) {
+ Logger::Log(LogLevel::Debug, "failed to upload blobs for tree.");
+ return std::nullopt;
+ }
+ if (tree_map_->AddTree(*digest, std::move(tree))) {
+ return ArtifactDigest{std::move(*digest)};
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] auto BazelApi::IsAvailable(
+ ArtifactDigest const& digest) const noexcept -> bool {
+ return network_->IsAvailable(digest);
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_api.hpp b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
new file mode 100644
index 00000000..1405737b
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_api.hpp
@@ -0,0 +1,65 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_API_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_API_HPP
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/common/local_tree_map.hpp"
+
+// forward declaration for actual implementations
+class BazelNetwork;
+struct ExecutionConfiguration;
+
+/// \brief Bazel implementation of the abstract Execution API.
+class BazelApi final : public IExecutionApi {
+ public:
+ BazelApi(std::string const& instance_name,
+ std::string const& host,
+ Port port,
+ ExecutionConfiguration const& exec_config) noexcept;
+ BazelApi(BazelApi const&) = delete;
+ BazelApi(BazelApi&& other) noexcept;
+ auto operator=(BazelApi const&) -> BazelApi& = delete;
+ auto operator=(BazelApi &&) -> BazelApi& = delete;
+ ~BazelApi() final;
+
+ auto CreateAction(
+ ArtifactDigest const& root_digest,
+ std::vector<std::string> const& command,
+ std::vector<std::string> const& output_files,
+ std::vector<std::string> const& output_dirs,
+ std::map<std::string, std::string> const& env_vars,
+ std::map<std::string, std::string> const& properties) noexcept
+ -> IExecutionAction::Ptr final;
+
+ [[nodiscard]] auto RetrieveToPaths(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<std::filesystem::path> const& output_paths) noexcept
+ -> bool final;
+
+ [[nodiscard]] auto RetrieveToFds(
+ std::vector<Artifact::ObjectInfo> const& artifacts_info,
+ std::vector<int> const& fds) noexcept -> bool final;
+
+ [[nodiscard]] auto Upload(BlobContainer const& blobs,
+ bool skip_find_missing) noexcept -> bool final;
+
+ [[nodiscard]] auto UploadTree(
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const&
+ artifacts) noexcept -> std::optional<ArtifactDigest> final;
+
+ [[nodiscard]] auto IsAvailable(ArtifactDigest const& digest) const noexcept
+ -> bool final;
+
+ private:
+ std::shared_ptr<BazelNetwork> network_;
+ std::shared_ptr<LocalTreeMap> tree_map_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_API_HPP
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp
new file mode 100644
index 00000000..87ceb4ae
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.cpp
@@ -0,0 +1,354 @@
+#include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp"
+
+#include "grpcpp/grpcpp.h"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/common/execution_common.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp"
+
+namespace {
+
+[[nodiscard]] auto ToResourceName(std::string const& instance_name,
+ bazel_re::Digest const& digest) noexcept
+ -> std::string {
+ return fmt::format(
+ "{}/blobs/{}/{}", instance_name, digest.hash(), digest.size_bytes());
+}
+
+} // namespace
+
+BazelCasClient::BazelCasClient(std::string const& server,
+ Port port,
+ std::string const& user,
+ std::string const& pwd) noexcept
+ : stream_{std::make_unique<ByteStreamClient>(server, port, user, pwd)} {
+ stub_ = bazel_re::ContentAddressableStorage::NewStub(
+ CreateChannelWithCredentials(server, port, user, pwd));
+}
+
+auto BazelCasClient::FindMissingBlobs(
+ std::string const& instance_name,
+ std::vector<bazel_re::Digest> const& digests) noexcept
+ -> std::vector<bazel_re::Digest> {
+ return FindMissingBlobs(instance_name, digests.begin(), digests.end());
+}
+
+auto BazelCasClient::FindMissingBlobs(
+ std::string const& instance_name,
+ BlobContainer::DigestList const& digests) noexcept
+ -> std::vector<bazel_re::Digest> {
+ return FindMissingBlobs(instance_name, digests.begin(), digests.end());
+}
+
+auto BazelCasClient::BatchUpdateBlobs(
+ std::string const& instance_name,
+ std::vector<BazelBlob>::const_iterator const& begin,
+ std::vector<BazelBlob>::const_iterator const& end) noexcept
+ -> std::vector<bazel_re::Digest> {
+ return DoBatchUpdateBlobs(instance_name, begin, end);
+}
+
+auto BazelCasClient::BatchUpdateBlobs(
+ std::string const& instance_name,
+ BlobContainer::iterator const& begin,
+ BlobContainer::iterator const& end) noexcept
+ -> std::vector<bazel_re::Digest> {
+ return DoBatchUpdateBlobs(instance_name, begin, end);
+}
+
+auto BazelCasClient::BatchUpdateBlobs(
+ std::string const& instance_name,
+ BlobContainer::RelatedBlobList::iterator const& begin,
+ BlobContainer::RelatedBlobList::iterator const& end) noexcept
+ -> std::vector<bazel_re::Digest> {
+ return DoBatchUpdateBlobs(instance_name, begin, end);
+}
+
+auto BazelCasClient::BatchReadBlobs(
+ std::string const& instance_name,
+ std::vector<bazel_re::Digest>::const_iterator const& begin,
+ std::vector<bazel_re::Digest>::const_iterator const& end) noexcept
+ -> std::vector<BazelBlob> {
+ auto request =
+ CreateRequest<bazel_re::BatchReadBlobsRequest, bazel_re::Digest>(
+ instance_name, begin, end);
+ grpc::ClientContext context;
+ bazel_re::BatchReadBlobsResponse response;
+ grpc::Status status = stub_->BatchReadBlobs(&context, request, &response);
+
+ std::vector<BazelBlob> result{};
+ if (status.ok()) {
+ result =
+ ProcessBatchResponse<BazelBlob,
+ bazel_re::BatchReadBlobsResponse_Response>(
+ response,
+ [](std::vector<BazelBlob>* v,
+ bazel_re::BatchReadBlobsResponse_Response const& r) {
+ v->emplace_back(r.digest(), r.data());
+ });
+ }
+ else {
+ LogStatus(&logger_, LogLevel::Debug, status);
+ }
+
+ return result;
+}
+
+auto BazelCasClient::GetTree(std::string const& instance_name,
+ bazel_re::Digest const& root_digest,
+ std::int32_t page_size,
+ std::string const& page_token) noexcept
+ -> std::vector<bazel_re::Directory> {
+ auto request =
+ CreateGetTreeRequest(instance_name, root_digest, page_size, page_token);
+
+ grpc::ClientContext context;
+ bazel_re::GetTreeResponse response;
+ auto stream = stub_->GetTree(&context, request);
+
+ std::vector<bazel_re::Directory> result;
+ while (stream->Read(&response)) {
+ result = ProcessResponseContents<bazel_re::Directory>(response);
+ auto const& next_page_token = response.next_page_token();
+ if (!next_page_token.empty()) {
+ // recursively call this function with token for next page
+ auto next_result =
+ GetTree(instance_name, root_digest, page_size, next_page_token);
+ std::move(next_result.begin(),
+ next_result.end(),
+ std::back_inserter(result));
+ }
+ }
+
+ auto status = stream->Finish();
+ if (not status.ok()) {
+ LogStatus(&logger_, LogLevel::Debug, status);
+ }
+
+ return result;
+}
+
+auto BazelCasClient::UpdateSingleBlob(std::string const& instance_name,
+ BazelBlob const& blob) noexcept -> bool {
+ thread_local static std::string uuid{};
+ if (uuid.empty()) {
+ auto id = CreateProcessUniqueId();
+ if (not id) {
+ logger_.Emit(LogLevel::Debug, "Failed creating process unique id.");
+ return false;
+ }
+ uuid = CreateUUIDVersion4(*id);
+ }
+ return stream_->Write(fmt::format("{}/uploads/{}/blobs/{}/{}",
+ instance_name,
+ uuid,
+ blob.digest.hash(),
+ blob.digest.size_bytes()),
+ blob.data);
+}
+
+auto BazelCasClient::IncrementalReadSingleBlob(
+ std::string const& instance_name,
+ bazel_re::Digest const& digest) noexcept
+ -> ByteStreamClient::IncrementalReader {
+ return stream_->IncrementalRead(ToResourceName(instance_name, digest));
+}
+
+auto BazelCasClient::ReadSingleBlob(std::string const& instance_name,
+ bazel_re::Digest const& digest) noexcept
+ -> std::optional<BazelBlob> {
+ if (auto data = stream_->Read(ToResourceName(instance_name, digest))) {
+ return BazelBlob{ArtifactDigest::Create(*data), std::move(*data)};
+ }
+ return std::nullopt;
+}
+
+template <class T_OutputIter>
+auto BazelCasClient::FindMissingBlobs(std::string const& instance_name,
+ T_OutputIter const& start,
+ T_OutputIter const& end) noexcept
+ -> std::vector<bazel_re::Digest> {
+ auto request =
+ CreateRequest<bazel_re::FindMissingBlobsRequest, bazel_re::Digest>(
+ instance_name, start, end);
+
+ grpc::ClientContext context;
+ bazel_re::FindMissingBlobsResponse response;
+ grpc::Status status = stub_->FindMissingBlobs(&context, request, &response);
+
+ std::vector<bazel_re::Digest> result{};
+ if (status.ok()) {
+ result = ProcessResponseContents<bazel_re::Digest>(response);
+ }
+ else {
+ LogStatus(&logger_, LogLevel::Debug, status);
+ }
+
+ return result;
+}
+
+template <class T_OutputIter>
+auto BazelCasClient::DoBatchUpdateBlobs(std::string const& instance_name,
+ T_OutputIter const& start,
+ T_OutputIter const& end) noexcept
+ -> std::vector<bazel_re::Digest> {
+ auto request = CreateUpdateBlobsRequest(instance_name, start, end);
+
+ grpc::ClientContext context;
+ bazel_re::BatchUpdateBlobsResponse response;
+ grpc::Status status = stub_->BatchUpdateBlobs(&context, request, &response);
+
+ std::vector<bazel_re::Digest> result{};
+ if (status.ok()) {
+ result =
+ ProcessBatchResponse<bazel_re::Digest,
+ bazel_re::BatchUpdateBlobsResponse_Response>(
+ response,
+ [](std::vector<bazel_re::Digest>* v,
+ bazel_re::BatchUpdateBlobsResponse_Response const& r) {
+ v->push_back(r.digest());
+ });
+ }
+ else {
+ LogStatus(&logger_, LogLevel::Debug, status);
+ if (status.error_code() == grpc::StatusCode::RESOURCE_EXHAUSTED) {
+ logger_.Emit(LogLevel::Debug,
+ "Falling back to single blob transfers");
+ auto current = start;
+ while (current != end) {
+ if (UpdateSingleBlob(instance_name, (*current))) {
+ result.emplace_back((*current).digest);
+ }
+ ++current;
+ }
+ }
+ }
+
+ return result;
+}
+
+namespace detail {
+
+// Getter for request contents (needs specialization, never implemented)
+template <class T_Content, class T_Request>
+static auto GetRequestContents(T_Request&) noexcept
+ -> pb::RepeatedPtrField<T_Content>*;
+
+// Getter for response contents (needs specialization, never implemented)
+template <class T_Content, class T_Response>
+static auto GetResponseContents(T_Response const&) noexcept
+ -> pb::RepeatedPtrField<T_Content> const&;
+
+// Specialization of GetRequestContents for 'FindMissingBlobsRequest'
+template <>
+auto GetRequestContents<bazel_re::Digest, bazel_re::FindMissingBlobsRequest>(
+ bazel_re::FindMissingBlobsRequest& request) noexcept
+ -> pb::RepeatedPtrField<bazel_re::Digest>* {
+ return request.mutable_blob_digests();
+}
+
+// Specialization of GetRequestContents for 'BatchReadBlobsRequest'
+template <>
+auto GetRequestContents<bazel_re::Digest, bazel_re::BatchReadBlobsRequest>(
+ bazel_re::BatchReadBlobsRequest& request) noexcept
+ -> pb::RepeatedPtrField<bazel_re::Digest>* {
+ return request.mutable_digests();
+}
+
+// Specialization of GetResponseContents for 'FindMissingBlobsResponse'
+template <>
+auto GetResponseContents<bazel_re::Digest, bazel_re::FindMissingBlobsResponse>(
+ bazel_re::FindMissingBlobsResponse const& response) noexcept
+ -> pb::RepeatedPtrField<bazel_re::Digest> const& {
+ return response.missing_blob_digests();
+}
+
+// Specialization of GetResponseContents for 'GetTreeResponse'
+template <>
+auto GetResponseContents<bazel_re::Directory, bazel_re::GetTreeResponse>(
+ bazel_re::GetTreeResponse const& response) noexcept
+ -> pb::RepeatedPtrField<bazel_re::Directory> const& {
+ return response.directories();
+}
+
+} // namespace detail
+
+template <class T_Request, class T_Content, class T_OutputIter>
+auto BazelCasClient::CreateRequest(std::string const& instance_name,
+ T_OutputIter const& start,
+ T_OutputIter const& end) const noexcept
+ -> T_Request {
+ T_Request request;
+ request.set_instance_name(instance_name);
+ std::copy(
+ start,
+ end,
+ pb::back_inserter(detail::GetRequestContents<T_Content>(request)));
+ return request;
+}
+
+template <class T_OutputIter>
+auto BazelCasClient::CreateUpdateBlobsRequest(std::string const& instance_name,
+ T_OutputIter const& start,
+ T_OutputIter const& end)
+ const noexcept -> bazel_re::BatchUpdateBlobsRequest {
+ bazel_re::BatchUpdateBlobsRequest request;
+ request.set_instance_name(instance_name);
+ std::transform(start,
+ end,
+ pb::back_inserter(request.mutable_requests()),
+ [](BazelBlob const& b) {
+ return BazelCasClient::CreateUpdateBlobsSingleRequest(b);
+ });
+ return request;
+}
+
+auto BazelCasClient::CreateUpdateBlobsSingleRequest(BazelBlob const& b) noexcept
+ -> bazel_re::BatchUpdateBlobsRequest_Request {
+ bazel_re::BatchUpdateBlobsRequest_Request r{};
+ r.set_allocated_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{b.digest}});
+ r.set_data(b.data);
+ return r;
+}
+
+auto BazelCasClient::CreateGetTreeRequest(
+ std::string const& instance_name,
+ bazel_re::Digest const& root_digest,
+ int page_size,
+ std::string const& page_token) noexcept -> bazel_re::GetTreeRequest {
+ bazel_re::GetTreeRequest request;
+ request.set_instance_name(instance_name);
+ request.set_allocated_root_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest{root_digest}});
+ request.set_page_size(page_size);
+ request.set_page_token(page_token);
+ return request;
+}
+
+template <class T_Content, class T_Inner, class T_Response>
+auto BazelCasClient::ProcessBatchResponse(
+ T_Response const& response,
+ std::function<void(std::vector<T_Content>*, T_Inner const&)> const&
+ inserter) const noexcept -> std::vector<T_Content> {
+ std::vector<T_Content> output;
+ for (auto const& res : response.responses()) {
+ auto const& res_status = res.status();
+ if (res_status.code() == static_cast<int>(grpc::StatusCode::OK)) {
+ inserter(&output, res);
+ }
+ else {
+ LogStatus(&logger_, LogLevel::Debug, res_status);
+ }
+ }
+ return output;
+}
+
+template <class T_Content, class T_Response>
+auto BazelCasClient::ProcessResponseContents(
+ T_Response const& response) const noexcept -> std::vector<T_Content> {
+ std::vector<T_Content> output;
+ auto const& contents = detail::GetResponseContents<T_Content>(response);
+ std::copy(contents.begin(), contents.end(), std::back_inserter(output));
+ return output;
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp
new file mode 100644
index 00000000..f3d6daff
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp
@@ -0,0 +1,169 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAS_CLIENT_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAS_CLIENT_HPP
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bytestream_client.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// Implements client side for serivce defined here:
+/// https://github.com/bazelbuild/bazel/blob/4b6ad34dbba15dacebfb6cbf76fa741649cdb007/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto#L243
+class BazelCasClient {
+ public:
+ BazelCasClient(std::string const& server,
+ Port port,
+ std::string const& user = "",
+ std::string const& pwd = "") noexcept;
+
+ /// \brief Find missing blobs
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] digests The blob digests to search for
+ /// \returns The digests of blobs not found in CAS
+ [[nodiscard]] auto FindMissingBlobs(
+ std::string const& instance_name,
+ std::vector<bazel_re::Digest> const& digests) noexcept
+ -> std::vector<bazel_re::Digest>;
+
+ /// \brief Find missing blobs
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] digests The blob digests to search for
+ /// \returns The digests of blobs not found in CAS
+ [[nodiscard]] auto FindMissingBlobs(
+ std::string const& instance_name,
+ BlobContainer::DigestList const& digests) noexcept
+ -> std::vector<bazel_re::Digest>;
+
+ /// \brief Upload multiple blobs in batch transfer
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] begin Start of the blobs to upload
+ /// \param[in] end End of the blobs to upload
+ /// \returns The digests of blobs sucessfully updated
+ [[nodiscard]] auto BatchUpdateBlobs(
+ std::string const& instance_name,
+ std::vector<BazelBlob>::const_iterator const& begin,
+ std::vector<BazelBlob>::const_iterator const& end) noexcept
+ -> std::vector<bazel_re::Digest>;
+
+ /// \brief Upload multiple blobs in batch transfer
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] begin Start of the blobs to upload
+ /// \param[in] end End of the blobs to upload
+ /// \returns The digests of blobs sucessfully updated
+ [[nodiscard]] auto BatchUpdateBlobs(
+ std::string const& instance_name,
+ BlobContainer::iterator const& begin,
+ BlobContainer::iterator const& end) noexcept
+ -> std::vector<bazel_re::Digest>;
+
+ /// \brief Upload multiple blobs in batch transfer
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] begin Start of the blobs to upload
+ /// \param[in] end End of the blobs to upload
+ /// \returns The digests of blobs sucessfully updated
+ [[nodiscard]] auto BatchUpdateBlobs(
+ std::string const& instance_name,
+ BlobContainer::RelatedBlobList::iterator const& begin,
+ BlobContainer::RelatedBlobList::iterator const& end) noexcept
+ -> std::vector<bazel_re::Digest>;
+
+ /// \brief Read multiple blobs in batch transfer
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] begin Start of the blob digests to read
+ /// \param[in] end End of the blob digests to read
+ /// \returns The blobs sucessfully read
+ [[nodiscard]] auto BatchReadBlobs(
+ std::string const& instance_name,
+ std::vector<bazel_re::Digest>::const_iterator const& begin,
+ std::vector<bazel_re::Digest>::const_iterator const& end) noexcept
+ -> std::vector<BazelBlob>;
+
+ [[nodiscard]] auto GetTree(std::string const& instance_name,
+ bazel_re::Digest const& root_digest,
+ std::int32_t page_size,
+ std::string const& page_token = "") noexcept
+ -> std::vector<bazel_re::Directory>;
+
+ /// \brief Upload single blob via bytestream
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] blob The blob to upload
+ /// \returns Boolean indicating successful upload
+ [[nodiscard]] auto UpdateSingleBlob(std::string const& instance_name,
+ BazelBlob const& blob) noexcept -> bool;
+
+ /// \brief Read single blob via incremental bytestream reader
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] digest Blob digest to read
+ /// \returns Incremental bytestream reader.
+ [[nodiscard]] auto IncrementalReadSingleBlob(
+ std::string const& instance_name,
+ bazel_re::Digest const& digest) noexcept
+ -> ByteStreamClient::IncrementalReader;
+
+ /// \brief Read single blob via bytestream
+ /// \param[in] instance_name Name of the CAS instance
+ /// \param[in] digest Blob digest to read
+ /// \returns The blob successfully read
+ [[nodiscard]] auto ReadSingleBlob(std::string const& instance_name,
+ bazel_re::Digest const& digest) noexcept
+ -> std::optional<BazelBlob>;
+
+ private:
+ std::unique_ptr<ByteStreamClient> stream_{};
+ std::unique_ptr<bazel_re::ContentAddressableStorage::Stub> stub_;
+ Logger logger_{"BazelCasClient"};
+
+ template <class T_OutputIter>
+ [[nodiscard]] auto FindMissingBlobs(std::string const& instance_name,
+ T_OutputIter const& start,
+ T_OutputIter const& end) noexcept
+ -> std::vector<bazel_re::Digest>;
+
+ template <class T_OutputIter>
+ [[nodiscard]] auto DoBatchUpdateBlobs(std::string const& instance_name,
+ T_OutputIter const& start,
+ T_OutputIter const& end) noexcept
+ -> std::vector<bazel_re::Digest>;
+
+ template <class T_Request, class T_Content, class T_OutputIter>
+ [[nodiscard]] auto CreateRequest(std::string const& instance_name,
+ T_OutputIter const& start,
+ T_OutputIter const& end) const noexcept
+ -> T_Request;
+
+ template <class T_OutputIter>
+ [[nodiscard]] auto CreateUpdateBlobsRequest(
+ std::string const& instance_name,
+ T_OutputIter const& start,
+ T_OutputIter const& end) const noexcept
+ -> bazel_re::BatchUpdateBlobsRequest;
+
+ [[nodiscard]] static auto CreateUpdateBlobsSingleRequest(
+ BazelBlob const& b) noexcept
+ -> bazel_re::BatchUpdateBlobsRequest_Request;
+
+ [[nodiscard]] static auto CreateGetTreeRequest(
+ std::string const& instance_name,
+ bazel_re::Digest const& root_digest,
+ int page_size,
+ std::string const& page_token) noexcept -> bazel_re::GetTreeRequest;
+
+ template <class T_Content, class T_Inner, class T_Response>
+ auto ProcessBatchResponse(
+ T_Response const& response,
+ std::function<void(std::vector<T_Content>*, T_Inner const&)> const&
+ inserter) const noexcept -> std::vector<T_Content>;
+
+ template <class T_Content, class T_Response>
+ auto ProcessResponseContents(T_Response const& response) const noexcept
+ -> std::vector<T_Content>;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CAS_CLIENT_HPP
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp b/src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp
new file mode 100644
index 00000000..6e37fa28
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp
@@ -0,0 +1,54 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CLIENT_COMMON_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CLIENT_COMMON_HPP
+
+/// \file bazel_client_common.hpp
+/// \brief Common types and functions required by client implementations.
+
+#include <sstream>
+#include <string>
+
+#include "grpcpp/grpcpp.h"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+[[maybe_unused]] [[nodiscard]] static inline auto CreateChannelWithCredentials(
+ std::string const& server,
+ Port port,
+ std::string const& user = "",
+ [[maybe_unused]] std::string const& pwd = "") noexcept {
+ std::shared_ptr<grpc::ChannelCredentials> cred;
+ std::string address = server + ':' + std::to_string(port);
+ if (user.empty()) {
+ cred = grpc::InsecureChannelCredentials();
+ }
+ else {
+ // TODO(oreiche): set up authentication credentials
+ }
+ return grpc::CreateChannel(address, cred);
+}
+
+[[maybe_unused]] static inline void LogStatus(Logger const* logger,
+ LogLevel level,
+ grpc::Status const& s) noexcept {
+ if (logger == nullptr) {
+ Logger::Log(level, "{}: {}", s.error_code(), s.error_message());
+ }
+ else {
+ logger->Emit(level, "{}: {}", s.error_code(), s.error_message());
+ }
+}
+
+[[maybe_unused]] static inline void LogStatus(
+ Logger const* logger,
+ LogLevel level,
+ google::rpc::Status const& s) noexcept {
+ if (logger == nullptr) {
+ Logger::Log(level, "{}: {}", s.code(), s.message());
+ }
+ else {
+ logger->Emit(level, "{}: {}", s.code(), s.message());
+ }
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_CLIENT_COMMON_HPP
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_execution_client.cpp b/src/buildtool/execution_api/remote/bazel/bazel_execution_client.cpp
new file mode 100644
index 00000000..3b95a8ff
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_execution_client.cpp
@@ -0,0 +1,129 @@
+#include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp"
+
+#include "grpcpp/grpcpp.h"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp"
+
+namespace bazel_re = build::bazel::remote::execution::v2;
+
+BazelExecutionClient::BazelExecutionClient(std::string const& server,
+ Port port,
+ std::string const& user,
+ std::string const& pwd) noexcept {
+ stub_ = bazel_re::Execution::NewStub(
+ CreateChannelWithCredentials(server, port, user, pwd));
+}
+
+auto BazelExecutionClient::Execute(std::string const& instance_name,
+ bazel_re::Digest const& action_digest,
+ ExecutionConfiguration const& config,
+ bool wait)
+ -> BazelExecutionClient::ExecutionResponse {
+ auto execution_policy = std::make_unique<bazel_re::ExecutionPolicy>();
+ execution_policy->set_priority(config.execution_priority);
+
+ auto results_cache_policy =
+ std::make_unique<bazel_re::ResultsCachePolicy>();
+ results_cache_policy->set_priority(config.results_cache_priority);
+
+ bazel_re::ExecuteRequest request;
+ request.set_instance_name(instance_name);
+ request.set_skip_cache_lookup(config.skip_cache_lookup);
+ request.set_allocated_action_digest(
+ gsl::owner<bazel_re::Digest*>{new bazel_re::Digest(action_digest)});
+ request.set_allocated_execution_policy(execution_policy.release());
+ request.set_allocated_results_cache_policy(results_cache_policy.release());
+
+ grpc::ClientContext context;
+ std::unique_ptr<grpc::ClientReader<google::longrunning::Operation>> reader(
+ stub_->Execute(&context, request));
+
+ return ExtractContents(ReadExecution(reader.get(), wait));
+}
+
+auto BazelExecutionClient::WaitExecution(std::string const& execution_handle)
+ -> BazelExecutionClient::ExecutionResponse {
+ bazel_re::WaitExecutionRequest request;
+ request.set_name(execution_handle);
+
+ grpc::ClientContext context;
+ std::unique_ptr<grpc::ClientReader<google::longrunning::Operation>> reader(
+ stub_->WaitExecution(&context, request));
+
+ return ExtractContents(ReadExecution(reader.get(), true));
+}
+
+auto BazelExecutionClient::ReadExecution(
+ grpc::ClientReader<google::longrunning::Operation>* reader,
+ bool wait) -> std::optional<google::longrunning::Operation> {
+ if (reader == nullptr) {
+ // TODO(vmoreno): log error
+ return std::nullopt;
+ }
+
+ google::longrunning::Operation operation;
+ if (not reader->Read(&operation)) {
+ grpc::Status status = reader->Finish();
+ // TODO(vmoreno): log error using data in status and operation
+ LogStatus(&logger_, LogLevel::Debug, status);
+ return std::nullopt;
+ }
+ // Important note: do not call reader->Finish() unless reader->Read()
+ // returned false, otherwise the thread will be never released
+ if (wait) {
+ while (reader->Read(&operation)) {
+ }
+ grpc::Status status = reader->Finish();
+ if (not status.ok()) {
+ // TODO(vmoreno): log error from status and operation
+ LogStatus(&logger_, LogLevel::Debug, status);
+ return std::nullopt;
+ }
+ }
+ return operation;
+}
+
+auto BazelExecutionClient::ExtractContents(
+ std::optional<google::longrunning::Operation>&& operation)
+ -> BazelExecutionClient::ExecutionResponse {
+ if (not operation) {
+ // Error was already logged in ReadExecution()
+ return ExecutionResponse::MakeEmptyFailed();
+ }
+ auto op = *operation;
+ ExecutionResponse response;
+ response.execution_handle = op.name();
+ if (not op.done()) {
+ response.state = ExecutionResponse::State::Ongoing;
+ return response;
+ }
+ if (op.has_error()) {
+ // TODO(vmoreno): log error from google::rpc::Status s = op.error()
+ LogStatus(&logger_, LogLevel::Debug, op.error());
+ response.state = ExecutionResponse::State::Failed;
+ return response;
+ }
+
+ // Get execution response Unpacked from Protobufs Any type to the actual
+ // type in our case
+ bazel_re::ExecuteResponse exec_response;
+ auto const& raw_response = op.response();
+ if (not raw_response.Is<bazel_re::ExecuteResponse>()) {
+ // Fatal error, the type should be correct
+ response.state = ExecutionResponse::State::Failed;
+ return response;
+ }
+
+ response.state = ExecutionResponse::State::Finished;
+
+ raw_response.UnpackTo(&exec_response);
+
+ ExecutionOutput output;
+ output.action_result = exec_response.result();
+ output.cached_result = exec_response.cached_result();
+ output.message = exec_response.message();
+ output.server_logs = exec_response.server_logs();
+ response.output = output;
+
+ return response;
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp b/src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp
new file mode 100644
index 00000000..b213a94d
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp
@@ -0,0 +1,66 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_EXECUTION_CLIENT_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_EXECUTION_CLIENT_HPP
+
+#include <memory>
+#include <optional>
+#include <string>
+
+#include "build/bazel/remote/execution/v2/remote_execution.grpc.pb.h"
+#include "google/longrunning/operations.pb.h"
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// Implements client side for serivce defined here:
+/// https://github.com/bazelbuild/bazel/blob/4b6ad34dbba15dacebfb6cbf76fa741649cdb007/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto#L42
+class BazelExecutionClient {
+ public:
+ struct ExecutionOutput {
+ bazel_re::ActionResult action_result{};
+ bool cached_result{};
+ grpc::Status status{};
+ // TODO(vmoreno): switch to non-google type for the map
+ google::protobuf::Map<std::string, bazel_re::LogFile> server_logs{};
+ std::string message{};
+ };
+
+ struct ExecutionResponse {
+ enum class State { Failed, Ongoing, Finished, Unknown };
+
+ std::string execution_handle{};
+ State state{State::Unknown};
+ std::optional<ExecutionOutput> output{std::nullopt};
+
+ static auto MakeEmptyFailed() -> ExecutionResponse {
+ return ExecutionResponse{
+ {}, ExecutionResponse::State::Failed, std::nullopt};
+ }
+ };
+
+ BazelExecutionClient(std::string const& server,
+ Port port,
+ std::string const& user = "",
+ std::string const& pwd = "") noexcept;
+
+ [[nodiscard]] auto Execute(std::string const& instance_name,
+ bazel_re::Digest const& action_digest,
+ ExecutionConfiguration const& config,
+ bool wait) -> ExecutionResponse;
+
+ [[nodiscard]] auto WaitExecution(std::string const& execution_handle)
+ -> ExecutionResponse;
+
+ private:
+ std::unique_ptr<bazel_re::Execution::Stub> stub_;
+ Logger logger_{"BazelExecutionClient"};
+
+ [[nodiscard]] auto ReadExecution(
+ grpc::ClientReader<google::longrunning::Operation>* reader,
+ bool wait) -> std::optional<google::longrunning::Operation>;
+
+ [[nodiscard]] auto ExtractContents(
+ std::optional<google::longrunning::Operation>&& operation)
+ -> ExecutionResponse;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_EXECUTION_CLIENT_HPP
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.cpp b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp
new file mode 100644
index 00000000..ad160d0c
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_network.cpp
@@ -0,0 +1,327 @@
+#include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp"
+
+#include "src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_response.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+namespace {
+
+[[nodiscard]] auto ReadDirectory(
+ gsl::not_null<BazelNetwork const*> const& network,
+ bazel_re::Digest const& digest) noexcept
+ -> std::optional<bazel_re::Directory> {
+ auto blobs = network->ReadBlobs({digest}).Next();
+ if (blobs.size() == 1) {
+ return BazelMsgFactory::MessageFromString<bazel_re::Directory>(
+ blobs.at(0).data);
+ }
+ Logger::Log(
+ LogLevel::Error, "Directory {} not found in CAS", digest.hash());
+ return std::nullopt;
+}
+
+[[nodiscard]] auto TreeToStream(
+ gsl::not_null<BazelNetwork const*> const& network,
+ bazel_re::Digest const& tree_digest,
+ gsl::not_null<FILE*> const& stream) noexcept -> bool {
+ if (auto dir = ReadDirectory(network, tree_digest)) {
+ if (auto data = BazelMsgFactory::DirectoryToString(*dir)) {
+ std::fwrite(data->data(), 1, data->size(), stream);
+ return true;
+ }
+ }
+ return false;
+}
+
+[[nodiscard]] auto BlobToStream(
+ gsl::not_null<BazelNetwork const*> const& network,
+ bazel_re::Digest const& blob_digest,
+ gsl::not_null<FILE*> const& stream) noexcept -> bool {
+ auto reader = network->IncrementalReadSingleBlob(blob_digest);
+ auto data = reader.Next();
+ while (data and not data->empty()) {
+ std::fwrite(data->data(), 1, data->size(), stream);
+ data = reader.Next();
+ }
+ return data.has_value();
+}
+
+} // namespace
+
+BazelNetwork::BazelNetwork(std::string instance_name,
+ std::string const& host,
+ Port port,
+ ExecutionConfiguration const& exec_config,
+ std::shared_ptr<LocalTreeMap> tree_map) noexcept
+ : instance_name_{std::move(instance_name)},
+ exec_config_{exec_config},
+ cas_{std::make_unique<BazelCasClient>(host, port)},
+ ac_{std::make_unique<BazelAcClient>(host, port)},
+ exec_{std::make_unique<BazelExecutionClient>(host, port)},
+ tree_map_{std::move(tree_map)} {}
+
+auto BazelNetwork::IsAvailable(bazel_re::Digest const& digest) const noexcept
+ -> bool {
+ return cas_
+ ->FindMissingBlobs(instance_name_,
+ std::vector<bazel_re::Digest>{digest})
+ .empty();
+}
+
+template <class T_Iter>
+auto BazelNetwork::DoUploadBlobs(T_Iter const& first,
+ T_Iter const& last) noexcept -> bool {
+ auto num_blobs = gsl::narrow<std::size_t>(std::distance(first, last));
+
+ std::vector<bazel_re::Digest> digests{};
+ digests.reserve(num_blobs);
+
+ auto begin = first;
+ auto current = first;
+ std::size_t transfer_size{};
+ while (current != last) {
+ auto const& blob = *current;
+ transfer_size += blob.data.size();
+ if (transfer_size > kMaxBatchTransferSize) {
+ if (begin == current) {
+ if (cas_->UpdateSingleBlob(instance_name_, blob)) {
+ digests.emplace_back(blob.digest);
+ }
+ ++current;
+ }
+ else {
+ for (auto& digest :
+ cas_->BatchUpdateBlobs(instance_name_, begin, current)) {
+ digests.emplace_back(std::move(digest));
+ }
+ }
+ begin = current;
+ transfer_size = 0;
+ }
+ else {
+ ++current;
+ }
+ }
+ if (begin != current) {
+ for (auto& digest :
+ cas_->BatchUpdateBlobs(instance_name_, begin, current)) {
+ digests.emplace_back(std::move(digest));
+ }
+ }
+
+ if (digests.size() != num_blobs) {
+ Logger::Log(LogLevel::Warning, "Failed to update all blobs");
+ return false;
+ }
+
+ return true;
+}
+
+auto BazelNetwork::UploadBlobs(BlobContainer const& blobs,
+ bool skip_find_missing) noexcept -> bool {
+ if (skip_find_missing) {
+ return DoUploadBlobs(blobs.begin(), blobs.end());
+ }
+
+ // find digests of blobs missing in CAS
+ auto missing_digests =
+ cas_->FindMissingBlobs(instance_name_, blobs.Digests());
+
+ if (not missing_digests.empty()) {
+ // update missing blobs
+ auto missing_blobs = blobs.RelatedBlobs(missing_digests);
+ return DoUploadBlobs(missing_blobs.begin(), missing_blobs.end());
+ }
+ return true;
+}
+
+auto BazelNetwork::ExecuteBazelActionSync(
+ bazel_re::Digest const& action) noexcept
+ -> std::optional<BazelExecutionClient::ExecutionOutput> {
+ auto response =
+ exec_->Execute(instance_name_, action, exec_config_, true /*wait*/);
+
+ if (response.state !=
+ BazelExecutionClient::ExecutionResponse::State::Finished or
+ not response.output) {
+ // TODO(oreiche): logging
+ return std::nullopt;
+ }
+
+ return response.output;
+}
+
+auto BazelNetwork::BlobReader::Next() noexcept -> std::vector<BazelBlob> {
+ std::size_t size{};
+ std::vector<BazelBlob> blobs{};
+
+ while (current_ != ids_.end()) {
+ size += gsl::narrow<std::size_t>(current_->size_bytes());
+ if (size > kMaxBatchTransferSize) {
+ if (begin_ == current_) {
+ auto blob = cas_->ReadSingleBlob(instance_name_, *begin_);
+ if (blob) {
+ blobs.emplace_back(std::move(*blob));
+ }
+ ++current_;
+ }
+ else {
+ blobs = cas_->BatchReadBlobs(instance_name_, begin_, current_);
+ }
+ begin_ = current_;
+ break;
+ }
+ ++current_;
+ }
+
+ if (begin_ != current_) {
+ blobs = cas_->BatchReadBlobs(instance_name_, begin_, current_);
+ begin_ = current_;
+ }
+
+ return blobs;
+}
+
+auto BazelNetwork::ReadBlobs(std::vector<bazel_re::Digest> ids) const noexcept
+ -> BlobReader {
+ return BlobReader{instance_name_, cas_.get(), std::move(ids)};
+}
+
+auto BazelNetwork::IncrementalReadSingleBlob(bazel_re::Digest const& id)
+ const noexcept -> ByteStreamClient::IncrementalReader {
+ return cas_->IncrementalReadSingleBlob(instance_name_, id);
+}
+
+auto BazelNetwork::GetCachedActionResult(
+ bazel_re::Digest const& action,
+ std::vector<std::string> const& output_files) const noexcept
+ -> std::optional<bazel_re::ActionResult> {
+ return ac_->GetActionResult(
+ instance_name_, action, false, false, output_files);
+}
+
+auto BazelNetwork::ReadTreeInfos(bazel_re::Digest const& tree_digest,
+ std::filesystem::path const& parent,
+ bool request_remote_tree) const noexcept
+ -> std::optional<std::pair<std::vector<std::filesystem::path>,
+ std::vector<Artifact::ObjectInfo>>> {
+ std::optional<DirectoryMap> dir_map{std::nullopt};
+ if (request_remote_tree or not tree_map_) {
+ // Query full tree from remote CAS. Note that this is currently not
+ // supported by Buildbarn revision c3c06bbe2a.
+ auto dirs =
+ cas_->GetTree(instance_name_, tree_digest, kMaxBatchTransferSize);
+
+ // Convert to Directory map
+ dir_map = DirectoryMap{};
+ dir_map->reserve(dirs.size());
+ for (auto& dir : dirs) {
+ try {
+ dir_map->emplace(
+ ArtifactDigest::Create(dir.SerializeAsString()),
+ std::move(dir));
+ } catch (...) {
+ return std::nullopt;
+ }
+ }
+ }
+
+ std::vector<std::filesystem::path> paths{};
+ std::vector<Artifact::ObjectInfo> infos{};
+
+ auto store_info = [&paths, &infos](auto path, auto info) {
+ paths.emplace_back(path);
+ infos.emplace_back(info);
+ return true;
+ };
+
+ if (ReadObjectInfosRecursively(dir_map, store_info, parent, tree_digest)) {
+ return std::make_pair(std::move(paths), std::move(infos));
+ }
+
+ return std::nullopt;
+}
+
+auto BazelNetwork::ReadObjectInfosRecursively(
+ std::optional<DirectoryMap> const& dir_map,
+ BazelMsgFactory::InfoStoreFunc const& store_info,
+ std::filesystem::path const& parent,
+ bazel_re::Digest const& digest) const noexcept -> bool {
+ // read from in-memory tree map
+ if (tree_map_) {
+ auto const* tree = tree_map_->GetTree(digest);
+ if (tree != nullptr) {
+ for (auto const& [path, info] : *tree) {
+ try {
+ if (IsTreeObject(info->type)
+ ? not ReadObjectInfosRecursively(dir_map,
+ store_info,
+ parent / path,
+ info->digest)
+ : not store_info(parent / path, *info)) {
+ return false;
+ }
+ } catch (...) { // satisfy clang-tidy, store_info() could throw
+ return false;
+ }
+ }
+ return true;
+ }
+ Logger::Log(
+ LogLevel::Debug, "tree {} not found in tree map", digest.hash());
+ }
+
+ // read from in-memory Directory map and cache it in in-memory tree map
+ if (dir_map) {
+ if (dir_map->contains(digest)) {
+ auto tree = tree_map_ ? std::make_optional(tree_map_->CreateTree())
+ : std::nullopt;
+ return BazelMsgFactory::ReadObjectInfosFromDirectory(
+ dir_map->at(digest),
+ [this, &dir_map, &store_info, &parent, &tree](
+ auto path, auto info) {
+ return IsTreeObject(info.type)
+ ? (not tree or
+ tree->AddInfo(path, info)) and
+ ReadObjectInfosRecursively(
+ dir_map,
+ store_info,
+ parent / path,
+ info.digest)
+ : store_info(parent / path, info);
+ }) and
+ (not tree_map_ or
+ tree_map_->AddTree(digest, std::move(*tree)));
+ }
+ Logger::Log(
+ LogLevel::Debug, "tree {} not found in dir map", digest.hash());
+ }
+
+ // fallback read from CAS and cache it in in-memory tree map
+ if (auto dir = ReadDirectory(this, digest)) {
+ auto tree = tree_map_ ? std::make_optional(tree_map_->CreateTree())
+ : std::nullopt;
+ return BazelMsgFactory::ReadObjectInfosFromDirectory(
+ *dir,
+ [this, &dir_map, &store_info, &parent, &tree](auto path,
+ auto info) {
+ return IsTreeObject(info.type)
+ ? (not tree or tree->AddInfo(path, info)) and
+ ReadObjectInfosRecursively(
+ dir_map,
+ store_info,
+ parent / path,
+ info.digest)
+ : store_info(parent / path, info);
+ }) and
+ (not tree_map_ or tree_map_->AddTree(digest, std::move(*tree)));
+ }
+ return false;
+}
+
+auto BazelNetwork::DumpToStream(
+ Artifact::ObjectInfo const& info,
+ gsl::not_null<FILE*> const& stream) const noexcept -> bool {
+ return IsTreeObject(info.type) ? TreeToStream(this, info.digest, stream)
+ : BlobToStream(this, info.digest, stream);
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_network.hpp b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp
new file mode 100644
index 00000000..644af2b4
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_network.hpp
@@ -0,0 +1,118 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_NETWORK_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_NETWORK_HPP
+
+#include <memory>
+#include <optional>
+#include <unordered_map>
+
+#include "src/buildtool/common/bazel_types.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/common/local_tree_map.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_ac_client.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp"
+
+/// \brief Contains all network clients and is responsible for all network IO.
+class BazelNetwork {
+ public:
+ class BlobReader {
+ friend class BazelNetwork;
+
+ public:
+ // Obtain the next batch of blobs that can be transferred in a single
+ // request.
+ [[nodiscard]] auto Next() noexcept -> std::vector<BazelBlob>;
+
+ private:
+ std::string instance_name_;
+ gsl::not_null<BazelCasClient*> cas_;
+ std::vector<bazel_re::Digest> const ids_;
+ std::vector<bazel_re::Digest>::const_iterator begin_;
+ std::vector<bazel_re::Digest>::const_iterator current_;
+
+ BlobReader(std::string instance_name,
+ gsl::not_null<BazelCasClient*> cas,
+ std::vector<bazel_re::Digest> ids)
+ : instance_name_{std::move(instance_name)},
+ cas_{std::move(cas)},
+ ids_{std::move(ids)},
+ begin_{ids_.begin()},
+ current_{begin_} {};
+ };
+
+ BazelNetwork(std::string instance_name,
+ std::string const& host,
+ Port port,
+ ExecutionConfiguration const& exec_config,
+ std::shared_ptr<LocalTreeMap> tree_map = nullptr) noexcept;
+
+ /// \brief Check if digest exists in CAS
+ /// \param[in] digest The digest to look up
+ /// \returns True if digest exists in CAS, false otherwise
+ [[nodiscard]] auto IsAvailable(
+ bazel_re::Digest const& digest) const noexcept -> bool;
+
+ /// \brief Uploads blobs to CAS
+ /// \param blobs The blobs to upload
+ /// \param skip_find_missing Skip finding missing blobs, just upload all
+ /// \returns True if upload was successful, false otherwise
+ [[nodiscard]] auto UploadBlobs(BlobContainer const& blobs,
+ bool skip_find_missing = false) noexcept
+ -> bool;
+
+ [[nodiscard]] auto ExecuteBazelActionSync(
+ bazel_re::Digest const& action) noexcept
+ -> std::optional<BazelExecutionClient::ExecutionOutput>;
+
+ [[nodiscard]] auto ReadBlobs(
+ std::vector<bazel_re::Digest> ids) const noexcept -> BlobReader;
+
+ [[nodiscard]] auto IncrementalReadSingleBlob(bazel_re::Digest const& id)
+ const noexcept -> ByteStreamClient::IncrementalReader;
+
+ [[nodiscard]] auto GetCachedActionResult(
+ bazel_re::Digest const& action,
+ std::vector<std::string> const& output_files) const noexcept
+ -> std::optional<bazel_re::ActionResult>;
+
+ [[nodiscard]] auto ReadTreeInfos(
+ bazel_re::Digest const& tree_digest,
+ std::filesystem::path const& parent,
+ bool request_remote_tree = false) const noexcept
+ -> std::optional<std::pair<std::vector<std::filesystem::path>,
+ std::vector<Artifact::ObjectInfo>>>;
+
+ [[nodiscard]] auto DumpToStream(
+ Artifact::ObjectInfo const& info,
+ gsl::not_null<FILE*> const& stream) const noexcept -> bool;
+
+ private:
+ using DirectoryMap =
+ std::unordered_map<bazel_re::Digest, bazel_re::Directory>;
+
+ // Max size for batch transfers
+ static constexpr std::size_t kMaxBatchTransferSize = 3 * 1024 * 1024;
+ static_assert(kMaxBatchTransferSize < GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH,
+ "Max batch transfer size too large.");
+
+ std::string const instance_name_{};
+ ExecutionConfiguration exec_config_{};
+ std::unique_ptr<BazelCasClient> cas_{};
+ std::unique_ptr<BazelAcClient> ac_{};
+ std::unique_ptr<BazelExecutionClient> exec_{};
+ std::shared_ptr<LocalTreeMap> tree_map_{};
+
+ template <class T_Iter>
+ [[nodiscard]] auto DoUploadBlobs(T_Iter const& first,
+ T_Iter const& last) noexcept -> bool;
+
+ [[nodiscard]] auto ReadObjectInfosRecursively(
+ std::optional<DirectoryMap> const& dir_map,
+ BazelMsgFactory::InfoStoreFunc const& store_info,
+ std::filesystem::path const& parent,
+ bazel_re::Digest const& digest) const noexcept -> bool;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_NETWORK_HPP
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_response.cpp b/src/buildtool/execution_api/remote/bazel/bazel_response.cpp
new file mode 100644
index 00000000..8b75a767
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_response.cpp
@@ -0,0 +1,125 @@
+#include "src/buildtool/execution_api/remote/bazel/bazel_response.hpp"
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_cas_client.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+auto BazelResponse::ReadStringBlob(bazel_re::Digest const& id) noexcept
+ -> std::string {
+ auto blobs = network_->ReadBlobs({id}).Next();
+ if (blobs.empty()) {
+ // TODO(oreiche): logging
+ return std::string{};
+ }
+ return blobs[0].data;
+}
+
+auto BazelResponse::Artifacts() const noexcept -> ArtifactInfos {
+ ArtifactInfos artifacts{};
+ auto const& action_result = output_.action_result;
+ artifacts.reserve(
+ static_cast<std::size_t>(action_result.output_files().size()));
+
+ // collect files and store them
+ for (auto const& file : action_result.output_files()) {
+ try {
+ artifacts.emplace(file.path(),
+ Artifact::ObjectInfo{
+ ArtifactDigest{file.digest()},
+ file.is_executable() ? ObjectType::Executable
+ : ObjectType::File});
+ } catch (...) {
+ return {};
+ }
+ }
+
+ // obtain tree digests for output directories
+ std::vector<bazel_re::Digest> tree_digests{};
+ tree_digests.reserve(
+ gsl::narrow<std::size_t>(action_result.output_directories_size()));
+ std::transform(action_result.output_directories().begin(),
+ action_result.output_directories().end(),
+ std::back_inserter(tree_digests),
+ [](auto dir) { return dir.tree_digest(); });
+
+ // collect root digests from trees and store them
+ auto blob_reader = network_->ReadBlobs(tree_digests);
+ auto tree_blobs = blob_reader.Next();
+ std::size_t pos{};
+ while (not tree_blobs.empty()) {
+ for (auto const& tree_blob : tree_blobs) {
+ try {
+ auto tree = BazelMsgFactory::MessageFromString<bazel_re::Tree>(
+ tree_blob.data);
+ if (not tree) {
+ return {};
+ }
+
+ // The server does not store the Directory messages it just has
+ // sent us as part of the Tree message. If we want to be able to
+ // use the Directories as inputs for actions, we have to upload
+ // them manually.
+ auto root_digest = UploadTreeMessageDirectories(*tree);
+ if (not root_digest) {
+ return {};
+ }
+ artifacts.emplace(
+ action_result.output_directories(pos).path(),
+ Artifact::ObjectInfo{*root_digest, ObjectType::Tree});
+ } catch (...) {
+ return {};
+ }
+ ++pos;
+ }
+ tree_blobs = blob_reader.Next();
+ }
+ return artifacts;
+}
+
+auto BazelResponse::UploadTreeMessageDirectories(
+ bazel_re::Tree const& tree) const -> std::optional<ArtifactDigest> {
+ BlobContainer dir_blobs{};
+
+ auto rootdir_blob = ProcessDirectoryMessage(tree.root());
+ if (not rootdir_blob) {
+ return std::nullopt;
+ }
+ auto root_digest = rootdir_blob->digest;
+ dir_blobs.Emplace(std::move(*rootdir_blob));
+
+ for (auto const& subdir : tree.children()) {
+ auto subdir_blob = ProcessDirectoryMessage(subdir);
+ if (not subdir_blob) {
+ return std::nullopt;
+ }
+ dir_blobs.Emplace(std::move(*subdir_blob));
+ }
+
+ if (not network_->UploadBlobs(dir_blobs)) {
+ Logger::Log(LogLevel::Error,
+ "uploading Tree's Directory messages failed");
+ return std::nullopt;
+ }
+ return ArtifactDigest{root_digest};
+}
+
+auto BazelResponse::ProcessDirectoryMessage(
+ bazel_re::Directory const& dir) const noexcept -> std::optional<BazelBlob> {
+ auto data = dir.SerializeAsString();
+ auto digest = ArtifactDigest::Create(data);
+
+ if (tree_map_ and not tree_map_->HasTree(digest)) {
+ // cache in local tree map
+ auto tree = tree_map_->CreateTree();
+ if (not BazelMsgFactory::ReadObjectInfosFromDirectory(
+ dir,
+ [&tree](auto path, auto info) {
+ return tree.AddInfo(path, info);
+ }) or
+ not tree_map_->AddTree(digest, std::move(tree))) {
+ return std::nullopt;
+ }
+ }
+
+ return BazelBlob{std::move(digest), std::move(data)};
+}
diff --git a/src/buildtool/execution_api/remote/bazel/bazel_response.hpp b/src/buildtool/execution_api/remote/bazel/bazel_response.hpp
new file mode 100644
index 00000000..778efa0a
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bazel_response.hpp
@@ -0,0 +1,77 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_RESPONSE_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_RESPONSE_HPP
+
+#include <string>
+#include <vector>
+
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_execution_client.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_network.hpp"
+
+class BazelAction;
+
+/// \brief Bazel implementation of the abstract Execution Response.
+/// Access Bazel execution output data and obtain a Bazel Artifact.
+class BazelResponse final : public IExecutionResponse {
+ friend class BazelAction;
+
+ public:
+ auto Status() const noexcept -> StatusCode final {
+ return output_.status.ok() ? StatusCode::Success : StatusCode::Failed;
+ }
+ auto HasStdErr() const noexcept -> bool final {
+ return IsDigestNotEmpty(output_.action_result.stderr_digest());
+ }
+ auto HasStdOut() const noexcept -> bool final {
+ return IsDigestNotEmpty(output_.action_result.stdout_digest());
+ }
+ auto StdErr() noexcept -> std::string final {
+ return ReadStringBlob(output_.action_result.stderr_digest());
+ }
+ auto StdOut() noexcept -> std::string final {
+ return ReadStringBlob(output_.action_result.stdout_digest());
+ }
+ auto ExitCode() const noexcept -> int final {
+ return output_.action_result.exit_code();
+ }
+ auto IsCached() const noexcept -> bool final {
+ return output_.cached_result;
+ };
+
+ auto ActionDigest() const noexcept -> std::string final {
+ return action_id_;
+ }
+
+ auto Artifacts() const noexcept -> ArtifactInfos final;
+
+ private:
+ std::string action_id_{};
+ std::shared_ptr<BazelNetwork> const network_{};
+ std::shared_ptr<LocalTreeMap> const tree_map_{};
+ BazelExecutionClient::ExecutionOutput output_{};
+
+ BazelResponse(std::string action_id,
+ std::shared_ptr<BazelNetwork> network,
+ std::shared_ptr<LocalTreeMap> tree_map,
+ BazelExecutionClient::ExecutionOutput output)
+ : action_id_{std::move(action_id)},
+ network_{std::move(network)},
+ tree_map_{std::move(tree_map)},
+ output_{std::move(output)} {}
+
+ [[nodiscard]] auto ReadStringBlob(bazel_re::Digest const& id) noexcept
+ -> std::string;
+
+ [[nodiscard]] static auto IsDigestNotEmpty(bazel_re::Digest const& id)
+ -> bool {
+ return id.size_bytes() != 0;
+ }
+
+ [[nodiscard]] auto UploadTreeMessageDirectories(
+ bazel_re::Tree const& tree) const -> std::optional<ArtifactDigest>;
+
+ [[nodiscard]] auto ProcessDirectoryMessage(bazel_re::Directory const& dir)
+ const noexcept -> std::optional<BazelBlob>;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BAZEL_RESPONSE_HPP
diff --git a/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp b/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp
new file mode 100644
index 00000000..b8823236
--- /dev/null
+++ b/src/buildtool/execution_api/remote/bazel/bytestream_client.hpp
@@ -0,0 +1,185 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BYTESTREAM_CLIENT_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BYTESTREAM_CLIENT_HPP
+
+#include <functional>
+#include <iomanip>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "google/bytestream/bytestream.grpc.pb.h"
+#include "src/buildtool/execution_api/remote/bazel/bazel_client_common.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// Implements client side for google.bytestream.ByteStream service.
+class ByteStreamClient {
+ public:
+ class IncrementalReader {
+ friend class ByteStreamClient;
+
+ public:
+ /// \brief Read next chunk of data.
+ /// \returns empty string if stream finished and std::nullopt on error.
+ [[nodiscard]] auto Next() -> std::optional<std::string> {
+ google::bytestream::ReadResponse response{};
+ if (reader_->Read(&response)) {
+ return std::move(*response.mutable_data());
+ }
+
+ if (not finished_) {
+ auto status = reader_->Finish();
+ if (not status.ok()) {
+ LogStatus(logger_, LogLevel::Debug, status);
+ return std::nullopt;
+ }
+ finished_ = true;
+ }
+ return std::string{};
+ }
+
+ private:
+ Logger const* logger_;
+ grpc::ClientContext ctx_;
+ std::unique_ptr<grpc::ClientReader<google::bytestream::ReadResponse>>
+ reader_;
+ bool finished_{false};
+
+ IncrementalReader(
+ gsl::not_null<google::bytestream::ByteStream::Stub*> const& stub,
+ Logger const* logger,
+ std::string const& resource_name)
+ : logger_{logger} {
+ google::bytestream::ReadRequest request{};
+ request.set_resource_name(resource_name);
+ reader_ = stub->Read(&ctx_, request);
+ }
+ };
+
+ ByteStreamClient(std::string const& server,
+ Port port,
+ std::string const& user = "",
+ std::string const& pwd = "") noexcept {
+ stub_ = google::bytestream::ByteStream::NewStub(
+ CreateChannelWithCredentials(server, port, user, pwd));
+ }
+
+ [[nodiscard]] auto IncrementalRead(
+ std::string const& resource_name) const noexcept -> IncrementalReader {
+ return IncrementalReader{stub_.get(), &logger_, resource_name};
+ }
+
+ [[nodiscard]] auto Read(std::string const& resource_name) const noexcept
+ -> std::optional<std::string> {
+ auto reader = IncrementalRead(resource_name);
+ std::string output{};
+ auto data = reader.Next();
+ while (data and not data->empty()) {
+ output.append(data->begin(), data->end());
+ data = reader.Next();
+ }
+ if (not data) {
+ return std::nullopt;
+ }
+ return output;
+ }
+
+ [[nodiscard]] auto Write(std::string const& resource_name,
+ std::string const& data) const noexcept -> bool {
+ grpc::ClientContext ctx;
+ google::bytestream::WriteResponse response{};
+ auto writer = stub_->Write(&ctx, &response);
+
+ auto* allocated_data =
+ std::make_unique<std::string>(kChunkSize, '\0').release();
+ google::bytestream::WriteRequest request{};
+ request.set_resource_name(resource_name);
+ request.set_allocated_data(allocated_data);
+ std::size_t pos{};
+ do {
+ auto const size = std::min(data.size() - pos, kChunkSize);
+ allocated_data->resize(size);
+ data.copy(allocated_data->data(), size, pos);
+ request.set_write_offset(static_cast<int>(pos));
+ request.set_finish_write(pos + size >= data.size());
+ if (not writer->Write(request)) {
+ // According to the docs, quote:
+ // If there is an error or the connection is broken during the
+ // `Write()`, the client should check the status of the
+ // `Write()` by calling `QueryWriteStatus()` and continue
+ // writing from the returned `committed_size`.
+ auto const committed_size = QueryWriteStatus(resource_name);
+ if (committed_size <= 0) {
+ logger_.Emit(LogLevel::Debug,
+ "broken stream for upload to resource name {}",
+ resource_name);
+ return false;
+ }
+ pos = gsl::narrow<std::size_t>(committed_size);
+ }
+ else {
+ pos += kChunkSize;
+ }
+ } while (pos < data.size());
+ if (not writer->WritesDone()) {
+ logger_.Emit(LogLevel::Debug,
+ "broken stream for upload to resource name {}",
+ resource_name);
+ return false;
+ }
+
+ auto status = writer->Finish();
+ if (not status.ok()) {
+ LogStatus(&logger_, LogLevel::Debug, status);
+ return false;
+ }
+
+ return gsl::narrow<std::size_t>(response.committed_size()) ==
+ data.size();
+ }
+
+ template <class T_Input>
+ void ReadMany(
+ std::vector<T_Input> const& inputs,
+ std::function<std::string(T_Input const&)> const& to_resource_name,
+ std::function<void(std::string)> const& parse_data) const noexcept {
+ for (auto const& i : inputs) {
+ auto data = Read(to_resource_name(i));
+ if (data) {
+ parse_data(std::move(*data));
+ }
+ }
+ }
+
+ template <class T_Input>
+ [[nodiscard]] auto WriteMany(
+ std::vector<T_Input> const& inputs,
+ std::function<std::string(T_Input const&)> const& to_resource_name,
+ std::function<std::string(T_Input const&)> const& to_data)
+ const noexcept -> bool {
+ for (auto const& i : inputs) {
+ if (not Write(to_resource_name(i), to_data(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private:
+ // Chunk size for uploads (default size used by BuildBarn)
+ constexpr static std::size_t kChunkSize = 64 * 1024;
+
+ std::unique_ptr<google::bytestream::ByteStream::Stub> stub_;
+ Logger logger_{"ByteStreamClient"};
+
+ [[nodiscard]] auto QueryWriteStatus(
+ std::string const& resource_name) const noexcept -> std::int64_t {
+ grpc::ClientContext ctx;
+ google::bytestream::QueryWriteStatusRequest request{};
+ request.set_resource_name(resource_name);
+ google::bytestream::QueryWriteStatusResponse response{};
+ stub_->QueryWriteStatus(&ctx, request, &response);
+ return response.committed_size();
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_BAZEL_BYTESTREAM_CLIENT_HPP
diff --git a/src/buildtool/execution_api/remote/config.hpp b/src/buildtool/execution_api/remote/config.hpp
new file mode 100644
index 00000000..e29b8aa7
--- /dev/null
+++ b/src/buildtool/execution_api/remote/config.hpp
@@ -0,0 +1,72 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONFIG_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONFIG_HPP
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+class RemoteExecutionConfig {
+ public:
+ [[nodiscard]] static auto ParseAddress(std::string const& address) noexcept
+ -> std::optional<std::pair<std::string, int>> {
+ std::istringstream iss(address);
+ std::string host;
+ std::string port;
+ if (not std::getline(iss, host, ':') or
+ not std::getline(iss, port, ':')) {
+ return std::nullopt;
+ }
+ try {
+ return std::make_pair(host, std::stoi(port));
+ } catch (std::out_of_range const& e) {
+ Logger::Log(LogLevel::Error, "Port raised out_of_range exception.");
+ return std::nullopt;
+ } catch (std::invalid_argument const& e) {
+ Logger::Log(LogLevel::Error,
+ "Port raised invalid_argument exception.");
+ return std::nullopt;
+ }
+ }
+
+ // Obtain global instance
+ [[nodiscard]] static auto Instance() noexcept -> RemoteExecutionConfig& {
+ static RemoteExecutionConfig config;
+ return config;
+ }
+
+ [[nodiscard]] auto IsValidAddress() const noexcept -> bool {
+ return valid_;
+ }
+
+ [[nodiscard]] auto SetAddress(std::string const& address) noexcept -> bool {
+ auto pair = ParseAddress(address);
+ return pair and SetAddress(pair->first, pair->second);
+ }
+
+ [[nodiscard]] auto SetAddress(std::string const& host, int port) noexcept
+ -> bool {
+ host_ = host;
+ port_ = port,
+ valid_ = (not host.empty() and port >= 0 and port <= kMaxPortNumber);
+ return valid_;
+ }
+
+ [[nodiscard]] auto Host() const noexcept -> std::string { return host_; }
+ [[nodiscard]] auto Port() const noexcept -> int { return port_; }
+
+ private:
+ static constexpr int kMaxPortNumber{std::numeric_limits<uint16_t>::max()};
+ std::string host_{};
+ int port_{};
+ bool valid_{false};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_API_REMOTE_CONFIG_HPP
diff --git a/src/buildtool/execution_engine/TARGETS b/src/buildtool/execution_engine/TARGETS
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/buildtool/execution_engine/TARGETS
@@ -0,0 +1 @@
+{} \ No newline at end of file
diff --git a/src/buildtool/execution_engine/dag/TARGETS b/src/buildtool/execution_engine/dag/TARGETS
new file mode 100644
index 00000000..3c46f75f
--- /dev/null
+++ b/src/buildtool/execution_engine/dag/TARGETS
@@ -0,0 +1,17 @@
+{ "dag":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["dag"]
+ , "hdrs": ["dag.hpp"]
+ , "srcs": ["dag.cpp"]
+ , "deps":
+ [ ["src/utils/cpp", "type_safe_arithmetic"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/common", "action_description"]
+ , ["src/buildtool/common", "artifact_description"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/logging", "logging"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "execution_engine", "dag"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/execution_engine/dag/dag.cpp b/src/buildtool/execution_engine/dag/dag.cpp
new file mode 100644
index 00000000..96a74650
--- /dev/null
+++ b/src/buildtool/execution_engine/dag/dag.cpp
@@ -0,0 +1,263 @@
+#include "src/buildtool/execution_engine/dag/dag.hpp"
+
+#include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+auto DependencyGraph::CreateOutputArtifactNodes(
+ std::string const& action_id,
+ std::vector<std::string> const& file_paths,
+ std::vector<std::string> const& dir_paths,
+ bool is_tree_action)
+ -> std::pair<std::vector<DependencyGraph::NamedArtifactNodePtr>,
+ std::vector<DependencyGraph::NamedArtifactNodePtr>> {
+ if (is_tree_action) { // create tree artifact
+ auto artifact = ArtifactDescription{action_id}.ToArtifact();
+ auto const node_id = AddArtifact(std::move(artifact));
+ return std::make_pair(std::vector<NamedArtifactNodePtr>{},
+ std::vector<NamedArtifactNodePtr>{
+ {{}, &(*artifact_nodes_[node_id])}});
+ }
+
+ // create action artifacts
+ auto node_creator = [this, &action_id](auto* nodes, auto const& paths) {
+ for (auto const& artifact_path : paths) {
+ auto artifact =
+ ArtifactDescription{action_id,
+ std::filesystem::path{artifact_path}}
+ .ToArtifact();
+ auto const node_id = AddArtifact(std::move(artifact));
+ nodes->emplace_back(NamedArtifactNodePtr{
+ artifact_path, &(*artifact_nodes_[node_id])});
+ }
+ };
+
+ std::vector<NamedArtifactNodePtr> file_nodes{};
+ file_nodes.reserve(file_paths.size());
+ node_creator(&file_nodes, file_paths);
+
+ std::vector<NamedArtifactNodePtr> dir_nodes{};
+ dir_nodes.reserve(dir_paths.size());
+ node_creator(&dir_nodes, dir_paths);
+
+ return std::make_pair(std::move(file_nodes), std::move(dir_nodes));
+}
+
+auto DependencyGraph::CreateInputArtifactNodes(
+ ActionDescription::inputs_t const& inputs)
+ -> std::optional<std::vector<DependencyGraph::NamedArtifactNodePtr>> {
+ std::vector<NamedArtifactNodePtr> nodes{};
+
+ for (auto const& [local_path, artifact_desc] : inputs) {
+ auto artifact = artifact_desc.ToArtifact();
+ auto const node_id = AddArtifact(std::move(artifact));
+ nodes.push_back({local_path, &(*artifact_nodes_[node_id])});
+ }
+ return nodes;
+}
+
+auto DependencyGraph::CreateActionNode(Action const& action) noexcept
+ -> DependencyGraph::ActionNode* {
+ if (action.IsTreeAction() or not action.Command().empty()) {
+ auto const node_id = AddAction(action);
+ return &(*action_nodes_[node_id]);
+ }
+ return nullptr;
+}
+
+auto DependencyGraph::LinkNodePointers(
+ std::vector<NamedArtifactNodePtr> const& output_files,
+ std::vector<NamedArtifactNodePtr> const& output_dirs,
+ gsl::not_null<ActionNode*> const& action_node,
+ std::vector<NamedArtifactNodePtr> const& input_nodes) noexcept -> bool {
+ for (auto const& named_file : output_files) {
+ if (!named_file.node->AddBuilderActionNode(action_node) ||
+ !action_node->AddOutputFile(named_file)) {
+ return false;
+ }
+ }
+ for (auto const& named_dir : output_dirs) {
+ if (!named_dir.node->AddBuilderActionNode(action_node) ||
+ !action_node->AddOutputDir(named_dir)) {
+ return false;
+ }
+ }
+
+ for (auto const& named_node : input_nodes) {
+ if (!named_node.node->AddConsumerActionNode(action_node) ||
+ !action_node->AddDependency(named_node)) {
+ return false;
+ }
+ }
+
+ action_node->NotifyDoneLinking();
+
+ return true;
+}
+
+auto DependencyGraph::Add(std::vector<ActionDescription> const& actions)
+ -> bool {
+ for (auto const& action : actions) {
+ if (not AddAction(action)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+auto DependencyGraph::AddArtifact(ArtifactDescription const& description)
+ -> ArtifactIdentifier {
+ auto artifact = description.ToArtifact();
+ auto id = artifact.Id();
+ [[maybe_unused]] auto const node_id = AddArtifact(std::move(artifact));
+ return id;
+}
+
+auto DependencyGraph::AddAction(ActionDescription const& description) -> bool {
+ auto output_nodes =
+ CreateOutputArtifactNodes(description.Id(),
+ description.OutputFiles(),
+ description.OutputDirs(),
+ description.GraphAction().IsTreeAction());
+ auto* action_node = CreateActionNode(description.GraphAction());
+ auto input_nodes = CreateInputArtifactNodes(description.Inputs());
+
+ if (action_node == nullptr or not input_nodes.has_value() or
+ (output_nodes.first.empty() and output_nodes.second.empty())) {
+ return false;
+ }
+
+ return LinkNodePointers(
+ output_nodes.first, output_nodes.second, action_node, *input_nodes);
+}
+
+auto DependencyGraph::AddAction(Action const& a) noexcept
+ -> DependencyGraph::ActionNodeIdentifier {
+ auto id = a.Id();
+ auto const action_it = action_ids_.find(id);
+ if (action_it != action_ids_.end()) {
+ return action_it->second;
+ }
+ action_nodes_.emplace_back(ActionNode::Create(a));
+ ActionNodeIdentifier node_id{action_nodes_.size() - 1};
+ action_ids_[id] = node_id;
+ return node_id;
+}
+
+auto DependencyGraph::AddAction(Action&& a) noexcept
+ -> DependencyGraph::ActionNodeIdentifier {
+ auto const& id = a.Id();
+ auto const action_it = action_ids_.find(id);
+ if (action_it != action_ids_.end()) {
+ return action_it->second;
+ }
+ action_nodes_.emplace_back(ActionNode::Create(std::move(a)));
+ ActionNodeIdentifier node_id{action_nodes_.size() - 1};
+ action_ids_[id] = node_id;
+ return node_id;
+}
+
+auto DependencyGraph::AddArtifact(Artifact const& a) noexcept
+ -> DependencyGraph::ArtifactNodeIdentifier {
+ auto const& id = a.Id();
+ auto const artifact_it = artifact_ids_.find(id);
+ if (artifact_it != artifact_ids_.end()) {
+ return artifact_it->second;
+ }
+ artifact_nodes_.emplace_back(ArtifactNode::Create(a));
+ ArtifactNodeIdentifier node_id{artifact_nodes_.size() - 1};
+ artifact_ids_[id] = node_id;
+ return node_id;
+}
+
+auto DependencyGraph::AddArtifact(Artifact&& a) noexcept
+ -> DependencyGraph::ArtifactNodeIdentifier {
+ auto id = a.Id();
+ auto const artifact_it = artifact_ids_.find(id);
+ if (artifact_it != artifact_ids_.end()) {
+ return artifact_it->second;
+ }
+ artifact_nodes_.emplace_back(ArtifactNode::Create(std::move(a)));
+ ArtifactNodeIdentifier node_id{artifact_nodes_.size() - 1};
+ artifact_ids_[id] = node_id;
+ return node_id;
+}
+
+auto DependencyGraph::ArtifactIdentifiers() const noexcept
+ -> std::unordered_set<ArtifactIdentifier> {
+ std::unordered_set<ArtifactIdentifier> ids;
+ std::transform(
+ std::begin(artifact_ids_),
+ std::end(artifact_ids_),
+ std::inserter(ids, std::begin(ids)),
+ [](auto const& artifact_id_pair) { return artifact_id_pair.first; });
+ return ids;
+}
+
+auto DependencyGraph::ArtifactNodeWithId(ArtifactIdentifier const& id)
+ const noexcept -> DependencyGraph::ArtifactNode const* {
+ auto it_to_artifact = artifact_ids_.find(id);
+ if (it_to_artifact == artifact_ids_.end()) {
+ return nullptr;
+ }
+ return &(*artifact_nodes_[it_to_artifact->second]);
+}
+
+auto DependencyGraph::ActionNodeWithId(ActionIdentifier const& id)
+ const noexcept -> DependencyGraph::ActionNode const* {
+ auto it_to_action = action_ids_.find(id);
+ if (it_to_action == action_ids_.end()) {
+ return nullptr;
+ }
+ return &(*action_nodes_[it_to_action->second]);
+}
+
+auto DependencyGraph::ActionNodeOfArtifactWithId(ArtifactIdentifier const& id)
+ const noexcept -> DependencyGraph::ActionNode const* {
+ auto const* node = ArtifactNodeWithId(id);
+ if (node != nullptr) {
+ auto const& children = node->Children();
+ if (children.empty()) {
+ return nullptr;
+ }
+ return children[0];
+ }
+ return nullptr;
+}
+
+auto DependencyGraph::ArtifactWithId(
+ ArtifactIdentifier const& id) const noexcept -> std::optional<Artifact> {
+ auto const* node = ArtifactNodeWithId(id);
+ if (node != nullptr) {
+ return node->Content();
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] auto DependencyGraph::ActionWithId(
+ ActionIdentifier const& id) const noexcept -> std::optional<Action> {
+ auto const* node = ActionNodeWithId(id);
+ if (node != nullptr) {
+ return node->Content();
+ }
+ return std::nullopt;
+}
+
+auto DependencyGraph::ActionOfArtifactWithId(
+ ArtifactIdentifier const& artifact_id) const noexcept
+ -> std::optional<Action> {
+ auto const* node = ActionNodeOfArtifactWithId(artifact_id);
+ if (node != nullptr) {
+ return node->Content();
+ }
+ return std::nullopt;
+}
+
+auto DependencyGraph::ActionIdOfArtifactWithId(
+ ArtifactIdentifier const& artifact_id) const noexcept
+ -> std::optional<ActionIdentifier> {
+ auto const& action = ActionOfArtifactWithId(artifact_id);
+ if (action) {
+ return action->Id();
+ }
+ return std::nullopt;
+}
diff --git a/src/buildtool/execution_engine/dag/dag.hpp b/src/buildtool/execution_engine/dag/dag.hpp
new file mode 100644
index 00000000..4f29d79a
--- /dev/null
+++ b/src/buildtool/execution_engine/dag/dag.hpp
@@ -0,0 +1,613 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_DAG_DAG_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_DAG_DAG_HPP
+
+#include <algorithm>
+#include <atomic>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <variant>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/action.hpp"
+#include "src/buildtool/common/action_description.hpp"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/utils/cpp/type_safe_arithmetic.hpp"
+
+/// \brief Plain DirectedAcyclicGraph.
+/// Additional properties (e.g. bipartiteness) can be encoded in nodes.
+/// Deliberately not using \ref DirectedAcyclicGraph::Node anywhere to avoid
+/// vtable lookups. For now, it does not hold any data.
+class DirectedAcyclicGraph {
+ public:
+ /// \brief Abstract class for DAG nodes.
+ /// \tparam T_Content Type of content.
+ /// \tparam T_Other Type of neighboring nodes.
+ /// Sub classes need to implement \ref IsValid method.
+ /// TODO: once we have hashes, require sub classes to implement generating
+ /// IDs depending on its unique content.
+ template <typename T_Content, typename T_Other>
+ class Node {
+ public:
+ using Id = std::uintptr_t;
+ using OtherNode = T_Other;
+ using OtherNodePtr = gsl::not_null<OtherNode*>;
+
+ explicit Node(T_Content&& content) noexcept
+ : content_{std::move(content)} {}
+
+ // NOLINTNEXTLINE(modernize-pass-by-value)
+ explicit Node(T_Content const& content) noexcept : content_{content} {}
+
+ Node(T_Content const& content,
+ std::vector<OtherNodePtr> const& parents,
+ std::vector<OtherNodePtr> const& children) noexcept
+ : content_{content}, parents_{parents}, children_{children} {}
+
+ // No copies and no moves are allowed. The only way to "copy" an
+ // instance is to copy its raw pointer.
+ Node(Node const&) = delete;
+ Node(Node&&) = delete;
+ auto operator=(Node const&) -> Node& = delete;
+ auto operator=(Node &&) -> Node& = delete;
+
+ [[nodiscard]] auto NodeId() const noexcept -> Id {
+ // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+ return reinterpret_cast<Id>(this);
+ }
+
+ [[nodiscard]] auto Content() const& noexcept -> T_Content const& {
+ return content_;
+ }
+
+ [[nodiscard]] auto Content() && noexcept -> T_Content {
+ return std::move(content_);
+ }
+
+ [[nodiscard]] auto Parents() const& noexcept
+ -> std::vector<OtherNodePtr> const& {
+ return parents_;
+ }
+
+ [[nodiscard]] auto Children() const& noexcept
+ -> std::vector<OtherNodePtr> const& {
+ return children_;
+ }
+
+ auto AddParent(OtherNodePtr const& parent) noexcept -> bool {
+ parents_.push_back(parent);
+ return true;
+ }
+
+ auto AddParent(OtherNodePtr&& parent) noexcept -> bool {
+ parents_.push_back(std::move(parent));
+ return true;
+ }
+
+ auto AddChild(OtherNodePtr const& child) noexcept -> bool {
+ children_.push_back(child);
+ return true;
+ }
+
+ auto AddChild(OtherNodePtr&& child) noexcept -> bool {
+ children_.push_back(std::move(child));
+ return true;
+ }
+
+ [[nodiscard]] virtual auto IsValid() const noexcept -> bool = 0;
+ virtual ~Node() noexcept = default;
+
+ private:
+ T_Content content_{};
+ std::vector<OtherNodePtr> parents_{};
+ std::vector<OtherNodePtr> children_{};
+ };
+
+ /// \brief Lock-free class for basic traversal state data
+ /// Provides the following atomic operations:
+ /// - Retrieve (previous) state and mark as discovered, which will allow us
+ /// to know whether we should queue a visit the node or not at the same
+ /// time that we mark that its visit should not be queued by other threads,
+ /// since it is being queued by the current caller to this method or it has
+ /// already been queued by a previous caller.
+ /// Note that "discovered" refers to "queued for visit" here.
+ /// - Retrieve (previous) state and mark as queued to be processed, which
+ /// will allow us to ensure that processing a node is queued at most once.
+ class NodeTraversalState {
+ public:
+ NodeTraversalState() noexcept = default;
+ NodeTraversalState(NodeTraversalState const&) = delete;
+ NodeTraversalState(NodeTraversalState&&) = delete;
+ auto operator=(NodeTraversalState const&)
+ -> NodeTraversalState& = delete;
+ auto operator=(NodeTraversalState &&) -> NodeTraversalState& = delete;
+ ~NodeTraversalState() noexcept = default;
+
+ /// \brief Sets traversal state as discovered
+ /// \returns True if it was already discovered, false otherwise
+ /// Note: this is an atomic, lock-free operation
+ [[nodiscard]] auto GetAndMarkDiscovered() noexcept -> bool {
+ return std::atomic_exchange(&has_been_discovered_, true);
+ }
+
+ /// \brief Sets traversal state as queued to be processed
+ /// \returns True if it was already queued to be processed, false
+ /// otherwise
+ /// Note: this is an atomic, lock-free operation
+ [[nodiscard]] auto GetAndMarkQueuedToBeProcessed() noexcept -> bool {
+ return std::atomic_exchange(&is_queued_to_be_processed_, true);
+ }
+
+ /// \brief Check if a node is required to be processed or not
+ [[nodiscard]] auto IsRequired() const noexcept -> bool {
+ return is_required_;
+ }
+
+ /// \brief Mark node as required to be executed
+ /// Note: this should be upon node discovery (visit) while traversing
+ /// the graph
+ void MarkRequired() noexcept { is_required_ = true; }
+
+ private:
+ std::atomic<bool> has_been_discovered_{false};
+ std::atomic<bool> is_queued_to_be_processed_{false};
+ std::atomic<bool> is_required_{false};
+ };
+
+ protected:
+ template <typename T_Node>
+ [[nodiscard]] static auto check_validity(
+ gsl::not_null<T_Node*> node) noexcept -> bool {
+ // Check node-specified validity (e.g. bipartiteness requirements)
+ if (!node->IsValid()) {
+ return false;
+ }
+
+ // Search for cycles
+ thread_local std::vector<typename T_Node::Id> stack{};
+ for (auto const& child : node->Children()) {
+ auto node_id = child->NodeId();
+
+ if (std::find(stack.begin(), stack.end(), node_id) != stack.end()) {
+ return false;
+ }
+
+ stack.push_back(node_id);
+ bool valid = check_validity(child);
+ stack.pop_back();
+
+ if (!valid) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+};
+
+class DependencyGraph : DirectedAcyclicGraph {
+ public:
+ // Forward declaration
+ class ArtifactNode;
+
+ // Node identifier for actions
+ struct ActionNodeIdentifierTag : type_safe_arithmetic_tag<std::size_t> {};
+ using ActionNodeIdentifier = type_safe_arithmetic<ActionNodeIdentifierTag>;
+
+ // Node identifier for artifacts
+ struct ArtifactNodeIdentifierTag : type_safe_arithmetic_tag<std::size_t> {};
+ using ArtifactNodeIdentifier =
+ type_safe_arithmetic<ArtifactNodeIdentifierTag>;
+
+ /// \brief Class for traversal state data specific for ActionNode's
+ /// Provides the following atomic operations (listed on the public methods):
+ class ActionNodeTraversalState : public NodeTraversalState {
+ public:
+ ActionNodeTraversalState() noexcept = default;
+ ActionNodeTraversalState(ActionNodeTraversalState const&) = delete;
+ ActionNodeTraversalState(ActionNodeTraversalState&&) = delete;
+ auto operator=(ActionNodeTraversalState const&)
+ -> ActionNodeTraversalState& = delete;
+ auto operator=(ActionNodeTraversalState &&)
+ -> ActionNodeTraversalState& = delete;
+ ~ActionNodeTraversalState() noexcept = default;
+
+ /// \brief Acknowledge that a dependency was made available and return
+ /// whether the action is ready to be executed
+ [[nodiscard]] auto NotifyAvailableDepAndCheckReady() noexcept -> bool {
+ return std::atomic_fetch_sub(&unavailable_deps_, 1) == 1;
+ }
+
+ /// \brief Check whether the action can be now executed or not.
+ /// Note: checking state without modifying (unlike
+ /// NotifyAvailableDepAndCheckReady()) is useful in the case that when
+ /// the action node is visited all its dependencies were already
+ /// available
+ [[nodiscard]] auto IsReady() const noexcept -> bool {
+ return unavailable_deps_ == 0;
+ }
+
+ /// \brief Initialise number of unavailable dependencies
+ /// \param[in] count Number of unavailable dependencies
+ /// Note: this method should be called previous to the start of the
+ /// traversal (once the action node is built)
+ void InitUnavailableDeps(std::size_t count) noexcept {
+ unavailable_deps_ = static_cast<int>(count);
+ }
+
+ private:
+ std::atomic<int> unavailable_deps_{-1};
+ };
+
+ /// \brief Class for traversal state data specific for ArtifactNode's
+ /// Provides the following atomic operations:
+ /// - Mark the artifact in this node as available
+ /// - Check whether the artifact in this node is available or not
+ class ArtifactNodeTraversalState : public NodeTraversalState {
+ public:
+ ArtifactNodeTraversalState() noexcept = default;
+ ArtifactNodeTraversalState(ArtifactNodeTraversalState const&) = delete;
+ ArtifactNodeTraversalState(ArtifactNodeTraversalState&&) = delete;
+ auto operator=(ArtifactNodeTraversalState const&)
+ -> ArtifactNodeTraversalState& = delete;
+ auto operator=(ArtifactNodeTraversalState &&)
+ -> ArtifactNodeTraversalState& = delete;
+ ~ArtifactNodeTraversalState() noexcept = default;
+
+ [[nodiscard]] auto IsAvailable() const noexcept -> bool {
+ return is_available_;
+ }
+
+ void MakeAvailable() noexcept { is_available_ = true; }
+
+ private:
+ std::atomic<bool> is_available_{false};
+ };
+
+ /// \brief Action node (bipartite)
+ /// Cannot be entry (see \ref IsValid).
+ class ActionNode final : public Node<Action, ArtifactNode> {
+ using base = Node<Action, ArtifactNode>;
+
+ public:
+ using base::base;
+ using Ptr = gsl::not_null<std::unique_ptr<ActionNode>>;
+ struct NamedOtherNodePtr {
+ Action::LocalPath path;
+ base::OtherNodePtr node;
+ };
+
+ [[nodiscard]] static auto Create(Action const& content) noexcept
+ -> Ptr {
+ return std::make_unique<ActionNode>(content);
+ }
+
+ [[nodiscard]] static auto Create(Action&& content) noexcept -> Ptr {
+ return std::make_unique<ActionNode>(std::move(content));
+ }
+
+ // only valid if it has parents
+ [[nodiscard]] auto IsValid() const noexcept -> bool final {
+ return (!base::Parents().empty());
+ }
+
+ [[nodiscard]] auto AddOutputFile(
+ NamedOtherNodePtr const& output) noexcept -> bool {
+ base::AddParent(output.node);
+ output_files_.push_back(output);
+ return true;
+ }
+
+ [[nodiscard]] auto AddOutputFile(NamedOtherNodePtr&& output) noexcept
+ -> bool {
+ base::AddParent(output.node);
+ output_files_.push_back(std::move(output));
+ return true;
+ }
+
+ [[nodiscard]] auto AddOutputDir(
+ NamedOtherNodePtr const& output) noexcept -> bool {
+ base::AddParent(output.node);
+ output_dirs_.push_back(output);
+ return true;
+ }
+
+ [[nodiscard]] auto AddOutputDir(NamedOtherNodePtr&& output) noexcept
+ -> bool {
+ base::AddParent(output.node);
+ output_dirs_.push_back(std::move(output));
+ return true;
+ }
+
+ [[nodiscard]] auto AddDependency(
+ NamedOtherNodePtr const& dependency) noexcept -> bool {
+ base::AddChild(dependency.node);
+ dependencies_.push_back(dependency);
+ return true;
+ }
+
+ [[nodiscard]] auto AddDependency(
+ NamedOtherNodePtr&& dependency) noexcept -> bool {
+ base::AddChild(dependency.node);
+ dependencies_.push_back(std::move(dependency));
+ return true;
+ }
+
+ [[nodiscard]] auto OutputFiles()
+ const& -> std::vector<NamedOtherNodePtr> const& {
+ return output_files_;
+ }
+
+ [[nodiscard]] auto OutputDirs()
+ const& -> std::vector<NamedOtherNodePtr> const& {
+ return output_dirs_;
+ }
+
+ [[nodiscard]] auto Command() const -> std::vector<std::string> {
+ return Content().Command();
+ }
+
+ [[nodiscard]] auto Env() const -> std::map<std::string, std::string> {
+ return Content().Env();
+ }
+
+ [[nodiscard]] auto MayFail() const -> std::optional<std::string> {
+ return Content().MayFail();
+ }
+
+ [[nodiscard]] auto NoCache() const -> bool {
+ return Content().NoCache();
+ }
+
+ [[nodiscard]] auto Dependencies()
+ const& -> std::vector<NamedOtherNodePtr> const& {
+ return dependencies_;
+ }
+
+ [[nodiscard]] auto OutputFilePaths() const
+ -> std::vector<Action::LocalPath> {
+ return NodePaths(output_files_);
+ }
+
+ [[nodiscard]] auto OutputFileIds() const
+ -> std::vector<ArtifactIdentifier> {
+ return Ids(output_files_);
+ }
+
+ [[nodiscard]] auto OutputDirPaths() const
+ -> std::vector<Action::LocalPath> {
+ return NodePaths(output_dirs_);
+ }
+
+ [[nodiscard]] auto OutputDirIds() const
+ -> std::vector<ArtifactIdentifier> {
+ return Ids(output_dirs_);
+ }
+
+ [[nodiscard]] auto DependencyPaths() const
+ -> std::vector<Action::LocalPath> {
+ return NodePaths(dependencies_);
+ }
+
+ [[nodiscard]] auto DependencyIds() const
+ -> std::vector<ArtifactIdentifier> {
+ return Ids(dependencies_);
+ }
+
+ // To initialise the action traversal specific data before traversing
+ // the graph
+ void NotifyDoneLinking() const noexcept {
+ traversal_state_->InitUnavailableDeps(Children().size());
+ }
+
+ [[nodiscard]] auto TraversalState() const noexcept
+ -> ActionNodeTraversalState* {
+ return traversal_state_.get();
+ }
+
+ private:
+ std::vector<NamedOtherNodePtr> output_files_;
+ std::vector<NamedOtherNodePtr> output_dirs_;
+ std::vector<NamedOtherNodePtr> dependencies_;
+ std::unique_ptr<ActionNodeTraversalState> traversal_state_{
+ std::make_unique<ActionNodeTraversalState>()};
+
+ // Collect paths from named nodes.
+ // TODO(oreiche): This could be potentially speed up by using a wrapper
+ // iterator to provide a read-only view (similar to BlobContainer)
+ [[nodiscard]] static auto NodePaths(
+ std::vector<NamedOtherNodePtr> const& nodes)
+ -> std::vector<Action::LocalPath> {
+ std::vector<Action::LocalPath> paths{nodes.size()};
+ std::transform(
+ nodes.cbegin(),
+ nodes.cend(),
+ paths.begin(),
+ [](auto const& named_node) { return named_node.path; });
+ return paths;
+ }
+
+ /// \brief Collect ids from named nodes (artifacts in this case)
+ [[nodiscard]] static auto Ids(
+ std::vector<NamedOtherNodePtr> const& nodes)
+ -> std::vector<ArtifactIdentifier> {
+ std::vector<ArtifactIdentifier> ids{nodes.size()};
+ std::transform(nodes.cbegin(),
+ nodes.cend(),
+ ids.begin(),
+ [](auto const& named_node) {
+ return named_node.node->Content().Id();
+ });
+ return ids;
+ }
+ };
+
+ /// \brief Artifact node (bipartite)
+ /// Can be entry or leaf (see \ref IsValid) and can only have single child
+ /// (see \ref AddChild)
+ class ArtifactNode final : public Node<Artifact, ActionNode> {
+ using base = Node<Artifact, ActionNode>;
+
+ public:
+ using base::base;
+ using typename base::OtherNode;
+ using typename base::OtherNodePtr;
+ using Ptr = gsl::not_null<std::unique_ptr<ArtifactNode>>;
+
+ [[nodiscard]] static auto Create(Artifact const& content) noexcept
+ -> Ptr {
+ return std::make_unique<ArtifactNode>(content);
+ }
+
+ [[nodiscard]] static auto Create(Artifact&& content) noexcept -> Ptr {
+ return std::make_unique<ArtifactNode>(std::move(content));
+ }
+
+ [[nodiscard]] auto AddBuilderActionNode(
+ OtherNodePtr const& action) noexcept -> bool {
+ if (base::Children().empty()) {
+ return base::AddChild(action);
+ }
+ Logger::Log(LogLevel::Error,
+ "cannot set a second builder for artifact {}",
+ ToHexString(Content().Id()));
+ return false;
+ }
+
+ [[nodiscard]] auto AddConsumerActionNode(
+ OtherNodePtr const& action) noexcept -> bool {
+ return base::AddParent(action);
+ }
+
+ [[nodiscard]] auto IsValid() const noexcept -> bool final {
+ return base::Children().size() <= 1;
+ }
+
+ [[nodiscard]] auto HasBuilderAction() const noexcept -> bool {
+ return !base::Children().empty();
+ }
+
+ [[nodiscard]] auto BuilderActionNode() const noexcept
+ -> ActionNode const* {
+ return HasBuilderAction() ? base::Children()[0].get() : nullptr;
+ }
+
+ [[nodiscard]] auto TraversalState() const noexcept
+ -> ArtifactNodeTraversalState* {
+ return traversal_state_.get();
+ }
+
+ private:
+ std::unique_ptr<ArtifactNodeTraversalState> traversal_state_{
+ std::make_unique<ArtifactNodeTraversalState>()};
+ };
+
+ using NamedArtifactNodePtr = ActionNode::NamedOtherNodePtr;
+
+ DependencyGraph() noexcept = default;
+
+ // DependencyGraph should not be copiable or movable. This could be changed
+ // in the case we want to make the graph construction to be functional
+ DependencyGraph(DependencyGraph const&) = delete;
+ DependencyGraph(DependencyGraph&&) = delete;
+ auto operator=(DependencyGraph const&) -> DependencyGraph& = delete;
+ auto operator=(DependencyGraph &&) -> DependencyGraph& = delete;
+ ~DependencyGraph() noexcept = default;
+
+ [[nodiscard]] auto Add(std::vector<ActionDescription> const& actions)
+ -> bool;
+
+ [[nodiscard]] auto AddAction(ActionDescription const& description) -> bool;
+
+ [[nodiscard]] auto AddArtifact(ArtifactDescription const& description)
+ -> ArtifactIdentifier;
+
+ [[nodiscard]] auto ArtifactIdentifiers() const noexcept
+ -> std::unordered_set<ArtifactIdentifier>;
+
+ [[nodiscard]] auto ArtifactNodeWithId(
+ ArtifactIdentifier const& id) const noexcept -> ArtifactNode const*;
+
+ [[nodiscard]] auto ActionNodeWithId(
+ ActionIdentifier const& id) const noexcept -> ActionNode const*;
+
+ [[nodiscard]] auto ActionNodeOfArtifactWithId(
+ ArtifactIdentifier const& artifact_id) const noexcept
+ -> ActionNode const*;
+
+ [[nodiscard]] auto ArtifactWithId(
+ ArtifactIdentifier const& id) const noexcept -> std::optional<Artifact>;
+
+ [[nodiscard]] auto ActionWithId(ActionIdentifier const& id) const noexcept
+ -> std::optional<Action>;
+
+ [[nodiscard]] auto ActionOfArtifactWithId(
+ ArtifactIdentifier const& artifact_id) const noexcept
+ -> std::optional<Action>;
+
+ [[nodiscard]] auto ActionIdOfArtifactWithId(
+ ArtifactIdentifier const& artifact_id) const noexcept
+ -> std::optional<ActionIdentifier>;
+
+ [[nodiscard]] auto IsValid() const noexcept -> bool {
+ for (auto const& node : artifact_nodes_) {
+ if (!DirectedAcyclicGraph::check_validity<
+ std::remove_reference_t<decltype(*node)>>(&(*node))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private:
+ // List of action nodes we already created
+ std::vector<ActionNode::Ptr> action_nodes_{};
+
+ // List of artifact nodes we already created
+ std::vector<ArtifactNode::Ptr> artifact_nodes_{};
+
+ // Associates global action identifier to local node id
+ std::unordered_map<ActionIdentifier, ActionNodeIdentifier> action_ids_{};
+
+ // Associates global artifact identifier to local node id
+ std::unordered_map<ArtifactIdentifier, ArtifactNodeIdentifier>
+ artifact_ids_{};
+
+ [[nodiscard]] auto CreateOutputArtifactNodes(
+ std::string const& action_id,
+ std::vector<std::string> const& file_paths,
+ std::vector<std::string> const& dir_paths,
+ bool is_tree_action)
+ -> std::pair<std::vector<DependencyGraph::NamedArtifactNodePtr>,
+ std::vector<DependencyGraph::NamedArtifactNodePtr>>;
+
+ [[nodiscard]] auto CreateInputArtifactNodes(
+ ActionDescription::inputs_t const& inputs)
+ -> std::optional<std::vector<NamedArtifactNodePtr>>;
+
+ [[nodiscard]] auto CreateActionNode(Action const& action) noexcept
+ -> ActionNode*;
+
+ [[nodiscard]] static auto LinkNodePointers(
+ std::vector<NamedArtifactNodePtr> const& output_files,
+ std::vector<NamedArtifactNodePtr> const& output_dirs,
+ gsl::not_null<ActionNode*> const& action_node,
+ std::vector<NamedArtifactNodePtr> const& input_nodes) noexcept -> bool;
+
+ [[nodiscard]] auto AddAction(Action const& a) noexcept
+ -> ActionNodeIdentifier;
+ [[nodiscard]] auto AddAction(Action&& a) noexcept -> ActionNodeIdentifier;
+ [[nodiscard]] auto AddArtifact(Artifact const& a) noexcept
+ -> ArtifactNodeIdentifier;
+ [[nodiscard]] auto AddArtifact(Artifact&& a) noexcept
+ -> ArtifactNodeIdentifier;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_DAG_DAG_HPP
diff --git a/src/buildtool/execution_engine/executor/TARGETS b/src/buildtool/execution_engine/executor/TARGETS
new file mode 100644
index 00000000..de0fd8f3
--- /dev/null
+++ b/src/buildtool/execution_engine/executor/TARGETS
@@ -0,0 +1,16 @@
+{ "executor":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["executor"]
+ , "hdrs": ["executor.hpp"]
+ , "deps":
+ [ ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/common", "config"]
+ , ["src/buildtool/common", "tree"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/execution_engine/dag", "dag"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "execution_engine", "executor"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/execution_engine/executor/executor.hpp b/src/buildtool/execution_engine/executor/executor.hpp
new file mode 100644
index 00000000..d7447ed8
--- /dev/null
+++ b/src/buildtool/execution_engine/executor/executor.hpp
@@ -0,0 +1,532 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_HPP
+
+#include <algorithm>
+#include <functional>
+#include <iostream>
+#include <map>
+#include <optional>
+#include <type_traits>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/common/statistics.hpp"
+#include "src/buildtool/common/tree.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_engine/dag/dag.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// \brief Implementations for executing actions and uploading artifacts.
+class ExecutorImpl {
+ public:
+ /// \brief Execute action and obtain response.
+ /// \returns std::nullopt for actions without response (e.g., tree actions).
+ /// \returns nullptr on error.
+ [[nodiscard]] static auto ExecuteAction(
+ Logger const& logger,
+ gsl::not_null<DependencyGraph::ActionNode const*> const& action,
+ gsl::not_null<IExecutionApi*> const& api,
+ std::map<std::string, std::string> const& properties,
+ IExecutionAction::CacheFlag cache_flag)
+ -> std::optional<IExecutionResponse::Ptr> {
+ auto const& inputs = action->Dependencies();
+ auto const root_digest = CreateRootDigest(api, inputs);
+ if (not root_digest) {
+ Logger::Log(LogLevel::Error,
+ "failed to create root digest for input artifacts.");
+ return nullptr;
+ }
+
+ if (action->Content().IsTreeAction()) {
+ auto const& tree_artifact = action->OutputDirs()[0].node->Content();
+ bool failed_inputs = false;
+ for (auto const& [local_path, artifact] : inputs) {
+ failed_inputs |= artifact->Content().Info()->failed;
+ }
+ tree_artifact.SetObjectInfo(
+ *root_digest, ObjectType::Tree, failed_inputs);
+ return std::nullopt;
+ }
+
+ Statistics::Instance().IncrementActionsQueuedCounter();
+
+ logger.Emit(LogLevel::Trace, [&inputs]() {
+ std::ostringstream oss{};
+ oss << "start processing" << std::endl;
+ for (auto const& [local_path, artifact] : inputs) {
+ auto const& info = artifact->Content().Info();
+ oss << fmt::format(
+ " - needs {} {}",
+ local_path,
+ info ? info->ToString() : std::string{"[???]"})
+ << std::endl;
+ }
+ return oss.str();
+ });
+
+ auto remote_action = api->CreateAction(*root_digest,
+ action->Command(),
+ action->OutputFilePaths(),
+ action->OutputDirPaths(),
+ action->Env(),
+ properties);
+
+ if (remote_action == nullptr) {
+ logger.Emit(LogLevel::Error,
+ "failed to create action for execution.");
+ return nullptr;
+ }
+
+ // set action options
+ remote_action->SetCacheFlag(cache_flag);
+ remote_action->SetTimeout(IExecutionAction::kDefaultTimeout);
+ return remote_action->Execute(&logger);
+ }
+
+ /// \brief Ensures the artifact is available to the CAS, either checking
+ /// that its existing digest corresponds to that of an object already
+ /// available or by uploading it if there is no digest in the artifact. In
+ /// the later case, the new digest is saved in the artifact
+ /// \param[in] artifact The artifact to process.
+ /// \returns True if artifact is available at the point of return, false
+ /// otherwise
+ [[nodiscard]] static auto VerifyOrUploadArtifact(
+ gsl::not_null<DependencyGraph::ArtifactNode const*> const& artifact,
+ gsl::not_null<IExecutionApi*> const& api) noexcept -> bool {
+ auto const object_info_opt = artifact->Content().Info();
+ auto const file_path_opt = artifact->Content().FilePath();
+ // If there is no object info and no file path, the artifact can not be
+ // processed: it means its definition is ill-formed or that it is the
+ // output of an action, in which case it shouldn't have reached here
+ if (not object_info_opt and not file_path_opt) {
+ Logger::Log(LogLevel::Error,
+ "artifact {} can not be processed.",
+ artifact->Content().Id());
+ return false;
+ }
+ // If the artifact has digest, we check that an object with this digest
+ // is available to the execution API
+ if (object_info_opt) {
+ if (not api->IsAvailable(object_info_opt->digest) and
+ not UploadGitBlob(api,
+ artifact->Content().Repository(),
+ object_info_opt->digest,
+ /*skip_check=*/true)) {
+ Logger::Log(
+ LogLevel::Error,
+ "artifact {} should be present in CAS but is missing.",
+ artifact->Content().Id());
+ return false;
+ }
+ return true;
+ }
+
+ // Otherwise, we upload the new file to make it available to the
+ // execution API
+ // Note that we can be sure now that file_path_opt has a value and
+ // that the path stored is relative to the workspace dir, so we need to
+ // prepend it
+ auto repo = artifact->Content().Repository();
+ auto new_info = UploadFile(api, repo, *file_path_opt);
+ if (not new_info) {
+ Logger::Log(LogLevel::Error,
+ "artifact in {} could not be uploaded to CAS.",
+ file_path_opt->string());
+ return false;
+ }
+
+ // And we save the digest object type in the artifact
+ artifact->Content().SetObjectInfo(*new_info, false);
+ return true;
+ }
+
+ /// \brief Lookup blob via digest in local git repositories and upload.
+ /// \param api The endpoint used for uploading
+ /// \param repo The global repository name, the artifact belongs to
+ /// \param digest The digest of the object
+ /// \param skip_check Skip check for existence before upload
+ /// \returns true on success
+ [[nodiscard]] static auto UploadGitBlob(
+ gsl::not_null<IExecutionApi*> const& api,
+ std::string const& repo,
+ ArtifactDigest const& digest,
+ bool skip_check) noexcept -> bool {
+ auto const& repo_config = RepositoryConfig::Instance();
+ std::optional<std::string> blob{};
+ if (auto const* ws_root = repo_config.WorkspaceRoot(repo)) {
+ // try to obtain blob from local workspace's Git CAS, if any
+ blob = ws_root->ReadBlob(digest.hash());
+ }
+ if (not blob) {
+ // try to obtain blob from global Git CAS, if any
+ blob = repo_config.ReadBlobFromGitCAS(digest.hash());
+ }
+ return blob and
+ api->Upload(BlobContainer{{BazelBlob{digest, std::move(*blob)}}},
+ skip_check);
+ }
+
+ /// \brief Lookup file via path in local workspace root and upload.
+ /// \param api The endpoint used for uploading
+ /// \param repo The global repository name, the artifact belongs to
+ /// \param file_path The path of the file to be read
+ /// \returns The computed object info on success
+ [[nodiscard]] static auto UploadFile(
+ gsl::not_null<IExecutionApi*> const& api,
+ std::string const& repo,
+ std::filesystem::path const& file_path) noexcept
+ -> std::optional<Artifact::ObjectInfo> {
+ auto const* ws_root = RepositoryConfig::Instance().WorkspaceRoot(repo);
+ if (ws_root == nullptr) {
+ return std::nullopt;
+ }
+ auto const object_type = ws_root->FileType(file_path);
+ if (not object_type) {
+ return std::nullopt;
+ }
+ auto content = ws_root->ReadFile(file_path);
+ if (not content.has_value()) {
+ return std::nullopt;
+ }
+ auto digest = ArtifactDigest{ComputeHash(*content), content->size()};
+ if (not api->Upload(
+ BlobContainer{{BazelBlob{digest, std::move(*content)}}})) {
+ return std::nullopt;
+ }
+ return Artifact::ObjectInfo{std::move(digest), *object_type};
+ }
+
+ /// \brief Add digests and object type to artifact nodes for all outputs of
+ /// the action that was run
+ void static SaveObjectInfo(
+ IExecutionResponse::ArtifactInfos const& artifacts,
+ gsl::not_null<DependencyGraph::ActionNode const*> const& action,
+ bool fail_artifacts) noexcept {
+ for (auto const& [name, node] : action->OutputFiles()) {
+ node->Content().SetObjectInfo(artifacts.at(name), fail_artifacts);
+ }
+ for (auto const& [name, node] : action->OutputDirs()) {
+ node->Content().SetObjectInfo(artifacts.at(name), fail_artifacts);
+ }
+ }
+
+ /// \brief Create root tree digest for input artifacts.
+ /// \param api The endpoint required for uploading
+ /// \param artifacts The artifacts to create the root tree digest from
+ [[nodiscard]] static auto CreateRootDigest(
+ gsl::not_null<IExecutionApi*> const& api,
+ std::vector<DependencyGraph::NamedArtifactNodePtr> const&
+ artifacts) noexcept -> std::optional<ArtifactDigest> {
+ if (artifacts.size() == 1 and
+ (artifacts.at(0).path == "." or artifacts.at(0).path.empty())) {
+ auto const& info = artifacts.at(0).node->Content().Info();
+ if (info and IsTreeObject(info->type)) {
+ // Artifact list contains single tree with path "." or "". Reuse
+ // the existing tree artifact by returning its digest.
+ return info->digest;
+ }
+ }
+ return api->UploadTree(artifacts);
+ }
+ /// \brief Check that all outputs expected from the action description
+ /// are present in the artifacts map
+ [[nodiscard]] static auto CheckOutputsExist(
+ IExecutionResponse::ArtifactInfos const& artifacts,
+ std::vector<Action::LocalPath> const& outputs) noexcept -> bool {
+ for (auto const& output : outputs) {
+ if (not artifacts.contains(output)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /// \brief Parse response and write object info to DAG's artifact nodes.
+ /// \returns false on non-zero exit code or if output artifacts are missing
+ [[nodiscard]] static auto ParseResponse(
+ Logger const& logger,
+ IExecutionResponse::Ptr const& response,
+ gsl::not_null<DependencyGraph::ActionNode const*> const& action)
+ -> bool {
+ logger.Emit(LogLevel::Trace, "finished execution");
+
+ if (!response) {
+ logger.Emit(LogLevel::Trace, "response is empty");
+ return false;
+ }
+
+ if (response->IsCached()) {
+ logger.Emit(LogLevel::Trace, " - served from cache");
+ Statistics::Instance().IncrementActionsCachedCounter();
+ }
+
+ PrintInfo(logger, action->Command(), response);
+ bool should_fail_outputs = false;
+ for (auto const& [local_path, node] : action->Dependencies()) {
+ should_fail_outputs |= node->Content().Info()->failed;
+ }
+ if (response->ExitCode() != 0) {
+ if (action->MayFail()) {
+ logger.Emit(LogLevel::Warning,
+ "{} (exit code {})",
+ *(action->MayFail()),
+ response->ExitCode());
+ should_fail_outputs = true;
+ }
+ else {
+ logger.Emit(LogLevel::Error,
+ "action returned non-zero exit code {}",
+ response->ExitCode());
+ return false;
+ }
+ }
+
+ auto artifacts = response->Artifacts();
+ auto output_files = action->OutputFilePaths();
+ auto output_dirs = action->OutputDirPaths();
+
+ if (artifacts.empty() or
+ not CheckOutputsExist(artifacts, output_files) or
+ not CheckOutputsExist(artifacts, output_dirs)) {
+ logger.Emit(LogLevel::Error, [&] {
+ std::string message{
+ "action executed with missing outputs.\n"
+ " Action outputs should be the following artifacts:"};
+ for (auto const& output : output_files) {
+ message += "\n - " + output;
+ }
+ return message;
+ });
+ return false;
+ }
+
+ SaveObjectInfo(artifacts, action, should_fail_outputs);
+
+ return true;
+ }
+
+ /// \brief Write out if response is empty and otherwise, write out
+ /// standard error/output if they are present
+ void static PrintInfo(Logger const& logger,
+ std::vector<std::string> const& command,
+ IExecutionResponse::Ptr const& response) noexcept {
+ if (!response) {
+ logger.Emit(LogLevel::Error, "response is empty");
+ return;
+ }
+ auto has_err = response->HasStdErr();
+ auto has_out = response->HasStdOut();
+ if (has_err or has_out) {
+ logger.Emit(LogLevel::Info, [&] {
+ auto message = std::string{has_err and has_out
+ ? "Output and error"
+ : has_out ? "Output" : "Error"} +
+ " of command: ";
+ message += nlohmann::json{command}.dump();
+ if (response->HasStdOut()) {
+ message += "\n" + response->StdOut();
+ }
+ if (response->HasStdErr()) {
+ message += "\n" + response->StdErr();
+ }
+ return message;
+ });
+ }
+ }
+};
+
+/// \brief Executor for using concrete Execution API.
+class Executor {
+ using Impl = ExecutorImpl;
+ using CF = IExecutionAction::CacheFlag;
+
+ public:
+ explicit Executor(IExecutionApi* api,
+ std::map<std::string, std::string> properties)
+ : api_{api}, properties_{std::move(properties)} {}
+
+ /// \brief Run an action in a blocking manner
+ /// This method must be thread-safe as it could be called in parallel
+ /// \param[in] action The action to execute.
+ /// \returns True if execution was successful, false otherwise
+ [[nodiscard]] auto Process(
+ gsl::not_null<DependencyGraph::ActionNode const*> const& action)
+ const noexcept -> bool {
+ Logger logger("action:" + action->Content().Id());
+
+ auto const response = Impl::ExecuteAction(
+ logger,
+ action,
+ api_,
+ properties_,
+ action->NoCache() ? CF::DoNotCacheOutput : CF::CacheOutput);
+
+ // check response and save digests of results
+ return not response or Impl::ParseResponse(logger, *response, action);
+ }
+
+ /// \brief Check artifact is available to the CAS or upload it.
+ /// \param[in] artifact The artifact to process.
+ /// \returns True if artifact is available or uploaded, false otherwise
+ [[nodiscard]] auto Process(
+ gsl::not_null<DependencyGraph::ArtifactNode const*> const& artifact)
+ const noexcept -> bool {
+ return Impl::VerifyOrUploadArtifact(artifact, api_);
+ }
+
+ private:
+ gsl::not_null<IExecutionApi*> api_;
+ std::map<std::string, std::string> properties_;
+};
+
+/// \brief Rebuilder for running and comparing actions of two API endpoints.
+class Rebuilder {
+ using Impl = ExecutorImpl;
+ using CF = IExecutionAction::CacheFlag;
+
+ public:
+ /// \brief Create rebuilder for action comparision of two endpoints.
+ /// \param api Rebuild endpoint, executes without action cache.
+ /// \param api_cached Reference endpoint, serves everything from cache.
+ /// \param properties Platform properties for execution.
+ Rebuilder(IExecutionApi* api,
+ IExecutionApi* api_cached,
+ std::map<std::string, std::string> properties)
+ : api_{api},
+ api_cached_{api_cached},
+ properties_{std::move(properties)} {}
+
+ [[nodiscard]] auto Process(
+ gsl::not_null<DependencyGraph::ActionNode const*> const& action)
+ const noexcept -> bool {
+ auto const& action_id = action->Content().Id();
+ Logger logger("rebuild:" + action_id);
+ auto response = Impl::ExecuteAction(
+ logger, action, api_, properties_, CF::PretendCached);
+
+ if (not response) {
+ return true; // action without response (e.g., tree action)
+ }
+
+ Logger logger_cached("cached:" + action_id);
+ auto response_cached = Impl::ExecuteAction(
+ logger_cached, action, api_cached_, properties_, CF::FromCacheOnly);
+
+ if (not response_cached) {
+ logger_cached.Emit(LogLevel::Error,
+ "expected regular action with response");
+ return false;
+ }
+
+ DetectFlakyAction(*response, *response_cached, action->Content());
+ return Impl::ParseResponse(logger, *response, action);
+ }
+
+ [[nodiscard]] auto Process(
+ gsl::not_null<DependencyGraph::ArtifactNode const*> const& artifact)
+ const noexcept -> bool {
+ return Impl::VerifyOrUploadArtifact(artifact, api_);
+ }
+
+ [[nodiscard]] auto DumpFlakyActions() const noexcept -> nlohmann::json {
+ std::unique_lock lock{m_};
+ auto actions = nlohmann::json::object();
+ for (auto const& [action_id, outputs] : flaky_actions_) {
+ for (auto const& [path, infos] : outputs) {
+ actions[action_id][path]["rebuilt"] = infos.first.ToJson();
+ actions[action_id][path]["cached"] = infos.second.ToJson();
+ }
+ }
+ return {{"flaky actions", actions}, {"cache misses", cache_misses_}};
+ }
+
+ private:
+ gsl::not_null<IExecutionApi*> api_;
+ gsl::not_null<IExecutionApi*> api_cached_;
+ std::map<std::string, std::string> properties_;
+ mutable std::mutex m_;
+ mutable std::vector<std::string> cache_misses_{};
+ mutable std::unordered_map<
+ std::string,
+ std::unordered_map<
+ std::string,
+ std::pair<Artifact::ObjectInfo, Artifact::ObjectInfo>>>
+ flaky_actions_{};
+
+ void DetectFlakyAction(IExecutionResponse::Ptr const& response,
+ IExecutionResponse::Ptr const& response_cached,
+ Action const& action) const noexcept {
+ if (response and response_cached and
+ response_cached->ActionDigest() == response->ActionDigest()) {
+ Statistics::Instance().IncrementRebuiltActionComparedCounter();
+ auto artifacts = response->Artifacts();
+ auto artifacts_cached = response_cached->Artifacts();
+ std::ostringstream msg{};
+ for (auto const& [path, info] : artifacts) {
+ auto const& info_cached = artifacts_cached[path];
+ if (info != info_cached) {
+ RecordFlakyAction(&msg, action, path, info, info_cached);
+ }
+ }
+ if (msg.tellp() > 0) {
+ Statistics::Instance().IncrementActionsFlakyCounter();
+ bool tainted = action.MayFail() or action.NoCache();
+ if (tainted) {
+ Statistics::Instance()
+ .IncrementActionsFlakyTaintedCounter();
+ }
+ Logger::Log(tainted ? LogLevel::Debug : LogLevel::Warning,
+ "{}",
+ msg.str());
+ }
+ }
+ else {
+ Statistics::Instance().IncrementRebuiltActionMissingCounter();
+ std::unique_lock lock{m_};
+ cache_misses_.emplace_back(action.Id());
+ }
+ }
+
+ void RecordFlakyAction(gsl::not_null<std::ostringstream*> const& msg,
+ Action const& action,
+ std::string const& path,
+ Artifact::ObjectInfo const& rebuilt,
+ Artifact::ObjectInfo const& cached) const noexcept {
+ auto const& action_id = action.Id();
+ if (msg->tellp() <= 0) {
+ bool tainted = action.MayFail() or action.NoCache();
+ static constexpr auto kMaxCmdChars = 69; // 80 - (prefix + suffix)
+ auto cmd = GetCmdString(action);
+ (*msg) << "Found flaky " << (tainted ? "tainted " : "")
+ << "action:" << std::endl
+ << " - id: " << action_id << std::endl
+ << " - cmd: " << cmd.substr(0, kMaxCmdChars)
+ << (cmd.length() > kMaxCmdChars ? "..." : "") << std::endl;
+ }
+ (*msg) << " - output '" << path << "' differs:" << std::endl
+ << " - " << rebuilt.ToString() << " (rebuilt)" << std::endl
+ << " - " << cached.ToString() << " (cached)" << std::endl;
+
+ std::unique_lock lock{m_};
+ auto& object_map = flaky_actions_[action_id];
+ try {
+ object_map.emplace(path, std::make_pair(rebuilt, cached));
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "recoding flaky action failed with: {}",
+ ex.what());
+ }
+ }
+
+ static auto GetCmdString(Action const& action) noexcept -> std::string {
+ try {
+ return nlohmann::json(action.Command()).dump();
+ } catch (std::exception const& ex) {
+ return fmt::format("<exception: {}>", ex.what());
+ }
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_EXECUTOR_EXECUTOR_HPP
diff --git a/src/buildtool/execution_engine/traverser/TARGETS b/src/buildtool/execution_engine/traverser/TARGETS
new file mode 100644
index 00000000..eb306b6e
--- /dev/null
+++ b/src/buildtool/execution_engine/traverser/TARGETS
@@ -0,0 +1,14 @@
+{ "traverser":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["traverser"]
+ , "hdrs": ["traverser.hpp"]
+ , "deps":
+ [ ["src/buildtool/execution_engine/dag", "dag"]
+ , ["src/buildtool/multithreading", "task_system"]
+ , ["src/buildtool/logging", "logging"]
+ , ["src/utils/cpp", "concepts"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "execution_engine", "traverser"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/execution_engine/traverser/traverser.hpp b/src/buildtool/execution_engine/traverser/traverser.hpp
new file mode 100644
index 00000000..ea44f30c
--- /dev/null
+++ b/src/buildtool/execution_engine/traverser/traverser.hpp
@@ -0,0 +1,187 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TRAVERSER_TRAVERSER_HPP
+#define INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TRAVERSER_TRAVERSER_HPP
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/execution_engine/dag/dag.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/multithreading/task_system.hpp"
+#include "src/utils/cpp/concepts.hpp"
+
+/// \brief Concept required for Runners used by the Traverser.
+template <class T>
+concept Runnable = requires(T const r,
+ DependencyGraph::ActionNode const* action,
+ DependencyGraph::ArtifactNode const* artifact) {
+ { r.Process(action) }
+ ->same_as<bool>;
+ { r.Process(artifact) }
+ ->same_as<bool>;
+};
+
+/// \brief Class to traverse the dependency graph executing necessary actions
+/// \tparam Executor Type of the executor
+// Traversal of the graph and execution of actions are concurrent, using
+/// the //src/buildtool/execution_engine/task_system.
+/// Graph remains constant and the only parts of the nodes that are modified are
+/// their traversal state
+template <Runnable Executor>
+class Traverser {
+ public:
+ Traverser(Executor const& r, DependencyGraph const& graph, std::size_t jobs)
+ : runner_{r}, graph_{graph}, tasker_{jobs} {}
+ Traverser() = delete;
+ Traverser(Traverser const&) = delete;
+ Traverser(Traverser&&) = delete;
+ auto operator=(Traverser const&) -> Traverser& = delete;
+ auto operator=(Traverser &&) -> Traverser& = delete;
+ ~Traverser() = default;
+
+ // Traverse the whole graph
+ [[nodiscard]] auto Traverse() noexcept -> bool {
+ auto const& ids = graph_.ArtifactIdentifiers();
+ return Traverse(ids);
+ };
+
+ // Traverse starting by the artifacts with the given identifiers, avoiding
+ // executing actions that are not strictly needed to build the given
+ // artifacs
+ [[nodiscard]] auto Traverse(
+ std::unordered_set<ArtifactIdentifier> const& target_ids) noexcept
+ -> bool;
+
+ private:
+ Executor const& runner_{};
+ DependencyGraph const& graph_;
+ TaskSystem tasker_{}; // THIS SHOULD BE THE LAST MEMBER VARIABLE
+
+ // Visits discover nodes and queue visits to their children nodes.
+ void Visit(gsl::not_null<DependencyGraph::ArtifactNode const*>
+ artifact_node) noexcept;
+ void Visit(
+ gsl::not_null<DependencyGraph::ActionNode const*> action_node) noexcept;
+
+ // Notify all actions that have it as a dependency that it is available and
+ // queue execution of those that become ready (that were only waiting for
+ // this artifact)
+ void NotifyAvailable(
+ gsl::not_null<DependencyGraph::ArtifactNode const*> const&
+ artifact_node) noexcept;
+
+ // Calls NotifyAvailable on all the action's outputs
+ void NotifyAvailable(
+ gsl::not_null<DependencyGraph::ActionNode const*> const&
+ action_node) noexcept;
+
+ // Visit to nodes are queued only once
+ template <typename NodeTypePtr>
+ void QueueVisit(NodeTypePtr node) noexcept {
+ // in case the node was already discovered, there is no need to queue
+ // the visit
+ if (node->TraversalState()->GetAndMarkDiscovered()) {
+ return;
+ }
+ tasker_.QueueTask([this, node]() noexcept { Visit(node); });
+ }
+
+ // Queue task to process the node by the executor after making sure that the
+ // node is required and that it was not yet queued to be processed. The task
+ // queued will call notify that the node is available in case processing it
+ // was successful
+ template <typename NodeTypePtr>
+ void QueueProcessing(NodeTypePtr node) noexcept {
+ if (not node->TraversalState()->IsRequired() or
+ node->TraversalState()->GetAndMarkQueuedToBeProcessed()) {
+ return;
+ }
+
+ auto process_node = [this, node]() {
+ if (runner_.Process(node)) {
+ NotifyAvailable(node);
+ }
+ else {
+ Logger::Log(LogLevel::Error, "Build failed.");
+ std::exit(EXIT_FAILURE);
+ }
+ };
+ tasker_.QueueTask(process_node);
+ }
+};
+
+template <Runnable Executor>
+auto Traverser<Executor>::Traverse(
+ std::unordered_set<ArtifactIdentifier> const& target_ids) noexcept -> bool {
+ for (auto artifact_id : target_ids) {
+ auto const* artifact_node = graph_.ArtifactNodeWithId(artifact_id);
+ if (artifact_node != nullptr) {
+ QueueVisit(artifact_node);
+ }
+ else {
+ Logger::Log(
+ LogLevel::Error,
+ "artifact with id {} can not be found in dependency graph.",
+ artifact_id);
+ return false;
+ }
+ }
+ return true;
+}
+
+template <Runnable Executor>
+void Traverser<Executor>::Visit(
+ gsl::not_null<DependencyGraph::ArtifactNode const*>
+ artifact_node) noexcept {
+ artifact_node->TraversalState()->MarkRequired();
+ // Visits are queued only once per artifact node, but it could be that the
+ // builder action had multiple outputs and was queued and executed through
+ // the visit to another of the outputs, in which case the current artifact
+ // would be available and there is nothing else to do
+ if (artifact_node->TraversalState()->IsAvailable()) {
+ return;
+ }
+
+ if (artifact_node->HasBuilderAction()) {
+ QueueVisit(gsl::not_null<DependencyGraph::ActionNode const*>(
+ artifact_node->BuilderActionNode()));
+ }
+ else {
+ QueueProcessing(artifact_node);
+ }
+}
+
+template <Runnable Executor>
+void Traverser<Executor>::Visit(
+ gsl::not_null<DependencyGraph::ActionNode const*> action_node) noexcept {
+ action_node->TraversalState()->MarkRequired();
+ for (auto const& dep : action_node->Children()) {
+ if (not dep->TraversalState()->IsAvailable()) {
+ QueueVisit(dep);
+ }
+ }
+
+ if (action_node->TraversalState()->IsReady()) {
+ QueueProcessing(action_node);
+ }
+}
+
+template <Runnable Executor>
+void Traverser<Executor>::NotifyAvailable(
+ gsl::not_null<DependencyGraph::ArtifactNode const*> const&
+ artifact_node) noexcept {
+ artifact_node->TraversalState()->MakeAvailable();
+ for (auto const& action_node : artifact_node->Parents()) {
+ if (action_node->TraversalState()->NotifyAvailableDepAndCheckReady()) {
+ QueueProcessing(action_node);
+ }
+ }
+}
+
+template <Runnable Executor>
+void Traverser<Executor>::NotifyAvailable(
+ gsl::not_null<DependencyGraph::ActionNode const*> const&
+ action_node) noexcept {
+ for (auto const& output : action_node->Parents()) {
+ NotifyAvailable(output);
+ }
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_EXECUTION_ENGINE_TRAVERSER_TRAVERSER_HPP
diff --git a/src/buildtool/file_system/TARGETS b/src/buildtool/file_system/TARGETS
new file mode 100644
index 00000000..478b5903
--- /dev/null
+++ b/src/buildtool/file_system/TARGETS
@@ -0,0 +1,79 @@
+{ "object_type":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["object_type"]
+ , "hdrs": ["object_type.hpp"]
+ , "stage": ["src", "buildtool", "file_system"]
+ }
+, "file_system_manager":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["file_system_manager"]
+ , "hdrs": ["file_system_manager.hpp"]
+ , "deps":
+ [ "object_type"
+ , ["src/buildtool/logging", "logging"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "file_system"]
+ }
+, "system_command":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["system_command"]
+ , "hdrs": ["system_command.hpp"]
+ , "deps":
+ [ "file_system_manager"
+ , ["src/buildtool/logging", "logging"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "file_system"]
+ }
+, "jsonfs":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["jsonfs"]
+ , "hdrs": ["jsonfs.hpp"]
+ , "deps": ["object_type", "file_system_manager", ["src/utils/cpp", "json"]]
+ , "stage": ["src", "buildtool", "file_system"]
+ }
+, "git_cas":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["git_cas"]
+ , "hdrs": ["git_cas.hpp"]
+ , "srcs": ["git_cas.cpp"]
+ , "deps":
+ [ "object_type"
+ , "file_system_manager"
+ , ["src/buildtool/logging", "logging"]
+ , ["src/utils/cpp", "hex_string"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ , ["", "libgit2"]
+ ]
+ , "stage": ["src", "buildtool", "file_system"]
+ }
+, "git_tree":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["git_tree"]
+ , "hdrs": ["git_tree.hpp"]
+ , "srcs": ["git_tree.cpp"]
+ , "deps":
+ [ "git_cas"
+ , "object_type"
+ , "file_system_manager"
+ , ["src/buildtool/logging", "logging"]
+ , ["src/utils/cpp", "atomic"]
+ , ["src/utils/cpp", "hex_string"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "file_system"]
+ }
+, "file_root":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["file_root"]
+ , "hdrs": ["file_root.hpp"]
+ , "deps":
+ [ "git_tree"
+ , "file_system_manager"
+ , ["src/buildtool/common", "artifact_description"]
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "file_system"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/file_system/file_root.hpp b/src/buildtool/file_system/file_root.hpp
new file mode 100644
index 00000000..1df40588
--- /dev/null
+++ b/src/buildtool/file_system/file_root.hpp
@@ -0,0 +1,239 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP
+#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP
+
+#include <filesystem>
+#include <memory>
+#include <string>
+#include <unordered_set>
+#include <variant>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/git_tree.hpp"
+
+class FileRoot {
+ using fs_root_t = std::filesystem::path;
+ struct git_root_t {
+ gsl::not_null<GitCASPtr> cas;
+ gsl::not_null<GitTreePtr> tree;
+ };
+ using root_t = std::variant<fs_root_t, git_root_t>;
+
+ public:
+ class DirectoryEntries {
+ using names_t = std::unordered_set<std::string>;
+ using tree_t = gsl::not_null<GitTree const*>;
+ using entries_t = std::variant<std::monostate, names_t, tree_t>;
+
+ public:
+ DirectoryEntries() noexcept = default;
+ explicit DirectoryEntries(names_t names) noexcept
+ : data_{std::move(names)} {}
+ explicit DirectoryEntries(tree_t git_tree) noexcept
+ : data_{std::move(git_tree)} {}
+ [[nodiscard]] auto Contains(std::string const& name) const noexcept
+ -> bool {
+ if (std::holds_alternative<tree_t>(data_)) {
+ return static_cast<bool>(
+ std::get<tree_t>(data_)->LookupEntryByName(name));
+ }
+ if (std::holds_alternative<names_t>(data_)) {
+ return std::get<names_t>(data_).contains(name);
+ }
+ return false;
+ }
+ [[nodiscard]] auto Empty() const noexcept -> bool {
+ if (std::holds_alternative<tree_t>(data_)) {
+ try {
+ auto const& tree = std::get<tree_t>(data_);
+ return tree->begin() == tree->end();
+ } catch (...) {
+ return false;
+ }
+ }
+ if (std::holds_alternative<names_t>(data_)) {
+ return std::get<names_t>(data_).empty();
+ }
+ return true;
+ }
+
+ private:
+ entries_t data_{};
+ };
+
+ FileRoot() noexcept = default;
+ explicit FileRoot(std::filesystem::path root) noexcept
+ : root_{std::move(root)} {}
+ FileRoot(gsl::not_null<GitCASPtr> cas,
+ gsl::not_null<GitTreePtr> tree) noexcept
+ : root_{git_root_t{std::move(cas), std::move(tree)}} {}
+
+ [[nodiscard]] static auto FromGit(std::filesystem::path const& repo_path,
+ std::string const& git_tree_id) noexcept
+ -> std::optional<FileRoot> {
+ if (auto cas = GitCAS::Open(repo_path)) {
+ if (auto tree = GitTree::Read(cas, git_tree_id)) {
+ try {
+ return FileRoot{
+ cas, std::make_shared<GitTree const>(std::move(*tree))};
+ } catch (...) {
+ }
+ }
+ }
+ return std::nullopt;
+ }
+
+ // Indicates that subsequent calls to `Exists()`, `IsFile()`,
+ // `IsDirectory()`, and `FileType()` on contents of the same directory will
+ // be served without any additional file system lookups.
+ [[nodiscard]] auto HasFastDirectoryLookup() const noexcept -> bool {
+ return std::holds_alternative<git_root_t>(root_);
+ }
+
+ [[nodiscard]] auto Exists(std::filesystem::path const& path) const noexcept
+ -> bool {
+ if (std::holds_alternative<git_root_t>(root_)) {
+ if (path == ".") {
+ return true;
+ }
+ return static_cast<bool>(
+ std::get<git_root_t>(root_).tree->LookupEntryByPath(path));
+ }
+ return FileSystemManager::Exists(std::get<fs_root_t>(root_) / path);
+ }
+
+ [[nodiscard]] auto IsFile(
+ std::filesystem::path const& file_path) const noexcept -> bool {
+ if (std::holds_alternative<git_root_t>(root_)) {
+ if (auto entry =
+ std::get<git_root_t>(root_).tree->LookupEntryByPath(
+ file_path)) {
+ return entry->IsBlob();
+ }
+ return false;
+ }
+ return FileSystemManager::IsFile(std::get<fs_root_t>(root_) /
+ file_path);
+ }
+
+ [[nodiscard]] auto IsDirectory(
+ std::filesystem::path const& dir_path) const noexcept -> bool {
+ if (std::holds_alternative<git_root_t>(root_)) {
+ if (dir_path == ".") {
+ return true;
+ }
+ if (auto entry =
+ std::get<git_root_t>(root_).tree->LookupEntryByPath(
+ dir_path)) {
+ return entry->IsTree();
+ }
+ return false;
+ }
+ return FileSystemManager::IsDirectory(std::get<fs_root_t>(root_) /
+ dir_path);
+ }
+
+ [[nodiscard]] auto ReadFile(std::filesystem::path const& file_path)
+ const noexcept -> std::optional<std::string> {
+ if (std::holds_alternative<git_root_t>(root_)) {
+ if (auto entry =
+ std::get<git_root_t>(root_).tree->LookupEntryByPath(
+ file_path)) {
+ return entry->Blob();
+ }
+ return std::nullopt;
+ }
+ return FileSystemManager::ReadFile(std::get<fs_root_t>(root_) /
+ file_path);
+ }
+
+ [[nodiscard]] auto ReadDirectory(std::filesystem::path const& dir_path)
+ const noexcept -> DirectoryEntries {
+ try {
+ if (std::holds_alternative<git_root_t>(root_)) {
+ auto const& tree = std::get<git_root_t>(root_).tree;
+ if (dir_path == ".") {
+ return DirectoryEntries{&(*tree)};
+ }
+ if (auto entry = tree->LookupEntryByPath(dir_path)) {
+ if (auto const& found_tree = entry->Tree()) {
+ return DirectoryEntries{&(*found_tree)};
+ }
+ }
+ }
+ else {
+ std::unordered_set<std::string> names{};
+ if (FileSystemManager::ReadDirectory(
+ std::get<fs_root_t>(root_) / dir_path,
+ [&names](auto name, auto /*type*/) {
+ names.emplace(name.string());
+ return true;
+ })) {
+ return DirectoryEntries{std::move(names)};
+ }
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "reading directory {} failed with:\n{}",
+ dir_path.string(),
+ ex.what());
+ }
+ return {};
+ }
+
+ [[nodiscard]] auto FileType(std::filesystem::path const& file_path)
+ const noexcept -> std::optional<ObjectType> {
+ if (std::holds_alternative<git_root_t>(root_)) {
+ if (auto entry =
+ std::get<git_root_t>(root_).tree->LookupEntryByPath(
+ file_path)) {
+ if (entry->IsBlob()) {
+ return entry->Type();
+ }
+ }
+ return std::nullopt;
+ }
+ auto type =
+ FileSystemManager::Type(std::get<fs_root_t>(root_) / file_path);
+ if (type and IsFileObject(*type)) {
+ return type;
+ }
+ return std::nullopt;
+ }
+
+ [[nodiscard]] auto ReadBlob(std::string const& blob_id) const noexcept
+ -> std::optional<std::string> {
+ if (std::holds_alternative<git_root_t>(root_)) {
+ return std::get<git_root_t>(root_).cas->ReadObject(
+ blob_id, /*is_hex_id=*/true);
+ }
+ return std::nullopt;
+ }
+
+ // Create LOCAL or KNOWN artifact. Does not check existence for LOCAL.
+ [[nodiscard]] auto ToArtifactDescription(
+ std::filesystem::path const& file_path,
+ std::string const& repository) const noexcept
+ -> std::optional<ArtifactDescription> {
+ if (std::holds_alternative<git_root_t>(root_)) {
+ if (auto entry =
+ std::get<git_root_t>(root_).tree->LookupEntryByPath(
+ file_path)) {
+ if (entry->IsBlob()) {
+ return ArtifactDescription{
+ ArtifactDigest{entry->Hash(), *entry->Size()},
+ entry->Type(),
+ repository};
+ }
+ }
+ return std::nullopt;
+ }
+ return ArtifactDescription{file_path, repository};
+ }
+
+ private:
+ root_t root_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_ROOT_HPP
diff --git a/src/buildtool/file_system/file_system_manager.hpp b/src/buildtool/file_system/file_system_manager.hpp
new file mode 100644
index 00000000..75902890
--- /dev/null
+++ b/src/buildtool/file_system/file_system_manager.hpp
@@ -0,0 +1,565 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_SYSTEM_MANAGER_HPP
+#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_SYSTEM_MANAGER_HPP
+
+#include <cstdio> // for std::fopen
+#include <exception>
+#include <filesystem>
+#include <fstream>
+#include <optional>
+
+#ifdef __unix__
+#include <unistd.h>
+#endif
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// \brief Implements primitive file system functionality.
+/// Catches all exceptions for use with exception-free callers.
+class FileSystemManager {
+ public:
+ using ReadDirEntryFunc =
+ std::function<bool(std::filesystem::path const&, ObjectType type)>;
+
+ class DirectoryAnchor {
+ friend class FileSystemManager;
+
+ public:
+ DirectoryAnchor(DirectoryAnchor const&) = delete;
+ auto operator=(DirectoryAnchor const&) -> DirectoryAnchor& = delete;
+ auto operator=(DirectoryAnchor &&) -> DirectoryAnchor& = delete;
+ ~DirectoryAnchor() noexcept {
+ if (!kRestorePath.empty()) {
+ try {
+ std::filesystem::current_path(kRestorePath);
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ }
+ }
+ }
+ [[nodiscard]] auto GetRestorePath() const noexcept
+ -> std::filesystem::path const& {
+ return kRestorePath;
+ }
+
+ private:
+ std::filesystem::path const kRestorePath{};
+
+ DirectoryAnchor()
+ : kRestorePath{FileSystemManager::GetCurrentDirectory()} {}
+ DirectoryAnchor(DirectoryAnchor&&) = default;
+ };
+
+ [[nodiscard]] static auto GetCurrentDirectory() noexcept
+ -> std::filesystem::path {
+ try {
+ return std::filesystem::current_path();
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return std::filesystem::path{};
+ }
+ }
+
+ [[nodiscard]] static auto ChangeDirectory(
+ std::filesystem::path const& dir) noexcept -> DirectoryAnchor {
+ DirectoryAnchor anchor{};
+ try {
+ std::filesystem::current_path(dir);
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "changing directory to {} from anchor {}:\n{}",
+ dir.string(),
+ anchor.GetRestorePath().string(),
+ e.what());
+ }
+ return anchor;
+ }
+
+ /// \brief Returns true if the directory was created or existed before.
+ [[nodiscard]] static auto CreateDirectory(
+ std::filesystem::path const& dir) noexcept -> bool {
+ return CreateDirectoryImpl(dir) != CreationStatus::Failed;
+ }
+
+ /// \brief Returns true if the directory was created by this call.
+ [[nodiscard]] static auto CreateDirectoryExclusive(
+ std::filesystem::path const& dir) noexcept -> bool {
+ return CreateDirectoryImpl(dir) == CreationStatus::Created;
+ }
+
+ /// \brief Returns true if the file was created or existed before.
+ [[nodiscard]] static auto CreateFile(
+ std::filesystem::path const& file) noexcept -> bool {
+ return CreateFileImpl(file) != CreationStatus::Failed;
+ }
+
+ /// \brief Returns true if the file was created by this call.
+ [[nodiscard]] static auto CreateFileExclusive(
+ std::filesystem::path const& file) noexcept -> bool {
+ return CreateFileImpl(file) == CreationStatus::Created;
+ }
+
+ [[nodiscard]] static auto CreateFileHardlink(
+ std::filesystem::path const& file_path,
+ std::filesystem::path const& link_path) noexcept -> bool {
+ try {
+ std::filesystem::create_hard_link(file_path, link_path);
+ return std::filesystem::is_regular_file(link_path);
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "hard linking {} to {}\n{}",
+ file_path.string(),
+ link_path.string(),
+ e.what());
+ return false;
+ }
+ }
+
+ [[nodiscard]] static auto Rename(std::filesystem::path const& src,
+ std::filesystem::path const& dst,
+ bool no_clobber = false) noexcept -> bool {
+ if (no_clobber) {
+#ifdef __unix__
+ return link(src.c_str(), dst.c_str()) == 0 and
+ unlink(src.c_str()) == 0;
+#else
+#error "Non-unix is not supported yet"
+#endif
+ }
+ try {
+ std::filesystem::rename(src, dst);
+ return true;
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return false;
+ }
+ }
+
+ [[nodiscard]] static auto CopyFile(
+ std::filesystem::path const& src,
+ std::filesystem::path const& dst,
+ std::filesystem::copy_options opt =
+ std::filesystem::copy_options::overwrite_existing) noexcept
+ -> bool {
+ try {
+ return std::filesystem::copy_file(src, dst, opt);
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "copying file from {} to {}:\n{}",
+ src.string(),
+ dst.string(),
+ e.what());
+ return false;
+ }
+ }
+
+ template <ObjectType kType>
+ requires(IsFileObject(kType)) [[nodiscard]] static auto CopyFileAs(
+ std::filesystem::path const& src,
+ std::filesystem::path const& dst,
+ std::filesystem::copy_options opt =
+ std::filesystem::copy_options::overwrite_existing) noexcept
+ -> bool {
+ return CopyFile(src, dst, opt) and
+ SetFilePermissions(dst, IsExecutableObject(kType));
+ }
+
+ [[nodiscard]] static auto CopyFileAs(
+ std::filesystem::path const& src,
+ std::filesystem::path const& dst,
+ ObjectType type,
+ std::filesystem::copy_options opt =
+ std::filesystem::copy_options::overwrite_existing) noexcept
+ -> bool {
+ switch (type) {
+ case ObjectType::File:
+ return CopyFileAs<ObjectType::File>(src, dst, opt);
+ case ObjectType::Executable:
+ return CopyFileAs<ObjectType::Executable>(src, dst, opt);
+ case ObjectType::Tree:
+ break;
+ }
+
+ return false;
+ }
+
+ [[nodiscard]] static auto RemoveFile(
+ std::filesystem::path const& file) noexcept -> bool {
+ try {
+ if (!std::filesystem::exists(file)) {
+ return true;
+ }
+ if (!IsFile(file)) {
+ return false;
+ }
+ return std::filesystem::remove(file);
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "removing file from {}:\n{}",
+ file.string(),
+ e.what());
+ return false;
+ }
+ }
+
+ [[nodiscard]] static auto RemoveDirectory(std::filesystem::path const& dir,
+ bool recursively = false) noexcept
+ -> bool {
+ try {
+ if (!std::filesystem::exists(dir)) {
+ return true;
+ }
+ if (recursively) {
+ return (std::filesystem::remove_all(dir) !=
+ static_cast<uintmax_t>(-1));
+ }
+ return std::filesystem::remove(dir);
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "removing directory {}:\n{}",
+ dir.string(),
+ e.what());
+ return false;
+ }
+ }
+
+ [[nodiscard]] static auto ResolveSymlinks(
+ gsl::not_null<std::filesystem::path*> const& path) noexcept -> bool {
+ try {
+ while (std::filesystem::is_symlink(*path)) {
+ *path = std::filesystem::read_symlink(*path);
+ }
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return false;
+ }
+
+ return true;
+ }
+
+ [[nodiscard]] static auto Exists(std::filesystem::path const& path) noexcept
+ -> bool {
+ try {
+ return std::filesystem::exists(path);
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "checking for existence of path{}:\n{}",
+ path.string(),
+ e.what());
+ return false;
+ }
+
+ return true;
+ }
+
+ [[nodiscard]] static auto IsFile(std::filesystem::path const& file) noexcept
+ -> bool {
+ try {
+ if (!std::filesystem::is_regular_file(file)) {
+ return false;
+ }
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "checking if path {} corresponds to a file:\n{}",
+ file.string(),
+ e.what());
+ return false;
+ }
+
+ return true;
+ }
+
+ [[nodiscard]] static auto IsDirectory(
+ std::filesystem::path const& dir) noexcept -> bool {
+ try {
+ return std::filesystem::is_directory(dir);
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "checking if path {} corresponds to a directory:\n{}",
+ dir.string(),
+ e.what());
+ return false;
+ }
+
+ return true;
+ }
+
+ /// \brief Checks whether a path corresponds to an executable or not.
+ /// \param[in] path Path to check
+ /// \param[in] is_file_known (Optional) If true, we assume that the path
+ /// corresponds to a file, if false, we check if it's a file or not first.
+ /// Default value is false
+ /// \returns true if path corresponds to an executable object, false
+ /// otherwise
+ [[nodiscard]] static auto IsExecutable(std::filesystem::path const& path,
+ bool is_file_known = false) noexcept
+ -> bool {
+ if (not is_file_known and not IsFile(path)) {
+ return false;
+ }
+
+ try {
+ namespace fs = std::filesystem;
+ auto exec_flags = fs::perms::owner_exec bitor
+ fs::perms::group_exec bitor
+ fs::perms::others_exec;
+ auto exec_perms = fs::status(path).permissions() bitand exec_flags;
+ if (exec_perms == fs::perms::none) {
+ return false;
+ }
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "checking if path {} corresponds to an executable:\n{}",
+ path.string(),
+ e.what());
+ return false;
+ }
+
+ return true;
+ }
+
+ /// \brief Gets type of object in path according to file system
+ [[nodiscard]] static auto Type(std::filesystem::path const& path) noexcept
+ -> std::optional<ObjectType> {
+ if (IsFile(path)) {
+ if (IsExecutable(path, true)) {
+ return ObjectType::Executable;
+ }
+ return ObjectType::File;
+ }
+ if (IsDirectory(path)) {
+ return ObjectType::Tree;
+ }
+ Logger::Log(LogLevel::Debug,
+ "object type for {} not supported yet.",
+ path.string());
+ return std::nullopt;
+ }
+
+ [[nodiscard]] static auto ReadFile(
+ std::filesystem::path const& file) noexcept
+ -> std::optional<std::string> {
+ auto const type = Type(file);
+ if (not type) {
+ Logger::Log(LogLevel::Debug,
+ "{} can not be read because it is not a file.",
+ file.string());
+ return std::nullopt;
+ }
+ return ReadFile(file, *type);
+ }
+
+ [[nodiscard]] static auto ReadFile(std::filesystem::path const& file,
+ ObjectType type) noexcept
+ -> std::optional<std::string> {
+ if (not IsFileObject(type)) {
+ Logger::Log(LogLevel::Debug,
+ "{} can not be read because it is not a file.",
+ file.string());
+ return std::nullopt;
+ }
+ try {
+ std::string chunk{};
+ std::string content{};
+ chunk.resize(kChunkSize);
+ std::ifstream file_reader(file.string(), std::ios::binary);
+ if (file_reader.is_open()) {
+ auto ssize = gsl::narrow<std::streamsize>(chunk.size());
+ do {
+ file_reader.read(chunk.data(), ssize);
+ auto count = file_reader.gcount();
+ if (count == ssize) {
+ content += chunk;
+ }
+ else {
+ content +=
+ chunk.substr(0, gsl::narrow<std::size_t>(count));
+ }
+ } while (file_reader.good());
+ file_reader.close();
+ return content;
+ }
+ return std::nullopt;
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "reading file {}:\n{}",
+ file.string(),
+ e.what());
+ return std::nullopt;
+ }
+ }
+
+ [[nodiscard]] static auto ReadDirectory(
+ std::filesystem::path const& dir,
+ ReadDirEntryFunc const& read_entry) noexcept -> bool {
+ try {
+ for (auto const& entry : std::filesystem::directory_iterator{dir}) {
+ std::optional<ObjectType> type{};
+ if (entry.is_regular_file()) {
+ type = Type(entry.path());
+ }
+ if (entry.is_directory()) {
+ type = ObjectType::Tree;
+ }
+ if (not type) {
+ Logger::Log(LogLevel::Error,
+ "unsupported type for dir entry {}",
+ entry.path().string());
+ return false;
+ }
+ read_entry(entry.path().filename(), *type);
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(
+ LogLevel::Error, "reading directory {} failed", dir.string());
+ return false;
+ }
+ return true;
+ }
+
+ [[nodiscard]] static auto WriteFile(
+ std::string const& content,
+ std::filesystem::path const& file) noexcept -> auto {
+ if (not CreateDirectory(file.parent_path())) {
+ Logger::Log(LogLevel::Error,
+ "can not create directory {}",
+ file.parent_path().string());
+ return false;
+ }
+ try {
+ std::ofstream writer{file};
+ if (!writer.is_open()) {
+ Logger::Log(
+ LogLevel::Error, "can not open file {}", file.string());
+ return false;
+ }
+ writer << content;
+ writer.close();
+ return true;
+ } catch (std::exception const& e) {
+ Logger::Log(
+ LogLevel::Error, "writing to {}:\n{}", file.string(), e.what());
+ return false;
+ }
+ }
+
+ template <ObjectType kType>
+ requires(IsFileObject(kType)) [[nodiscard]] static auto WriteFileAs(
+ std::string const& content,
+ std::filesystem::path const& file) noexcept -> bool {
+ return WriteFile(content, file) and
+ SetFilePermissions(file, IsExecutableObject(kType));
+ }
+
+ [[nodiscard]] static auto WriteFileAs(std::string const& content,
+ std::filesystem::path const& file,
+ ObjectType output_type) noexcept
+ -> bool {
+ switch (output_type) {
+ case ObjectType::File:
+ return WriteFileAs<ObjectType::File>(content, file);
+ case ObjectType::Executable:
+ return WriteFileAs<ObjectType::Executable>(content, file);
+ case ObjectType::Tree:
+ return false;
+ }
+ }
+
+ [[nodiscard]] static auto IsRelativePath(
+ std::filesystem::path const& path) noexcept -> bool {
+ try {
+ return path.is_relative();
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return false;
+ }
+ }
+
+ [[nodiscard]] static auto IsAbsolutePath(
+ std::filesystem::path const& path) noexcept -> bool {
+ try {
+ return path.is_absolute();
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return false;
+ }
+ }
+
+ private:
+ enum class CreationStatus { Created, Exists, Failed };
+
+ static constexpr std::size_t kChunkSize{256};
+
+ /// \brief Race condition free directory creation.
+ /// Solves the TOCTOU issue.
+ [[nodiscard]] static auto CreateDirectoryImpl(
+ std::filesystem::path const& dir) noexcept -> CreationStatus {
+ try {
+ if (std::filesystem::is_directory(dir)) {
+ return CreationStatus::Exists;
+ }
+ if (std::filesystem::create_directories(dir)) {
+ return CreationStatus::Created;
+ }
+ // It could be that another thread has created the directory right
+ // after the current thread checked if it existed. For that reason,
+ // we try to create it and check if it exists if create_directories
+ // was not successful.
+ if (std::filesystem::is_directory(dir)) {
+ return CreationStatus::Exists;
+ }
+
+ return CreationStatus::Failed;
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return CreationStatus::Failed;
+ }
+ }
+
+ /// \brief Race condition free file creation.
+ /// Solves the TOCTOU issue via C11's std::fopen.
+ [[nodiscard]] static auto CreateFileImpl(
+ std::filesystem::path const& file) noexcept -> CreationStatus {
+ try {
+ if (std::filesystem::is_regular_file(file)) {
+ return CreationStatus::Exists;
+ }
+ if (gsl::owner<FILE*> fp = std::fopen(file.c_str(), "wx")) {
+ std::fclose(fp);
+ return CreationStatus::Created;
+ }
+ // It could be that another thread has created the file right after
+ // the current thread checked if it existed. For that reason, we try
+ // to create it and check if it exists if fopen() with exclusive bit
+ // was not successful.
+ if (std::filesystem::is_regular_file(file)) {
+ return CreationStatus::Exists;
+ }
+ return CreationStatus::Failed;
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return CreationStatus::Failed;
+ }
+ }
+
+ /// \brief Set special permissions for files.
+ /// Set to 0444 for non-executables and set to 0555 for executables.
+ static auto SetFilePermissions(std::filesystem::path const& path,
+ bool is_executable) noexcept -> bool {
+ try {
+ using std::filesystem::perms;
+ perms p{perms::owner_read | perms::group_read | perms::others_read};
+ if (is_executable) {
+ p |= perms::owner_exec | perms::group_exec | perms::others_exec;
+ }
+ std::filesystem::permissions(path, p);
+ return true;
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return false;
+ }
+ }
+}; // class FileSystemManager
+
+#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_FILE_SYSTEM_MANAGER_HPP
diff --git a/src/buildtool/file_system/git_cas.cpp b/src/buildtool/file_system/git_cas.cpp
new file mode 100644
index 00000000..e77d8d6e
--- /dev/null
+++ b/src/buildtool/file_system/git_cas.cpp
@@ -0,0 +1,180 @@
+#include "src/buildtool/file_system/git_cas.hpp"
+
+#include <sstream>
+
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+extern "C" {
+#include <git2.h>
+}
+
+namespace {
+
+constexpr auto kOIDRawSize{GIT_OID_RAWSZ};
+constexpr auto kOIDHexSize{GIT_OID_HEXSZ};
+
+[[nodiscard]] auto GitLastError() noexcept -> std::string {
+ git_error const* err{nullptr};
+ if ((err = git_error_last()) != nullptr and err->message != nullptr) {
+ return fmt::format("error code {}: {}", err->klass, err->message);
+ }
+ return "<unknown error>";
+}
+
+[[nodiscard]] auto GitObjectID(std::string const& id,
+ bool is_hex_id = false) noexcept
+ -> std::optional<git_oid> {
+ if ((is_hex_id and id.size() < kOIDHexSize) or id.size() < kOIDRawSize) {
+ Logger::Log(LogLevel::Error,
+ "invalid git object id {}",
+ is_hex_id ? id : ToHexString(id));
+ return std::nullopt;
+ }
+ git_oid oid{};
+ if (is_hex_id and git_oid_fromstr(&oid, id.data()) == 0) {
+ return oid;
+ }
+ if (not is_hex_id and
+ git_oid_fromraw(
+ &oid,
+ reinterpret_cast<unsigned char const*>(id.data()) // NOLINT
+ ) == 0) {
+ return oid;
+ }
+ Logger::Log(LogLevel::Error,
+ "parsing git object id {} failed with:\n{}",
+ is_hex_id ? id : ToHexString(id),
+ GitLastError());
+ return std::nullopt;
+}
+
+[[nodiscard]] auto GitTypeToObjectType(git_object_t const& type) noexcept
+ -> std::optional<ObjectType> {
+ switch (type) {
+ case GIT_OBJECT_BLOB:
+ return ObjectType::File;
+ case GIT_OBJECT_TREE:
+ return ObjectType::Tree;
+ default:
+ Logger::Log(LogLevel::Error,
+ "unsupported git object type {}",
+ git_object_type2string(type));
+ return std::nullopt;
+ }
+}
+
+} // namespace
+
+auto GitCAS::Open(std::filesystem::path const& repo_path) noexcept
+ -> GitCASPtr {
+ try {
+ auto cas = std::make_shared<GitCAS>();
+ if (cas->OpenODB(repo_path)) {
+ return std::static_pointer_cast<GitCAS const>(cas);
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "opening git object database failed with:\n{}",
+ ex.what());
+ }
+ return nullptr;
+}
+
+GitCAS::GitCAS() noexcept {
+ if (not(initialized_ = (git_libgit2_init() >= 0))) {
+ Logger::Log(LogLevel::Error, "initializing libgit2 failed");
+ }
+}
+GitCAS::~GitCAS() noexcept {
+ if (odb_ != nullptr) {
+ git_odb_free(odb_);
+ odb_ = nullptr;
+ }
+ if (initialized_) {
+ git_libgit2_shutdown();
+ }
+}
+
+auto GitCAS::ReadObject(std::string const& id, bool is_hex_id) const noexcept
+ -> std::optional<std::string> {
+ if (not initialized_) {
+ return std::nullopt;
+ }
+
+ auto oid = GitObjectID(id, is_hex_id);
+ if (not oid) {
+ return std::nullopt;
+ }
+
+ git_odb_object* obj = nullptr;
+ if (git_odb_read(&obj, odb_, &oid.value()) != 0) {
+ Logger::Log(LogLevel::Error,
+ "reading git object {} from database failed with:\n{}",
+ is_hex_id ? id : ToHexString(id),
+ GitLastError());
+ return std::nullopt;
+ }
+
+ std::string data(static_cast<char const*>(git_odb_object_data(obj)),
+ git_odb_object_size(obj));
+ git_odb_object_free(obj);
+
+ return data;
+}
+
+auto GitCAS::ReadHeader(std::string const& id, bool is_hex_id) const noexcept
+ -> std::optional<std::pair<std::size_t, ObjectType>> {
+ if (not initialized_) {
+ return std::nullopt;
+ }
+
+ auto oid = GitObjectID(id, is_hex_id);
+ if (not oid) {
+ return std::nullopt;
+ }
+
+ std::size_t size{};
+ git_object_t type{};
+ if (git_odb_read_header(&size, &type, odb_, &oid.value()) != 0) {
+ Logger::Log(LogLevel::Error,
+ "reading git object header {} from database failed "
+ "with:\n{}",
+ is_hex_id ? id : ToHexString(id),
+ GitLastError());
+ return std::nullopt;
+ }
+
+ if (auto obj_type = GitTypeToObjectType(type)) {
+ return std::make_pair(size, *obj_type);
+ }
+
+ return std::nullopt;
+}
+
+auto GitCAS::OpenODB(std::filesystem::path const& repo_path) noexcept -> bool {
+ if (initialized_) {
+ { // lock as git_repository API has no thread-safety guarantees
+ std::unique_lock lock{repo_mutex_};
+ git_repository* repo = nullptr;
+ if (git_repository_open(&repo, repo_path.c_str()) != 0) {
+ Logger::Log(LogLevel::Error,
+ "opening git repository {} failed with:\n{}",
+ repo_path.string(),
+ GitLastError());
+ return false;
+ }
+ git_repository_odb(&odb_, repo);
+ git_repository_free(repo);
+ }
+ if (odb_ == nullptr) {
+ Logger::Log(LogLevel::Error,
+ "obtaining git object database {} failed with:\n{}",
+ repo_path.string(),
+ GitLastError());
+ return false;
+ }
+ }
+ return initialized_;
+}
diff --git a/src/buildtool/file_system/git_cas.hpp b/src/buildtool/file_system/git_cas.hpp
new file mode 100644
index 00000000..d4341482
--- /dev/null
+++ b/src/buildtool/file_system/git_cas.hpp
@@ -0,0 +1,60 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CAS_HPP
+#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CAS_HPP
+
+#include <filesystem>
+#include <memory>
+#include <mutex>
+#include <optional>
+
+#include "src/buildtool/file_system/object_type.hpp"
+
+extern "C" {
+using git_odb = struct git_odb;
+}
+
+class GitCAS;
+using GitCASPtr = std::shared_ptr<GitCAS const>;
+
+/// \brief Git CAS that maintains its own libgit2 global state.
+class GitCAS {
+ public:
+ static auto Open(std::filesystem::path const& repo_path) noexcept
+ -> GitCASPtr;
+
+ GitCAS() noexcept;
+ ~GitCAS() noexcept;
+
+ // prohibit moves and copies
+ GitCAS(GitCAS const&) = delete;
+ GitCAS(GitCAS&& other) = delete;
+ auto operator=(GitCAS const&) = delete;
+ auto operator=(GitCAS&& other) = delete;
+
+ /// \brief Read object from CAS.
+ /// \param id The object id.
+ /// \param is_hex_id Specify whether `id` is hex string or raw.
+ [[nodiscard]] auto ReadObject(std::string const& id,
+ bool is_hex_id = false) const noexcept
+ -> std::optional<std::string>;
+
+ /// \brief Read object header from CAS.
+ /// \param id The object id.
+ /// \param is_hex_id Specify whether `id` is hex string or raw.
+ // Use with care. Quote from git2/odb.h:138:
+ // Note that most backends do not support reading only the header of an
+ // object, so the whole object will be read and then the header will be
+ // returned.
+ [[nodiscard]] auto ReadHeader(std::string const& id,
+ bool is_hex_id = false) const noexcept
+ -> std::optional<std::pair<std::size_t, ObjectType>>;
+
+ private:
+ static inline std::mutex repo_mutex_{};
+ git_odb* odb_{nullptr};
+ bool initialized_{false};
+
+ [[nodiscard]] auto OpenODB(std::filesystem::path const& repo_path) noexcept
+ -> bool;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_CAS_HPP
diff --git a/src/buildtool/file_system/git_tree.cpp b/src/buildtool/file_system/git_tree.cpp
new file mode 100644
index 00000000..3e98ead1
--- /dev/null
+++ b/src/buildtool/file_system/git_tree.cpp
@@ -0,0 +1,178 @@
+#include "src/buildtool/file_system/git_tree.hpp"
+
+#include <sstream>
+
+#include "src/buildtool/logging/logger.hpp"
+
+extern "C" {
+#include <git2.h>
+}
+
+namespace {
+
+constexpr auto kOIDRawSize{GIT_OID_RAWSZ};
+
+auto const kLoadTreeError =
+ std::make_shared<std::optional<GitTree>>(std::nullopt);
+
+[[nodiscard]] auto PermToType(std::string const& perm_str) noexcept
+ -> std::optional<ObjectType> {
+ constexpr auto kPermBase = 8;
+ constexpr auto kTreePerm = 040000;
+ constexpr auto kFilePerm = 0100644;
+ constexpr auto kExecPerm = 0100755;
+ constexpr auto kLinkPerm = 0120000;
+
+ int perm = std::stoi(perm_str, nullptr, kPermBase);
+
+ switch (perm) {
+ case kTreePerm:
+ return ObjectType::Tree;
+ case kFilePerm:
+ return ObjectType::File;
+ case kExecPerm:
+ return ObjectType::Executable;
+ case kLinkPerm:
+ Logger::Log(LogLevel::Error, "symlinks are not yet supported");
+ return std::nullopt;
+ default:
+ Logger::Log(LogLevel::Error, "unsupported permission {}", perm_str);
+ return std::nullopt;
+ }
+}
+
+auto ParseRawTreeObject(GitCASPtr const& cas,
+ std::string const& raw_tree) noexcept
+ -> std::optional<GitTree::entries_t> {
+ std::string perm{};
+ std::string path{};
+ std::string hash(kOIDRawSize, '\0');
+ std::istringstream iss{raw_tree};
+ GitTree::entries_t entries{};
+ // raw tree format is: "<perm> <path>\0<hash>[next entries...]"
+ while (std::getline(iss, perm, ' ') and // <perm>
+ std::getline(iss, path, '\0') and // <path>
+ iss.read(hash.data(), // <hash>
+ static_cast<std::streamsize>(hash.size()))) {
+ auto type = PermToType(perm);
+ if (not type) {
+ return std::nullopt;
+ }
+ try {
+ entries.emplace(path,
+ std::make_shared<GitTreeEntry>(cas, hash, *type));
+ } catch (std::exception const& ex) {
+ Logger::Log(LogLevel::Error,
+ "parsing git raw tree object failed with:\n{}",
+ ex.what());
+ return std::nullopt;
+ }
+ }
+ return entries;
+}
+
+// resolve '.' and '..' in path.
+[[nodiscard]] auto ResolveRelativePath(
+ std::filesystem::path const& path) noexcept -> std::filesystem::path {
+ auto normalized = path.lexically_normal();
+ return (normalized / "").parent_path(); // strip trailing slash
+}
+
+[[nodiscard]] auto LookupEntryPyPath(
+ GitTree const& tree,
+ std::filesystem::path::const_iterator it,
+ std::filesystem::path::const_iterator const& end) noexcept
+ -> GitTreeEntryPtr {
+ auto segment = *it;
+ auto entry = tree.LookupEntryByName(segment);
+ if (not entry) {
+ return nullptr;
+ }
+ if (++it != end) {
+ if (not entry->IsTree()) {
+ return nullptr;
+ }
+ return LookupEntryPyPath(*entry->Tree(), it, end);
+ }
+ return entry;
+}
+
+} // namespace
+
+auto GitTree::Read(std::filesystem::path const& repo_path,
+ std::string const& tree_id) noexcept
+ -> std::optional<GitTree> {
+ auto cas = GitCAS::Open(repo_path);
+ if (not cas) {
+ return std::nullopt;
+ }
+ return Read(cas, tree_id);
+}
+
+auto GitTree::Read(gsl::not_null<GitCASPtr> const& cas,
+ std::string const& tree_id) noexcept
+ -> std::optional<GitTree> {
+ auto obj = cas->ReadObject(tree_id, /*is_hex_id=*/true);
+ if (not obj) {
+ return std::nullopt;
+ }
+ auto entries = ParseRawTreeObject(cas, *obj);
+ if (not entries) {
+ return std::nullopt;
+ }
+ return GitTree{cas, std::move(*entries)};
+}
+
+auto GitTree::LookupEntryByName(std::string const& name) const noexcept
+ -> GitTreeEntryPtr {
+ auto entry_it = entries_.find(name);
+ if (entry_it == entries_.end()) {
+ Logger::Log(
+ LogLevel::Error, "git tree does not contain entry {}", name);
+ return nullptr;
+ }
+ return entry_it->second;
+}
+
+auto GitTree::LookupEntryByPath(
+ std::filesystem::path const& path) const noexcept -> GitTreeEntryPtr {
+ auto resolved = ResolveRelativePath(path);
+ return LookupEntryPyPath(*this, resolved.begin(), resolved.end());
+}
+
+auto GitTreeEntry::Blob() const noexcept -> std::optional<std::string> {
+ if (not IsBlob()) {
+ return std::nullopt;
+ }
+ return cas_->ReadObject(raw_id_);
+}
+
+auto GitTreeEntry::Tree() const& noexcept -> std::optional<GitTree> const& {
+ auto ptr = tree_cached_.load();
+ if (not ptr) {
+ if (not tree_loading_.exchange(true)) {
+ ptr = kLoadTreeError;
+ std::optional<std::string> obj{};
+ if (IsTree() and (obj = cas_->ReadObject(raw_id_))) {
+ if (auto entries = ParseRawTreeObject(cas_, *obj)) {
+ ptr = std::make_shared<std::optional<GitTree>>(
+ GitTree{cas_, std::move(*entries)});
+ }
+ }
+ tree_cached_.store(ptr);
+ tree_cached_.notify_all();
+ }
+ else {
+ tree_cached_.wait(nullptr);
+ ptr = tree_cached_.load();
+ }
+ }
+ return *ptr;
+}
+
+auto GitTreeEntry::Size() const noexcept -> std::optional<std::size_t> {
+ if (auto header = cas_->ReadHeader(raw_id_)) {
+ return header->first;
+ }
+ return std::nullopt;
+}
diff --git a/src/buildtool/file_system/git_tree.hpp b/src/buildtool/file_system/git_tree.hpp
new file mode 100644
index 00000000..57cb3b52
--- /dev/null
+++ b/src/buildtool/file_system/git_tree.hpp
@@ -0,0 +1,87 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_HPP
+#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_HPP
+
+#include <filesystem>
+#include <optional>
+#include <unordered_map>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/file_system/git_cas.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/utils/cpp/atomic.hpp"
+#include "src/utils/cpp/hex_string.hpp"
+
+class GitTreeEntry;
+using GitTreeEntryPtr = std::shared_ptr<GitTreeEntry const>;
+
+class GitTree {
+ friend class GitTreeEntry;
+
+ public:
+ using entries_t =
+ std::unordered_map<std::string, gsl::not_null<GitTreeEntryPtr>>;
+
+ /// \brief Read tree with given id from Git repository.
+ /// \param repo_path Path to the Git repository.
+ /// \param tree_id Tree id as as hex string.
+ [[nodiscard]] static auto Read(std::filesystem::path const& repo_path,
+ std::string const& tree_id) noexcept
+ -> std::optional<GitTree>;
+
+ /// \brief Read tree with given id from CAS.
+ /// \param cas Git CAS that contains the tree id.
+ /// \param tree_id Tree id as as hex string.
+ [[nodiscard]] static auto Read(gsl::not_null<GitCASPtr> const& cas,
+ std::string const& tree_id) noexcept
+ -> std::optional<GitTree>;
+
+ /// \brief Lookup by dir entry name. '.' and '..' are not allowed.
+ [[nodiscard]] auto LookupEntryByName(std::string const& name) const noexcept
+ -> GitTreeEntryPtr;
+
+ /// \brief Lookup by relative path. '.' is not allowed.
+ [[nodiscard]] auto LookupEntryByPath(
+ std::filesystem::path const& path) const noexcept -> GitTreeEntryPtr;
+
+ [[nodiscard]] auto begin() const noexcept { return entries_.begin(); }
+ [[nodiscard]] auto end() const noexcept { return entries_.end(); }
+
+ private:
+ gsl::not_null<GitCASPtr> cas_;
+ entries_t entries_;
+
+ GitTree(gsl::not_null<GitCASPtr> cas, entries_t&& entries) noexcept
+ : cas_{std::move(cas)}, entries_{std::move(entries)} {}
+};
+
+class GitTreeEntry {
+ public:
+ GitTreeEntry(gsl::not_null<GitCASPtr> cas,
+ std::string raw_id,
+ ObjectType type) noexcept
+ : cas_{std::move(cas)}, raw_id_{std::move(raw_id)}, type_{type} {}
+
+ [[nodiscard]] auto IsBlob() const noexcept { return IsFileObject(type_); }
+ [[nodiscard]] auto IsTree() const noexcept { return IsTreeObject(type_); }
+
+ [[nodiscard]] auto Blob() const noexcept -> std::optional<std::string>;
+ [[nodiscard]] auto Tree() && = delete;
+ [[nodiscard]] auto Tree() const& noexcept -> std::optional<GitTree> const&;
+
+ [[nodiscard]] auto Hash() const noexcept { return ToHexString(raw_id_); }
+ [[nodiscard]] auto Type() const noexcept { return type_; }
+ // Use with care. Implementation might read entire object to obtain size.
+ // Consider using Blob()->size() instead.
+ [[nodiscard]] auto Size() const noexcept -> std::optional<std::size_t>;
+
+ private:
+ gsl::not_null<GitCASPtr> cas_;
+ std::string raw_id_;
+ ObjectType type_;
+ mutable atomic_shared_ptr<std::optional<GitTree>> tree_cached_{nullptr};
+ mutable std::atomic<bool> tree_loading_{false};
+};
+
+using GitTreePtr = std::shared_ptr<GitTree const>;
+
+#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_GIT_TREE_HPP
diff --git a/src/buildtool/file_system/jsonfs.hpp b/src/buildtool/file_system/jsonfs.hpp
new file mode 100644
index 00000000..0ec381c7
--- /dev/null
+++ b/src/buildtool/file_system/jsonfs.hpp
@@ -0,0 +1,47 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_JSONFS_HPP
+#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_JSONFS_HPP
+
+#include <exception>
+#include <filesystem>
+#include <fstream>
+#include <optional>
+
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/utils/cpp/json.hpp"
+
+class Json {
+ public:
+ // Note that we are not using std::pair<std::nlohmann, bool> and being
+ // coherent with FileSystemManager::ReadFile because there is a bug in llvm
+ // toolchain related to type_traits that does not allow us to use
+ // std::pair<T,U> where T or U are nlohmann::json.
+ // LLVM bug report: https://bugs.llvm.org/show_bug.cgi?id=48507
+ // Minimal example: https://godbolt.org/z/zacedsGzo
+ [[nodiscard]] static auto ReadFile(
+ std::filesystem::path const& file) noexcept
+ -> std::optional<nlohmann::json> {
+ auto const type = FileSystemManager::Type(file);
+ if (not type or not IsFileObject(*type)) {
+ Logger::Log(LogLevel::Debug,
+ "{} can not be read because it is not a file.",
+ file.string());
+ return std::nullopt;
+ }
+ try {
+ nlohmann::json content;
+ std::ifstream file_reader(file.string());
+ if (file_reader.is_open()) {
+ file_reader >> content;
+ return content;
+ }
+ return std::nullopt;
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error, e.what());
+ return std::nullopt;
+ }
+ }
+
+}; // Class Json
+
+#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_JSONFS_HPP
diff --git a/src/buildtool/file_system/object_type.hpp b/src/buildtool/file_system/object_type.hpp
new file mode 100644
index 00000000..6209f05d
--- /dev/null
+++ b/src/buildtool/file_system/object_type.hpp
@@ -0,0 +1,44 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_TYPE_HPP
+#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_TYPE_HPP
+
+enum class ObjectType {
+ File,
+ Executable,
+ Tree,
+};
+
+[[nodiscard]] constexpr auto FromChar(char c) -> ObjectType {
+ switch (c) {
+ case 'x':
+ return ObjectType::Executable;
+ case 't':
+ return ObjectType::Tree;
+ default:
+ return ObjectType::File;
+ }
+}
+
+[[nodiscard]] constexpr auto ToChar(ObjectType type) -> char {
+ switch (type) {
+ case ObjectType::File:
+ return 'f';
+ case ObjectType::Executable:
+ return 'x';
+ case ObjectType::Tree:
+ return 't';
+ }
+}
+
+[[nodiscard]] constexpr auto IsFileObject(ObjectType type) -> bool {
+ return type == ObjectType::Executable or type == ObjectType::File;
+}
+
+[[nodiscard]] constexpr auto IsExecutableObject(ObjectType type) -> bool {
+ return type == ObjectType::Executable;
+}
+
+[[nodiscard]] constexpr auto IsTreeObject(ObjectType type) -> bool {
+ return type == ObjectType::Tree;
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_OBJECT_TYPE_HPP
diff --git a/src/buildtool/file_system/system_command.hpp b/src/buildtool/file_system/system_command.hpp
new file mode 100644
index 00000000..66470ade
--- /dev/null
+++ b/src/buildtool/file_system/system_command.hpp
@@ -0,0 +1,202 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_EXECUTION_SYSTEM_HPP
+#define INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_EXECUTION_SYSTEM_HPP
+
+#include <array>
+#include <cstdio>
+#include <cstring> // for strerror()
+#include <iterator>
+#include <map>
+#include <optional>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// \brief Execute system commands and obtain stdout, stderr and return value.
+/// Subsequent commands are context free and are not affected by previous
+/// commands. This class is not thread-safe.
+class SystemCommand {
+ public:
+ struct ExecOutput {
+ int return_value{};
+ std::filesystem::path stdout_file{};
+ std::filesystem::path stderr_file{};
+ };
+
+ /// \brief Create execution system with name.
+ explicit SystemCommand(std::string name) : logger_{std::move(name)} {}
+
+ /// \brief Execute command and arguments.
+ /// \param argv argv vector with the command to execute
+ /// \param env Environment variables set for execution.
+ /// \param cwd Working directory for execution.
+ /// \param tmpdir Temporary directory for storing stdout/stderr files.
+ /// \returns std::nullopt if there was an error in the execution setup
+ /// outside running the command itself, SystemCommand::ExecOutput otherwise.
+ [[nodiscard]] auto Execute(std::vector<std::string> argv,
+ std::map<std::string, std::string> env,
+ std::filesystem::path const& cwd,
+ std::filesystem::path const& tmpdir) noexcept
+ -> std::optional<ExecOutput> {
+ if (not FileSystemManager::IsDirectory(tmpdir)) {
+ logger_.Emit(LogLevel::Error,
+ "Temporary directory does not exist {}",
+ tmpdir.string());
+ return std::nullopt;
+ }
+
+ if (argv.empty()) {
+ logger_.Emit(LogLevel::Error, "Command cannot be empty.");
+ return std::nullopt;
+ }
+
+ std::vector<char*> cmd = UnwrapStrings(&argv);
+
+ std::vector<std::string> env_string{};
+ std::transform(std::begin(env),
+ std::end(env),
+ std::back_inserter(env_string),
+ [](auto& name_value) {
+ return name_value.first + "=" + name_value.second;
+ });
+ std::vector<char*> envp = UnwrapStrings(&env_string);
+ return ExecuteCommand(cmd.data(), envp.data(), cwd, tmpdir);
+ }
+
+ private:
+ Logger logger_;
+
+ /// \brief Open file exclusively as write-only.
+ [[nodiscard]] static auto OpenFile(
+ std::filesystem::path const& file_path) noexcept {
+ static auto file_closer = [](gsl::owner<FILE*> f) {
+ if (f != nullptr) {
+ std::fclose(f);
+ }
+ };
+ return std::unique_ptr<FILE, decltype(file_closer)>(
+ std::fopen(file_path.c_str(), "wx"), file_closer);
+ }
+
+ /// \brief Execute command and arguments.
+ /// \param cmd Command arguments as char pointer array.
+ /// \param envp Environment variables as char pointer array.
+ /// \param cwd Working directory for execution.
+ /// \param tmpdir Temporary directory for storing stdout/stderr files.
+ /// \returns ExecOutput if command was successfully submitted to the system.
+ /// \returns std::nullopt on internal failure.
+ [[nodiscard]] auto ExecuteCommand(
+ char* const* cmd,
+ char* const* envp,
+ std::filesystem::path const& cwd,
+ std::filesystem::path const& tmpdir) noexcept
+ -> std::optional<ExecOutput> {
+ auto stdout_file = tmpdir / "stdout";
+ auto stderr_file = tmpdir / "stderr";
+ if (auto const out = OpenFile(stdout_file)) {
+ if (auto const err = OpenFile(stderr_file)) {
+ if (auto retval = ForkAndExecute(
+ cmd, envp, cwd, fileno(out.get()), fileno(err.get()))) {
+ return ExecOutput{*retval,
+ std::move(stdout_file),
+ std::move(stderr_file)};
+ }
+ }
+ else {
+ logger_.Emit(LogLevel::Error,
+ "Failed to open stderr file '{}' with error: {}",
+ stderr_file.string(),
+ strerror(errno));
+ }
+ }
+ else {
+ logger_.Emit(LogLevel::Error,
+ "Failed to open stdout file '{}' with error: {}",
+ stdout_file.string(),
+ strerror(errno));
+ }
+
+ return std::nullopt;
+ }
+
+ /// \brief Fork process and exec command.
+ /// \param cmd Command arguments as char pointer array.
+ /// \param envp Environment variables as char pointer array.
+ /// \param cwd Working directory for execution.
+ /// \param out_fd File descriptor to standard output file.
+ /// \param err_fd File descriptor to standard erro file.
+ /// \returns return code if command was successfully submitted to system.
+ /// \returns std::nullopt if fork or exec failed.
+ [[nodiscard]] auto ForkAndExecute(char* const* cmd,
+ char* const* envp,
+ std::filesystem::path const& cwd,
+ int out_fd,
+ int err_fd) const noexcept
+ -> std::optional<int> {
+ // fork child process
+ pid_t pid = ::fork();
+ if (-1 == pid) {
+ logger_.Emit(LogLevel::Error,
+ "Failed to execute '{}': cannot fork a child process.",
+ *cmd);
+ return std::nullopt;
+ }
+
+ // dispatch child/parent process
+ if (pid == 0) {
+ // some executables require an open (possibly seekable) stdin, and
+ // therefore, we use an open temporary file that does not appear
+ // on the file system and will be removed automatically once the
+ // descriptor is closed.
+ gsl::owner<FILE*> in_file = std::tmpfile();
+ auto in_fd = fileno(in_file);
+
+ // redirect and close fds
+ ::dup2(in_fd, STDIN_FILENO);
+ ::dup2(out_fd, STDOUT_FILENO);
+ ::dup2(err_fd, STDERR_FILENO);
+ ::close(in_fd);
+ ::close(out_fd);
+ ::close(err_fd);
+
+ [[maybe_unused]] auto anchor =
+ FileSystemManager::ChangeDirectory(cwd);
+
+ // execute command in child process and exit
+ ::execvpe(*cmd, cmd, envp);
+
+ // report error and terminate child process if ::execvp did not exit
+ logger_.Emit(LogLevel::Error,
+ "Failed to execute '{}' with error: {}",
+ *cmd,
+ strerror(errno));
+
+ std::exit(EXIT_FAILURE);
+ }
+
+ // wait for child to finish and obtain return value
+ int status{};
+ ::waitpid(pid, &status, 0);
+ // NOLINTNEXTLINE(hicpp-signed-bitwise)
+ return WEXITSTATUS(status);
+ }
+
+ static auto UnwrapStrings(std::vector<std::string>* v) noexcept
+ -> std::vector<char*> {
+ std::vector<char*> raw{};
+ std::transform(std::begin(*v),
+ std::end(*v),
+ std::back_inserter(raw),
+ [](auto& str) { return str.data(); });
+ raw.push_back(nullptr);
+ return raw;
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_FILE_SYSTEM_EXECUTION_SYSTEM_HPP
diff --git a/src/buildtool/graph_traverser/TARGETS b/src/buildtool/graph_traverser/TARGETS
new file mode 100644
index 00000000..ab13a2bc
--- /dev/null
+++ b/src/buildtool/graph_traverser/TARGETS
@@ -0,0 +1,22 @@
+{ "graph_traverser":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["graph_traverser"]
+ , "hdrs": ["graph_traverser.hpp"]
+ , "deps":
+ [ ["src/buildtool/common", "cli"]
+ , ["src/buildtool/common", "tree"]
+ , ["src/buildtool/execution_engine/dag", "dag"]
+ , ["src/buildtool/execution_engine/executor", "executor"]
+ , ["src/buildtool/execution_engine/traverser", "traverser"]
+ , ["src/buildtool/execution_api/local", "local"]
+ , ["src/buildtool/execution_api/remote", "bazel"]
+ , ["src/buildtool/execution_api/remote", "config"]
+ , ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/file_system", "object_type"]
+ , ["src/buildtool/file_system", "jsonfs"]
+ , ["src/utils/cpp", "json"]
+ , ["@", "fmt", "", "fmt"]
+ ]
+ , "stage": ["src", "buildtool", "graph_traverser"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/graph_traverser/graph_traverser.hpp b/src/buildtool/graph_traverser/graph_traverser.hpp
new file mode 100644
index 00000000..c92fbbf8
--- /dev/null
+++ b/src/buildtool/graph_traverser/graph_traverser.hpp
@@ -0,0 +1,569 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_GRAPH_TRAVERSER_GRAPH_TRAVERSER_HPP
+#define INCLUDED_SRC_BUILDTOOL_GRAPH_TRAVERSER_GRAPH_TRAVERSER_HPP
+
+#include <cstdlib>
+#include <filesystem>
+#include <map>
+#include <optional>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+
+#include "fmt/core.h"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/common/cli.hpp"
+#include "src/buildtool/common/statistics.hpp"
+#include "src/buildtool/common/tree.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_blob_container.hpp"
+#include "src/buildtool/execution_api/local/local_api.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
+#include "src/buildtool/execution_api/remote/config.hpp"
+#include "src/buildtool/execution_engine/dag/dag.hpp"
+#include "src/buildtool/execution_engine/executor/executor.hpp"
+#include "src/buildtool/execution_engine/traverser/traverser.hpp"
+#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/jsonfs.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
+#include "src/buildtool/logging/log_sink_cmdline.hpp"
+#include "src/buildtool/logging/log_sink_file.hpp"
+#include "src/buildtool/logging/logger.hpp"
+#include "src/utils/cpp/json.hpp"
+
+class GraphTraverser {
+ public:
+ struct CommandLineArguments {
+ std::size_t jobs;
+ EndpointArguments endpoint;
+ BuildArguments build;
+ std::optional<StageArguments> stage;
+ std::optional<RebuildArguments> rebuild;
+ };
+
+ explicit GraphTraverser(CommandLineArguments clargs)
+ : clargs_{std::move(clargs)},
+ api_{CreateExecutionApi(clargs_.endpoint)} {}
+
+ /// \brief Parses actions and blobs into graph, traverses it and retrieves
+ /// outputs specified by command line arguments
+ [[nodiscard]] auto BuildAndStage(
+ std::map<std::string, ArtifactDescription> const& artifact_descriptions,
+ std::map<std::string, ArtifactDescription> const& runfile_descriptions,
+ std::vector<ActionDescription> const& action_descriptions,
+ std::vector<std::string> const& blobs,
+ std::vector<Tree> const& trees) const
+ -> std::optional<std::pair<std::vector<std::filesystem::path>, bool>> {
+ DependencyGraph graph; // must outlive artifact_nodes
+ auto artifacts = BuildArtifacts(&graph,
+ artifact_descriptions,
+ runfile_descriptions,
+ action_descriptions,
+ trees,
+ blobs);
+ if (not artifacts) {
+ return std::nullopt;
+ }
+ auto const [rel_paths, artifact_nodes] = *artifacts;
+
+ auto const object_infos = CollectObjectInfos(artifact_nodes);
+ if (not object_infos) {
+ return std::nullopt;
+ }
+ bool failed_artifacts = false;
+ for (auto const& obj_info : *object_infos) {
+ failed_artifacts = failed_artifacts || obj_info.failed;
+ }
+
+ if (not clargs_.stage) {
+ PrintOutputs("Artifacts built, logical paths are:",
+ rel_paths,
+ artifact_nodes,
+ runfile_descriptions);
+ MaybePrintToStdout(*artifacts);
+ return std::make_pair(std::move(artifacts->first),
+ failed_artifacts);
+ }
+
+ auto output_paths = RetrieveOutputs(rel_paths, *object_infos);
+ if (not output_paths) {
+ return std::nullopt;
+ }
+ PrintOutputs("Artifacts can be found in:",
+ *output_paths,
+ artifact_nodes,
+ runfile_descriptions);
+
+ MaybePrintToStdout(*artifacts);
+
+ return std::make_pair(*output_paths, failed_artifacts);
+ }
+
+ /// \brief Parses graph description into graph, traverses it and retrieves
+ /// outputs specified by command line arguments
+ [[nodiscard]] auto BuildAndStage(
+ std::filesystem::path const& graph_description,
+ nlohmann::json const& artifacts) const
+ -> std::optional<std::pair<std::vector<std::filesystem::path>, bool>> {
+ // Read blobs to upload and actions from graph description file
+ auto desc = ReadGraphDescription(graph_description);
+ if (not desc) {
+ return std::nullopt;
+ }
+ auto const [blobs, tree_descs, actions] = *desc;
+
+ std::vector<ActionDescription> action_descriptions{};
+ action_descriptions.reserve(actions.size());
+ for (auto const& [id, description] : actions.items()) {
+ auto action = ActionDescription::FromJson(id, description);
+ if (not action) {
+ return std::nullopt; // Error already logged
+ }
+ action_descriptions.emplace_back(std::move(*action));
+ }
+
+ std::vector<Tree> trees{};
+ for (auto const& [id, description] : tree_descs.items()) {
+ auto tree = Tree::FromJson(id, description);
+ if (not tree) {
+ return std::nullopt;
+ }
+ trees.emplace_back(std::move(*tree));
+ }
+
+ std::map<std::string, ArtifactDescription> artifact_descriptions{};
+ for (auto const& [rel_path, description] : artifacts.items()) {
+ auto artifact = ArtifactDescription::FromJson(description);
+ if (not artifact) {
+ return std::nullopt; // Error already logged
+ }
+ artifact_descriptions.emplace(rel_path, std::move(*artifact));
+ }
+
+ return BuildAndStage(
+ artifact_descriptions, {}, action_descriptions, blobs, trees);
+ }
+
+ [[nodiscard]] auto ExecutionApi() const -> gsl::not_null<IExecutionApi*> {
+ return &(*api_);
+ }
+
+ private:
+ CommandLineArguments const clargs_;
+ gsl::not_null<IExecutionApi::Ptr> const api_;
+
+ /// \brief Reads contents of graph description file as json object. In case
+ /// the description is missing "blobs" or "actions" key/value pairs or they
+ /// can't be retrieved with the appropriate types, execution is terminated
+ /// after logging error
+ /// \returns A pair containing the blobs to upload (as a vector of strings)
+ /// and the actions as a json object.
+ [[nodiscard]] static auto ReadGraphDescription(
+ std::filesystem::path const& graph_description)
+ -> std::optional<
+ std::tuple<nlohmann::json, nlohmann::json, nlohmann::json>> {
+ auto const graph_description_opt = Json::ReadFile(graph_description);
+ if (not graph_description_opt.has_value()) {
+ Logger::Log(LogLevel::Error,
+ "parsing graph from {}",
+ graph_description.string());
+ return std::nullopt;
+ }
+ auto blobs_opt = ExtractValueAs<std::vector<std::string>>(
+ *graph_description_opt, "blobs", [](std::string const& s) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"blobs\" from "
+ "graph description.",
+ s);
+ });
+ auto trees_opt = ExtractValueAs<nlohmann::json>(
+ *graph_description_opt, "trees", [](std::string const& s) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"trees\" from "
+ "graph description.",
+ s);
+ });
+ auto actions_opt = ExtractValueAs<nlohmann::json>(
+ *graph_description_opt, "actions", [](std::string const& s) {
+ Logger::Log(LogLevel::Error,
+ "{}\ncan not retrieve value for \"actions\" from "
+ "graph description.",
+ s);
+ });
+ if (not blobs_opt or not trees_opt or not actions_opt) {
+ return std::nullopt;
+ }
+ return std::make_tuple(std::move(*blobs_opt),
+ std::move(*trees_opt),
+ std::move(*actions_opt));
+ }
+
+ [[nodiscard]] static auto CreateExecutionApi(
+ EndpointArguments const& clargs) -> gsl::not_null<IExecutionApi::Ptr> {
+ if (clargs.remote_execution_address) {
+ auto remote = RemoteExecutionConfig{};
+ if (not remote.SetAddress(*clargs.remote_execution_address)) {
+ Logger::Log(LogLevel::Error,
+ "parsing remote execution address '{}' failed.",
+ *clargs.remote_execution_address);
+ std::exit(EXIT_FAILURE);
+ }
+
+ ExecutionConfiguration config;
+ config.skip_cache_lookup = false;
+
+ return std::make_unique<BazelApi>(
+ "remote-execution", remote.Host(), remote.Port(), config);
+ }
+ return std::make_unique<LocalApi>();
+ }
+
+ /// \brief Requires for the executor to upload blobs to CAS. In the case any
+ /// of the uploads fails, execution is terminated
+ /// \param[in] blobs blobs to be uploaded
+ [[nodiscard]] auto UploadBlobs(
+ std::vector<std::string> const& blobs) const noexcept -> bool {
+ BlobContainer container;
+ for (auto const& blob : blobs) {
+ auto digest = ArtifactDigest{ComputeHash(blob), blob.size()};
+ Logger::Log(LogLevel::Trace, [&]() {
+ return fmt::format(
+ "Uploaded blob {}, its digest has id {} and size {}.",
+ nlohmann::json(blob).dump(),
+ digest.hash(),
+ digest.size());
+ });
+ try {
+ container.Emplace(BazelBlob{std::move(digest), blob});
+ } catch (std::exception const& ex) {
+ Logger::Log(
+ LogLevel::Error, "failed to create blob with: ", ex.what());
+ return false;
+ }
+ }
+ return api_->Upload(container);
+ }
+
+ /// \brief Adds the artifacts to be retrieved to the graph
+ /// \param[in] g dependency graph
+ /// \param[in] artifacts output artifact map
+ /// \param[in] runfiles output runfile map
+ /// \returns pair of vectors where the first vector contains the absolute
+ /// paths to which the artifacts will be retrieved and the second one
+ /// contains the ids of the artifacts to be retrieved
+ [[nodiscard]] static auto AddArtifactsToRetrieve(
+ gsl::not_null<DependencyGraph*> const& g,
+ std::map<std::string, ArtifactDescription> const& artifacts,
+ std::map<std::string, ArtifactDescription> const& runfiles)
+ -> std::optional<std::pair<std::vector<std::filesystem::path>,
+ std::vector<ArtifactIdentifier>>> {
+ std::vector<std::filesystem::path> rel_paths;
+ std::vector<ArtifactIdentifier> ids;
+ auto total_size = artifacts.size() + runfiles.size();
+ rel_paths.reserve(total_size);
+ ids.reserve(total_size);
+ auto add_and_get_info =
+ [&g, &rel_paths, &ids](
+ std::map<std::string, ArtifactDescription> const& descriptions)
+ -> bool {
+ for (auto const& [rel_path, artifact] : descriptions) {
+ rel_paths.emplace_back(rel_path);
+ ids.emplace_back(g->AddArtifact(artifact));
+ }
+ return true;
+ };
+ if (add_and_get_info(artifacts) and add_and_get_info(runfiles)) {
+ return std::make_pair(std::move(rel_paths), std::move(ids));
+ }
+ return std::nullopt;
+ }
+
+ /// \brief Traverses the graph. In case any of the artifact ids
+ /// specified by the command line arguments is duplicated, execution is
+ /// terminated.
+ [[nodiscard]] auto Traverse(
+ DependencyGraph const& g,
+ std::vector<ArtifactIdentifier> const& artifact_ids) const -> bool {
+ Executor executor{&(*api_), clargs_.build.platform_properties};
+ Traverser t{executor, g, clargs_.jobs};
+ return t.Traverse({std::begin(artifact_ids), std::end(artifact_ids)});
+ }
+
+ [[nodiscard]] auto TraverseRebuild(
+ DependencyGraph const& g,
+ std::vector<ArtifactIdentifier> const& artifact_ids) const -> bool {
+ // create second configuration for cache endpoint
+ auto cache_args = clargs_.endpoint;
+ if (not clargs_.rebuild->cache_endpoint.value_or("").empty()) {
+ cache_args.remote_execution_address =
+ *clargs_.rebuild->cache_endpoint == "local"
+ ? std::nullopt // disable
+ : clargs_.rebuild->cache_endpoint; // set endpoint
+ }
+
+ // setup rebuilder with api for cache endpoint
+ auto api_cached = CreateExecutionApi(cache_args);
+ Rebuilder executor{
+ &(*api_), &(*api_cached), clargs_.build.platform_properties};
+ bool success{false};
+ {
+ Traverser t{executor, g, clargs_.jobs};
+ success =
+ t.Traverse({std::begin(artifact_ids), std::end(artifact_ids)});
+ }
+
+ if (success and clargs_.rebuild->dump_flaky) {
+ std::ofstream file{*clargs_.rebuild->dump_flaky};
+ file << executor.DumpFlakyActions().dump(2);
+ }
+ return success;
+ }
+
+ /// \brief Retrieves nodes corresponding to artifacts with ids in artifacts.
+ /// In case any of the identifiers doesn't correspond to a node inside the
+ /// graph, we write out error message and stop execution with failure code
+ [[nodiscard]] static auto GetArtifactNodes(
+ DependencyGraph const& g,
+ std::vector<ArtifactIdentifier> const& artifact_ids) noexcept
+ -> std::optional<std::vector<DependencyGraph::ArtifactNode const*>> {
+ std::vector<DependencyGraph::ArtifactNode const*> nodes{};
+
+ for (auto const& art_id : artifact_ids) {
+ auto const* node = g.ArtifactNodeWithId(art_id);
+ if (node == nullptr) {
+ Logger::Log(
+ LogLevel::Error, "Artifact {} not found in graph.", art_id);
+ return std::nullopt;
+ }
+ nodes.push_back(node);
+ }
+ return nodes;
+ }
+
+ void LogStatistics() const noexcept {
+ auto const& stats = Statistics::Instance();
+ if (clargs_.rebuild) {
+ std::stringstream ss{};
+ ss << stats.RebuiltActionComparedCounter()
+ << " actions compared with cache";
+ if (stats.ActionsFlakyCounter() > 0) {
+ ss << ", " << stats.ActionsFlakyCounter()
+ << " flaky actions found";
+ ss << " (" << stats.ActionsFlakyTaintedCounter()
+ << " of which tainted)";
+ }
+ if (stats.RebuiltActionMissingCounter() > 0) {
+ ss << ", no cache entry found for "
+ << stats.RebuiltActionMissingCounter() << " actions";
+ }
+ ss << ".";
+ Logger::Log(LogLevel::Info, ss.str());
+ }
+ else {
+ Logger::Log(LogLevel::Info,
+ "Processed {} actions, {} cache hits.",
+ stats.ActionsQueuedCounter(),
+ stats.ActionsCachedCounter());
+ }
+ }
+
+ [[nodiscard]] auto BuildArtifacts(
+ gsl::not_null<DependencyGraph*> const& graph,
+ std::map<std::string, ArtifactDescription> const& artifacts,
+ std::map<std::string, ArtifactDescription> const& runfiles,
+ std::vector<ActionDescription> const& actions,
+ std::vector<Tree> const& trees,
+ std::vector<std::string> const& blobs) const
+ -> std::optional<
+ std::pair<std::vector<std::filesystem::path>,
+ std::vector<DependencyGraph::ArtifactNode const*>>> {
+ if (not UploadBlobs(blobs)) {
+ return std::nullopt;
+ }
+
+ auto artifact_infos =
+ AddArtifactsToRetrieve(graph, artifacts, runfiles);
+ if (not artifact_infos) {
+ return std::nullopt;
+ }
+ auto& [output_paths, artifact_ids] = *artifact_infos;
+
+ std::vector<ActionDescription> tree_actions{};
+ tree_actions.reserve(trees.size());
+ for (auto const& tree : trees) {
+ tree_actions.emplace_back(tree.Action());
+ }
+
+ if (not graph->Add(actions) or not graph->Add(tree_actions)) {
+ Logger::Log(LogLevel::Error, [&actions]() {
+ auto json = nlohmann::json::array();
+ for (auto const& desc : actions) {
+ json.push_back(desc.ToJson());
+ }
+ return fmt::format(
+ "could not build the dependency graph from the actions "
+ "described in {}.",
+ json.dump());
+ });
+ return std::nullopt;
+ }
+
+ if (clargs_.rebuild ? not TraverseRebuild(*graph, artifact_ids)
+ : not Traverse(*graph, artifact_ids)) {
+ Logger::Log(LogLevel::Error, "traversing graph failed.");
+ return std::nullopt;
+ }
+
+ LogStatistics();
+
+ auto artifact_nodes = GetArtifactNodes(*graph, artifact_ids);
+ if (not artifact_nodes) {
+ return std::nullopt;
+ }
+ return std::make_pair(std::move(output_paths),
+ std::move(*artifact_nodes));
+ }
+
+ [[nodiscard]] auto PrepareOutputPaths(
+ std::vector<std::filesystem::path> const& rel_paths) const
+ -> std::optional<std::vector<std::filesystem::path>> {
+ std::vector<std::filesystem::path> output_paths{};
+ output_paths.reserve(rel_paths.size());
+ for (auto const& rel_path : rel_paths) {
+ auto output_path = clargs_.stage->output_dir / rel_path;
+ if (FileSystemManager::IsFile(output_path) and
+ not FileSystemManager::RemoveFile(output_path)) {
+ Logger::Log(LogLevel::Error,
+ "Could not clean output path {}",
+ output_path.string());
+ return std::nullopt;
+ }
+ output_paths.emplace_back(std::move(output_path));
+ }
+ return output_paths;
+ }
+
+ [[nodiscard]] static auto CollectObjectInfos(
+ std::vector<DependencyGraph::ArtifactNode const*> const& artifact_nodes)
+ -> std::optional<std::vector<Artifact::ObjectInfo>> {
+ std::vector<Artifact::ObjectInfo> object_infos;
+ object_infos.reserve(artifact_nodes.size());
+ for (auto const* art_ptr : artifact_nodes) {
+ auto const& info = art_ptr->Content().Info();
+ if (info) {
+ object_infos.push_back(*info);
+ }
+ else {
+ Logger::Log(LogLevel::Error,
+ "artifact {} could not be retrieved, it can not be "
+ "found in CAS.",
+ art_ptr->Content().Id());
+ return std::nullopt;
+ }
+ }
+ return object_infos;
+ }
+
+ /// \brief Asks execution API to copy output artifacts to paths specified by
+ /// command line arguments and writes location info. In case the executor
+ /// couldn't retrieve any of the outputs, execution is terminated.
+ [[nodiscard]] auto RetrieveOutputs(
+ std::vector<std::filesystem::path> const& rel_paths,
+ std::vector<Artifact::ObjectInfo> const& object_infos) const
+ -> std::optional<std::vector<std::filesystem::path>> {
+ // Create output directory
+ if (not FileSystemManager::CreateDirectory(clargs_.stage->output_dir)) {
+ return std::nullopt; // Message logged in the file system manager
+ }
+
+ auto output_paths = PrepareOutputPaths(rel_paths);
+
+ if (not output_paths or
+ not api_->RetrieveToPaths(object_infos, *output_paths)) {
+ Logger::Log(LogLevel::Error, "Could not retrieve outputs.");
+ return std::nullopt;
+ }
+
+ return std::move(*output_paths);
+ }
+
+ void PrintOutputs(
+ std::string message,
+ std::vector<std::filesystem::path> const& paths,
+ std::vector<DependencyGraph::ArtifactNode const*> const& artifact_nodes,
+ std::map<std::string, ArtifactDescription> const& runfiles) const {
+ std::string msg_dbg{"Artifact ids:"};
+ nlohmann::json json{};
+ for (std::size_t pos = 0; pos < paths.size(); ++pos) {
+ auto path = paths[pos].string();
+ auto id = IdentifierToString(artifact_nodes[pos]->Content().Id());
+ if (clargs_.build.show_runfiles or
+ not runfiles.contains(clargs_.stage
+ ? std::filesystem::proximate(
+ path, clargs_.stage->output_dir)
+ .string()
+ : path)) {
+ auto info = artifact_nodes[pos]->Content().Info();
+ if (info) {
+ message += fmt::format("\n {} {}", path, info->ToString());
+ if (clargs_.build.dump_artifacts) {
+ json[path] = info->ToJson();
+ }
+ }
+ else {
+ Logger::Log(
+ LogLevel::Error, "Missing info for artifact {}.", id);
+ }
+ }
+ msg_dbg += fmt::format("\n {}: {}", path, id);
+ }
+
+ if (not clargs_.build.show_runfiles and !runfiles.empty()) {
+ message += fmt::format("\n({} runfiles omitted.)", runfiles.size());
+ }
+
+ Logger::Log(LogLevel::Info, "{}", message);
+ Logger::Log(LogLevel::Debug, "{}", msg_dbg);
+
+ if (clargs_.build.dump_artifacts) {
+ if (*clargs_.build.dump_artifacts == "-") {
+ std::cout << std::setw(2) << json << std::endl;
+ }
+ else {
+ std::ofstream os(*clargs_.build.dump_artifacts);
+ os << std::setw(2) << json << std::endl;
+ }
+ }
+ }
+
+ void MaybePrintToStdout(
+ std::pair<std::vector<std::filesystem::path>,
+ std::vector<DependencyGraph::ArtifactNode const*>> artifacts)
+ const {
+ if (clargs_.build.print_to_stdout) {
+ for (size_t i = 0; i < artifacts.first.size(); i++) {
+ if (artifacts.first[i] == *(clargs_.build.print_to_stdout)) {
+ auto info = artifacts.second[i]->Content().Info();
+ if (info) {
+ if (not api_->RetrieveToFds({*info},
+ {dup(fileno(stdout))})) {
+ Logger::Log(LogLevel::Error,
+ "Failed to retrieve {}",
+ *(clargs_.build.print_to_stdout));
+ }
+ }
+ else {
+ Logger::Log(
+ LogLevel::Error,
+ "Failed to obtain object information for {}",
+ *(clargs_.build.print_to_stdout));
+ }
+ return;
+ }
+ }
+ Logger::Log(LogLevel::Warning,
+ "{} not a logical path of the specified target",
+ *(clargs_.build.print_to_stdout));
+ }
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_GRAPH_TRAVERSER_GRAPH_TRAVERSER_HPP
diff --git a/src/buildtool/logging/TARGETS b/src/buildtool/logging/TARGETS
new file mode 100644
index 00000000..2a02cd93
--- /dev/null
+++ b/src/buildtool/logging/TARGETS
@@ -0,0 +1,20 @@
+{ "log_level":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["log_level"]
+ , "hdrs": ["log_level.hpp"]
+ , "stage": ["src", "buildtool", "logging"]
+ }
+, "logging":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["logging"]
+ , "hdrs":
+ [ "log_config.hpp"
+ , "log_sink.hpp"
+ , "log_sink_cmdline.hpp"
+ , "log_sink_file.hpp"
+ , "logger.hpp"
+ ]
+ , "deps": ["log_level", ["@", "fmt", "", "fmt"]]
+ , "stage": ["src", "buildtool", "logging"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/logging/log_config.hpp b/src/buildtool/logging/log_config.hpp
new file mode 100644
index 00000000..799a6ad5
--- /dev/null
+++ b/src/buildtool/logging/log_config.hpp
@@ -0,0 +1,69 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_CONFIG_HPP
+#define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_CONFIG_HPP
+
+#include <mutex>
+#include <vector>
+
+#include "src/buildtool/logging/log_level.hpp"
+#include "src/buildtool/logging/log_sink.hpp"
+
+/// \brief Global static logging configuration.
+/// The entire class is thread-safe.
+class LogConfig {
+ public:
+ /// \brief Set the log limit.
+ static void SetLogLimit(LogLevel level) noexcept { log_limit_ = level; }
+
+ /// \brief Replace all configured sinks.
+ /// NOTE: Reinitializes all internal factories.
+ static void SetSinks(std::vector<LogSinkFactory>&& factories) noexcept {
+ std::lock_guard lock{mutex_};
+ sinks_.clear();
+ sinks_.reserve(factories.size());
+ std::transform(factories.cbegin(),
+ factories.cend(),
+ std::back_inserter(sinks_),
+ [](auto& f) { return f(); });
+ factories_ = std::move(factories);
+ }
+
+ /// \brief Add new a new sink.
+ static void AddSink(LogSinkFactory&& factory) noexcept {
+ std::lock_guard lock{mutex_};
+ sinks_.push_back(factory());
+ factories_.push_back(std::move(factory));
+ }
+
+ /// \brief Get the currently configured log limit.
+ [[nodiscard]] static auto LogLimit() noexcept -> LogLevel {
+ return log_limit_;
+ }
+
+ /// \brief Get sink instances for all configured sink factories.
+ /// Returns a const copy of shared_ptrs, so accessing the sinks in the
+ /// calling context is thread-safe.
+ // NOLINTNEXTLINE(readability-const-return-type)
+ [[nodiscard]] static auto Sinks() noexcept
+ -> std::vector<ILogSink::Ptr> const {
+ std::lock_guard lock{mutex_};
+ return sinks_;
+ }
+
+ /// \brief Get all configured sink factories.
+ /// Returns a const copy of shared_ptrs, so accessing the factories in the
+ /// calling context is thread-safe.
+ // NOLINTNEXTLINE(readability-const-return-type)
+ [[nodiscard]] static auto SinkFactories() noexcept
+ -> std::vector<LogSinkFactory> const {
+ std::lock_guard lock{mutex_};
+ return factories_;
+ }
+
+ private:
+ static inline std::mutex mutex_{};
+ static inline LogLevel log_limit_{LogLevel::Info};
+ static inline std::vector<ILogSink::Ptr> sinks_{};
+ static inline std::vector<LogSinkFactory> factories_{};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_CONFIG_HPP
diff --git a/src/buildtool/logging/log_level.hpp b/src/buildtool/logging/log_level.hpp
new file mode 100644
index 00000000..6847e69c
--- /dev/null
+++ b/src/buildtool/logging/log_level.hpp
@@ -0,0 +1,41 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_LEVEL_HPP
+#define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_LEVEL_HPP
+
+#include <algorithm>
+#include <string>
+#include <type_traits>
+
+enum class LogLevel {
+ Error, ///< Error messages, fatal errors
+ Warning, ///< Warning messages, recoverable situations that shouldn't occur
+ Info, ///< Informative messages, such as reporting status or statistics
+ Debug, ///< Debug messages, such as details from internal processes
+ Trace ///< Trace messages, verbose details such as function calls
+};
+
+constexpr auto kFirstLogLevel = LogLevel::Error;
+constexpr auto kLastLogLevel = LogLevel::Trace;
+
+[[nodiscard]] static inline auto ToLogLevel(
+ std::underlying_type_t<LogLevel> level) -> LogLevel {
+ return std::min(std::max(static_cast<LogLevel>(level), kFirstLogLevel),
+ kLastLogLevel);
+}
+
+[[nodiscard]] static inline auto LogLevelToString(LogLevel level)
+ -> std::string {
+ switch (level) {
+ case LogLevel::Error:
+ return "ERROR";
+ case LogLevel::Warning:
+ return "WARN";
+ case LogLevel::Info:
+ return "INFO";
+ case LogLevel::Debug:
+ return "DEBUG";
+ case LogLevel::Trace:
+ return "TRACE";
+ }
+}
+
+#endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_LEVEL_HPP
diff --git a/src/buildtool/logging/log_sink.hpp b/src/buildtool/logging/log_sink.hpp
new file mode 100644
index 00000000..3c1028cd
--- /dev/null
+++ b/src/buildtool/logging/log_sink.hpp
@@ -0,0 +1,41 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_HPP
+#define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_HPP
+
+#include <functional>
+#include <istream>
+#include <memory>
+#include <string>
+
+#include "src/buildtool/logging/log_level.hpp"
+
+// forward declaration
+class Logger;
+
+class ILogSink {
+ public:
+ using Ptr = std::shared_ptr<ILogSink>;
+ ILogSink() noexcept = default;
+ ILogSink(ILogSink const&) = delete;
+ ILogSink(ILogSink&&) = delete;
+ auto operator=(ILogSink const&) -> ILogSink& = delete;
+ auto operator=(ILogSink &&) -> ILogSink& = delete;
+ virtual ~ILogSink() noexcept = default;
+
+ /// \brief Thread-safe emitting of log messages.
+ /// Logger might be 'nullptr' if called from the global context.
+ virtual void Emit(Logger const* logger,
+ LogLevel level,
+ std::string const& msg) const noexcept = 0;
+
+ protected:
+ /// \brief Helper class for line iteration with std::istream_iterator.
+ class Line : public std::string {
+ friend auto operator>>(std::istream& is, Line& line) -> std::istream& {
+ return std::getline(is, line);
+ }
+ };
+};
+
+using LogSinkFactory = std::function<ILogSink::Ptr()>;
+
+#endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_HPP
diff --git a/src/buildtool/logging/log_sink_cmdline.hpp b/src/buildtool/logging/log_sink_cmdline.hpp
new file mode 100644
index 00000000..f7e5f915
--- /dev/null
+++ b/src/buildtool/logging/log_sink_cmdline.hpp
@@ -0,0 +1,93 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_CMDLINE_HPP
+#define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_CMDLINE_HPP
+
+#include <iterator>
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <string>
+
+#include "fmt/color.h"
+#include "fmt/core.h"
+#include "src/buildtool/logging/log_sink.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+class LogSinkCmdLine final : public ILogSink {
+ public:
+ static auto CreateFactory(bool colored = true) -> LogSinkFactory {
+ return [=]() { return std::make_shared<LogSinkCmdLine>(colored); };
+ }
+
+ explicit LogSinkCmdLine(bool colored) noexcept : colored_{colored} {}
+ ~LogSinkCmdLine() noexcept final = default;
+ LogSinkCmdLine(LogSinkCmdLine const&) noexcept = delete;
+ LogSinkCmdLine(LogSinkCmdLine&&) noexcept = delete;
+ auto operator=(LogSinkCmdLine const&) noexcept -> LogSinkCmdLine& = delete;
+ auto operator=(LogSinkCmdLine&&) noexcept -> LogSinkCmdLine& = delete;
+
+ /// \brief Thread-safe emitting of log messages to stderr.
+ void Emit(Logger const* logger,
+ LogLevel level,
+ std::string const& msg) const noexcept final {
+ auto prefix = LogLevelToString(level);
+
+ if (logger != nullptr) {
+ // append logger name
+ prefix = fmt::format("{} ({})", prefix, logger->Name());
+ }
+ prefix = prefix + ":";
+ auto cont_prefix = std::string(prefix.size(), ' ');
+ prefix = FormatPrefix(level, prefix);
+ bool msg_on_continuation{false};
+ if (logger != nullptr and msg.find('\n') != std::string::npos) {
+ cont_prefix = " ";
+ msg_on_continuation = true;
+ }
+
+ {
+ std::lock_guard lock{mutex_};
+ if (msg_on_continuation) {
+ fmt::print(stderr, "{}\n", prefix);
+ prefix = cont_prefix;
+ }
+ using it = std::istream_iterator<ILogSink::Line>;
+ std::istringstream iss{msg};
+ for_each(it{iss}, it{}, [&](auto const& line) {
+ fmt::print(stderr, "{} {}\n", prefix, line);
+ prefix = cont_prefix;
+ });
+ }
+ }
+
+ private:
+ bool colored_{};
+ static inline std::mutex mutex_{};
+
+ [[nodiscard]] auto FormatPrefix(LogLevel level,
+ std::string const& prefix) const noexcept
+ -> std::string {
+ fmt::text_style style{};
+ if (colored_) {
+ switch (level) {
+ case LogLevel::Error:
+ style = fg(fmt::color::red);
+ break;
+ case LogLevel::Warning:
+ style = fg(fmt::color::orange);
+ break;
+ case LogLevel::Info:
+ style = fg(fmt::color::lime_green);
+ break;
+ case LogLevel::Debug:
+ style = fg(fmt::color::yellow);
+ break;
+ case LogLevel::Trace:
+ style = fg(fmt::color::light_sky_blue);
+ break;
+ }
+ }
+ return fmt::format(style, "{}", prefix);
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_CMDLINE_HPP
diff --git a/src/buildtool/logging/log_sink_file.hpp b/src/buildtool/logging/log_sink_file.hpp
new file mode 100644
index 00000000..2ca1b75b
--- /dev/null
+++ b/src/buildtool/logging/log_sink_file.hpp
@@ -0,0 +1,129 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_FILE_HPP
+#define INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_FILE_HPP
+
+#include <cstdio>
+#include <filesystem>
+#include <functional>
+#include <iterator>
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+
+#ifdef __unix__
+#include <sys/time.h>
+#endif
+
+#include "fmt/chrono.h"
+#include "fmt/core.h"
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/logging/log_sink.hpp"
+#include "src/buildtool/logging/logger.hpp"
+
+/// \brief Thread-safe map of mutexes.
+template <class T_Key>
+class MutexMap {
+ public:
+ /// \brief Create mutex for key and run callback if successfully created.
+ /// Callback is executed while the internal map is still held exclusively.
+ void Create(T_Key const& key, std::function<void()> const& callback) {
+ std::lock_guard lock(mutex_);
+ if (not map_.contains(key)) {
+ [[maybe_unused]] auto& mutex = map_[key];
+ callback();
+ }
+ }
+ /// \brief Get mutex for key, creates mutex if key does not exist.
+ [[nodiscard]] auto Get(T_Key const& key) noexcept -> std::mutex& {
+ std::lock_guard lock(mutex_);
+ return map_[key];
+ }
+
+ private:
+ std::mutex mutex_{};
+ std::unordered_map<T_Key, std::mutex> map_{};
+};
+
+class LogSinkFile final : public ILogSink {
+ public:
+ enum class Mode {
+ Append, ///< Append if log file already exists.
+ Overwrite ///< Overwrite log file with each new program instantiation.
+ };
+
+ static auto CreateFactory(std::filesystem::path const& file_path,
+ Mode file_mode = Mode::Append) -> LogSinkFactory {
+ return
+ [=] { return std::make_shared<LogSinkFile>(file_path, file_mode); };
+ }
+
+ LogSinkFile(std::filesystem::path const& file_path, Mode file_mode)
+ : file_path_{std::filesystem::weakly_canonical(file_path).string()} {
+ // create file mutex for canonical path
+ file_mutexes_.Create(file_path_, [&] {
+ if (file_mode == Mode::Overwrite) {
+ // clear file contents
+ if (gsl::owner<FILE*> file =
+ std::fopen(file_path_.c_str(), "w")) {
+ std::fclose(file);
+ }
+ }
+ });
+ }
+ ~LogSinkFile() noexcept final = default;
+ LogSinkFile(LogSinkFile const&) noexcept = delete;
+ LogSinkFile(LogSinkFile&&) noexcept = delete;
+ auto operator=(LogSinkFile const&) noexcept -> LogSinkFile& = delete;
+ auto operator=(LogSinkFile&&) noexcept -> LogSinkFile& = delete;
+
+ /// \brief Thread-safe emitting of log messages to file.
+ /// Race-conditions for file writes are resolved via a separate mutexes for
+ /// every canonical file path shared across all instances of this class.
+ void Emit(Logger const* logger,
+ LogLevel level,
+ std::string const& msg) const noexcept final {
+#ifdef __unix__ // support nanoseconds for timestamp
+ timespec ts{};
+ clock_gettime(CLOCK_REALTIME, &ts);
+ auto timestamp = fmt::format(
+ "{:%Y-%m-%d %H:%M:%S}.{}", fmt::localtime(ts.tv_sec), ts.tv_nsec);
+#else
+ auto timestamp = fmt::format(
+ "{:%Y-%m-%d %H:%M:%S}", fmt::localtime(std::time(nullptr));
+#endif
+
+ std::ostringstream id{};
+ id << "thread:" << std::this_thread::get_id();
+ auto thread = id.str();
+
+ auto prefix = fmt::format(
+ "{}, [{}] {}", thread, timestamp, LogLevelToString(level));
+
+ if (logger != nullptr) {
+ // append logger name
+ prefix = fmt::format("{} ({})", prefix, logger->Name());
+ }
+ prefix = fmt::format("{}:", prefix);
+ auto cont_prefix = std::string(prefix.size(), ' ');
+
+ {
+ std::lock_guard lock{file_mutexes_.Get(file_path_)};
+ if (gsl::owner<FILE*> file = std::fopen(file_path_.c_str(), "a")) {
+ using it = std::istream_iterator<ILogSink::Line>;
+ std::istringstream iss{msg};
+ for_each(it{iss}, it{}, [&](auto const& line) {
+ fmt::print(file, "{} {}\n", prefix, line);
+ prefix = cont_prefix;
+ });
+ std::fclose(file);
+ }
+ }
+ }
+
+ private:
+ std::string file_path_{};
+ static inline MutexMap<std::string> file_mutexes_{};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOG_SINK_FILE_HPP
diff --git a/src/buildtool/logging/logger.hpp b/src/buildtool/logging/logger.hpp
new file mode 100644
index 00000000..60742607
--- /dev/null
+++ b/src/buildtool/logging/logger.hpp
@@ -0,0 +1,123 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_LOGGING_LOGGER_HPP
+#define INCLUDED_SRC_BUILDTOOL_LOGGING_LOGGER_HPP
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "fmt/core.h"
+#include "src/buildtool/logging/log_config.hpp"
+#include "src/buildtool/logging/log_sink.hpp"
+
+class Logger {
+ public:
+ using MessageCreateFunc = std::function<std::string()>;
+
+ /// \brief Create logger with sink instances from LogConfig::Sinks().
+ explicit Logger(std::string name) noexcept
+ : name_{std::move(name)},
+ log_limit_{LogConfig::LogLimit()},
+ sinks_{LogConfig::Sinks()} {}
+
+ /// \brief Create logger with new sink instances from specified factories.
+ Logger(std::string name,
+ std::vector<LogSinkFactory> const& factories) noexcept
+ : name_{std::move(name)}, log_limit_{LogConfig::LogLimit()} {
+ sinks_.reserve(factories.size());
+ std::transform(factories.cbegin(),
+ factories.cend(),
+ std::back_inserter(sinks_),
+ [](auto& f) { return f(); });
+ }
+
+ ~Logger() noexcept = default;
+ Logger(Logger const&) noexcept = delete;
+ Logger(Logger&&) noexcept = delete;
+ auto operator=(Logger const&) noexcept -> Logger& = delete;
+ auto operator=(Logger&&) noexcept -> Logger& = delete;
+
+ /// \brief Get logger name.
+ [[nodiscard]] auto Name() const& noexcept -> std::string const& {
+ return name_;
+ }
+
+ /// \brief Get log limit.
+ [[nodiscard]] auto LogLimit() const noexcept -> LogLevel {
+ return log_limit_;
+ }
+
+ /// \brief Set log limit.
+ void SetLogLimit(LogLevel level) noexcept { log_limit_ = level; }
+
+ /// \brief Emit log message from string via this logger instance.
+ template <class... T_Args>
+ void Emit(LogLevel level,
+ std::string const& msg,
+ T_Args&&... args) const noexcept {
+ if (static_cast<int>(level) <= static_cast<int>(log_limit_)) {
+ FormatAndForward(
+ this, sinks_, level, msg, std::forward<T_Args>(args)...);
+ }
+ }
+
+ /// \brief Emit log message from lambda via this logger instance.
+ void Emit(LogLevel level,
+ MessageCreateFunc const& msg_creator) const noexcept {
+ if (static_cast<int>(level) <= static_cast<int>(log_limit_)) {
+ FormatAndForward(this, sinks_, level, msg_creator());
+ }
+ }
+
+ /// \brief Log message from string via LogConfig's sinks and log limit.
+ template <class... T_Args>
+ static void Log(LogLevel level,
+ std::string const& msg,
+ T_Args&&... args) noexcept {
+ if (static_cast<int>(level) <=
+ static_cast<int>(LogConfig::LogLimit())) {
+ FormatAndForward(nullptr,
+ LogConfig::Sinks(),
+ level,
+ msg,
+ std::forward<T_Args>(args)...);
+ }
+ }
+
+ /// \brief Log message from lambda via LogConfig's sinks and log limit.
+ static void Log(LogLevel level,
+ MessageCreateFunc const& msg_creator) noexcept {
+ if (static_cast<int>(level) <=
+ static_cast<int>(LogConfig::LogLimit())) {
+ FormatAndForward(nullptr, LogConfig::Sinks(), level, msg_creator());
+ }
+ }
+
+ private:
+ std::string name_{};
+ LogLevel log_limit_{};
+ std::vector<ILogSink::Ptr> sinks_{};
+
+ /// \brief Format message and forward to sinks.
+ template <class... T_Args>
+ static void FormatAndForward(Logger const* logger,
+ std::vector<ILogSink::Ptr> const& sinks,
+ LogLevel level,
+ std::string const& msg,
+ T_Args&&... args) noexcept {
+ if constexpr (sizeof...(T_Args) == 0) {
+ // forward to sinks
+ std::for_each(sinks.cbegin(), sinks.cend(), [&](auto& sink) {
+ sink->Emit(logger, level, msg);
+ });
+ }
+ else {
+ // format the message
+ auto fmsg = fmt::format(msg, std::forward<T_Args>(args)...);
+ // recursive call without format arguments
+ FormatAndForward(logger, sinks, level, fmsg);
+ }
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_LOGGING_LOGGER_HPP
diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS
new file mode 100644
index 00000000..ad4ab98a
--- /dev/null
+++ b/src/buildtool/main/TARGETS
@@ -0,0 +1,21 @@
+{ "just":
+ { "type": ["@", "rules", "CC", "binary"]
+ , "name": ["just"]
+ , "srcs": ["main.cpp"]
+ , "private-hdrs": ["main.hpp"]
+ , "deps":
+ [ ["src/buildtool/common", "cli"]
+ , ["src/buildtool/common", "config"]
+ , ["src/buildtool/graph_traverser", "graph_traverser"]
+ , ["src/buildtool/logging", "logging"]
+ , ["src/buildtool/build_engine/base_maps", "directory_map"]
+ , ["src/buildtool/build_engine/base_maps", "rule_map"]
+ , ["src/buildtool/build_engine/base_maps", "source_map"]
+ , ["src/buildtool/build_engine/base_maps", "targets_file_map"]
+ , ["src/buildtool/build_engine/target_map", "target_map"]
+ , ["src/utils/cpp", "concepts"]
+ , ["src/utils/cpp", "json"]
+ ]
+ , "stage": ["src", "buildtool", "main"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
new file mode 100644
index 00000000..74d75d56
--- /dev/null
+++ b/src/buildtool/main/main.cpp
@@ -0,0 +1,1292 @@
+#include "src/buildtool/main/main.hpp"
+
+#include <algorithm>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <string>
+
+#include "src/buildtool/build_engine/base_maps/directory_map.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name.hpp"
+#include "src/buildtool/build_engine/base_maps/expression_map.hpp"
+#include "src/buildtool/build_engine/base_maps/rule_map.hpp"
+#include "src/buildtool/build_engine/base_maps/source_map.hpp"
+#include "src/buildtool/build_engine/base_maps/targets_file_map.hpp"
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/build_engine/target_map/target_map.hpp"
+#include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/common/cli.hpp"
+#include "src/buildtool/common/repository_config.hpp"
+#ifndef BOOTSTRAP_BUILD_TOOL
+#include "src/buildtool/graph_traverser/graph_traverser.hpp"
+#endif
+#include "src/buildtool/logging/log_config.hpp"
+#include "src/buildtool/logging/log_sink_cmdline.hpp"
+#include "src/buildtool/logging/log_sink_file.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+#include "src/buildtool/multithreading/task_system.hpp"
+#include "src/utils/cpp/concepts.hpp"
+#include "src/utils/cpp/json.hpp"
+
+namespace {
+
+namespace Base = BuildMaps::Base;
+namespace Target = BuildMaps::Target;
+
+enum class SubCommand {
+ kUnknown,
+ kDescribe,
+ kAnalyse,
+ kBuild,
+ kInstall,
+ kRebuild,
+ kInstallCas,
+ kTraverse
+};
+
+struct CommandLineArguments {
+ SubCommand cmd{SubCommand::kUnknown};
+ CommonArguments common;
+ AnalysisArguments analysis;
+ DiagnosticArguments diagnose;
+ EndpointArguments endpoint;
+ BuildArguments build;
+ StageArguments stage;
+ RebuildArguments rebuild;
+ FetchArguments fetch;
+ GraphArguments graph;
+};
+
+/// \brief Setup arguments for sub command "just describe".
+auto SetupDescribeCommandArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupCommonArguments(app, &clargs->common);
+ SetupAnalysisArguments(app, &clargs->analysis, false);
+}
+
+/// \brief Setup arguments for sub command "just analyse".
+auto SetupAnalyseCommandArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupCommonArguments(app, &clargs->common);
+ SetupAnalysisArguments(app, &clargs->analysis);
+ SetupDiagnosticArguments(app, &clargs->diagnose);
+}
+
+/// \brief Setup arguments for sub command "just build".
+auto SetupBuildCommandArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupCommonArguments(app, &clargs->common);
+ SetupAnalysisArguments(app, &clargs->analysis);
+ SetupEndpointArguments(app, &clargs->endpoint);
+ SetupBuildArguments(app, &clargs->build);
+}
+
+/// \brief Setup arguments for sub command "just install".
+auto SetupInstallCommandArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupBuildCommandArguments(app, clargs); // same as build
+ SetupStageArguments(app, &clargs->stage); // plus stage
+}
+
+/// \brief Setup arguments for sub command "just rebuild".
+auto SetupRebuildCommandArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupBuildCommandArguments(app, clargs); // same as build
+ SetupRebuildArguments(app, &clargs->rebuild); // plus rebuild
+}
+
+/// \brief Setup arguments for sub command "just install-cas".
+auto SetupInstallCasCommandArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupEndpointArguments(app, &clargs->endpoint);
+ SetupFetchArguments(app, &clargs->fetch);
+}
+
+/// \brief Setup arguments for sub command "just traverse".
+auto SetupTraverseCommandArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupCommonArguments(app, &clargs->common);
+ SetupEndpointArguments(app, &clargs->endpoint);
+ SetupGraphArguments(app, &clargs->graph); // instead of analysis
+ SetupBuildArguments(app, &clargs->build);
+ SetupStageArguments(app, &clargs->stage);
+}
+
+auto ParseCommandLineArguments(int argc, char const* const* argv)
+ -> CommandLineArguments {
+ CLI::App app("just");
+ app.option_defaults()->take_last();
+
+ auto* cmd_describe = app.add_subcommand(
+ "describe", "Describe the rule generating a target.");
+ auto* cmd_analyse =
+ app.add_subcommand("analyse", "Analyse specified targets.");
+ auto* cmd_build = app.add_subcommand("build", "Build specified targets.");
+ auto* cmd_install =
+ app.add_subcommand("install", "Build and stage specified targets.");
+ auto* cmd_rebuild = app.add_subcommand(
+ "rebuild", "Rebuild and compare artifacts to cached build.");
+ auto* cmd_install_cas =
+ app.add_subcommand("install-cas", "Fetch and stage artifact from CAS.");
+ auto* cmd_traverse =
+ app.group("") // group for creating hidden options
+ ->add_subcommand("traverse",
+ "Build and stage artifacts from graph file.");
+ app.require_subcommand(1);
+
+ CommandLineArguments clargs;
+ SetupDescribeCommandArguments(cmd_describe, &clargs);
+ SetupAnalyseCommandArguments(cmd_analyse, &clargs);
+ SetupBuildCommandArguments(cmd_build, &clargs);
+ SetupInstallCommandArguments(cmd_install, &clargs);
+ SetupRebuildCommandArguments(cmd_rebuild, &clargs);
+ SetupInstallCasCommandArguments(cmd_install_cas, &clargs);
+ SetupTraverseCommandArguments(cmd_traverse, &clargs);
+
+ try {
+ app.parse(argc, argv);
+ } catch (CLI::Error& e) {
+ std::exit(app.exit(e));
+ }
+
+ if (*cmd_describe) {
+ clargs.cmd = SubCommand::kDescribe;
+ }
+ else if (*cmd_analyse) {
+ clargs.cmd = SubCommand::kAnalyse;
+ }
+ else if (*cmd_build) {
+ clargs.cmd = SubCommand::kBuild;
+ }
+ else if (*cmd_install) {
+ clargs.cmd = SubCommand::kInstall;
+ }
+ else if (*cmd_rebuild) {
+ clargs.cmd = SubCommand::kRebuild;
+ }
+ else if (*cmd_install_cas) {
+ clargs.cmd = SubCommand::kInstallCas;
+ }
+ else if (*cmd_traverse) {
+ clargs.cmd = SubCommand::kTraverse;
+ }
+
+ return clargs;
+}
+
+void SetupLogging(CommonArguments const& clargs) {
+ LogConfig::SetLogLimit(clargs.log_limit);
+ LogConfig::SetSinks({LogSinkCmdLine::CreateFactory()});
+ if (clargs.log_file) {
+ LogConfig::AddSink(LogSinkFile::CreateFactory(
+ *clargs.log_file, LogSinkFile::Mode::Overwrite));
+ }
+}
+
+#ifndef BOOTSTRAP_BUILD_TOOL
+void SetupLocalExecution(EndpointArguments const& eargs,
+ BuildArguments const& bargs) {
+ using LocalConfig = LocalExecutionConfig;
+ if (not LocalConfig::SetKeepBuildDir(bargs.persistent_build_dir) or
+ not(not eargs.local_root or
+ (LocalConfig::SetBuildRoot(*eargs.local_root) and
+ LocalConfig::SetDiskCache(*eargs.local_root))) or
+ not(not bargs.local_launcher or
+ LocalConfig::SetLauncher(*bargs.local_launcher))) {
+ Logger::Log(LogLevel::Error, "failed to configure local execution.");
+ }
+}
+#endif
+
+// returns path relative to `root`.
+[[nodiscard]] auto FindRoot(std::filesystem::path const& subdir,
+ FileRoot const& root,
+ std::vector<std::string> const& markers)
+ -> std::optional<std::filesystem::path> {
+ gsl_Expects(subdir.is_relative());
+ auto current = subdir;
+ while (true) {
+ for (auto const& marker : markers) {
+ if (root.Exists(current / marker)) {
+ return current;
+ }
+ }
+ if (current.empty()) {
+ break;
+ }
+ current = current.parent_path();
+ }
+ return std::nullopt;
+}
+
+[[nodiscard]] auto ReadConfiguration(AnalysisArguments const& clargs) noexcept
+ -> Configuration {
+ if (not clargs.config_file.empty()) {
+ if (not std::filesystem::exists(clargs.config_file)) {
+ Logger::Log(LogLevel::Error,
+ "Config file {} does not exist.",
+ clargs.config_file.string());
+ std::exit(kExitFailure);
+ }
+ try {
+ std::ifstream fs(clargs.config_file);
+ auto map = Expression::FromJson(nlohmann::json::parse(fs));
+ if (not map->IsMap()) {
+ Logger::Log(LogLevel::Error,
+ "Config file {} does not contain a map.",
+ clargs.config_file.string());
+ std::exit(kExitFailure);
+ }
+ return Configuration{map};
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "Parsing config file {} failed with error:\n{}",
+ clargs.config_file.string(),
+ e.what());
+ std::exit(kExitFailure);
+ }
+ }
+ return Configuration{};
+}
+
+[[nodiscard]] auto DetermineCurrentModule(
+ std::filesystem::path const& workspace_root,
+ FileRoot const& target_root,
+ std::optional<std::string> const& target_file_name_opt) -> std::string {
+ auto cwd = std::filesystem::current_path();
+ auto subdir = std::filesystem::proximate(cwd, workspace_root);
+ if (subdir.is_relative() and (*subdir.begin() != "..")) {
+ // cwd is subdir of workspace_root
+ std::string target_file_name =
+ target_file_name_opt ? *target_file_name_opt : "TARGETS";
+ if (auto root_dir = FindRoot(subdir, target_root, {target_file_name})) {
+ return root_dir->string();
+ }
+ }
+ return ".";
+}
+
+[[nodiscard]] auto ReadConfiguredTarget(
+ AnalysisArguments const& clargs,
+ std::string const& main_repo,
+ std::optional<std::filesystem::path> const& main_ws_root)
+ -> Target::ConfiguredTarget {
+ auto const* target_root =
+ RepositoryConfig::Instance().TargetRoot(main_repo);
+ if (target_root == nullptr) {
+ Logger::Log(LogLevel::Error,
+ "Cannot obtain target root for main repo {}.",
+ main_repo);
+ std::exit(kExitFailure);
+ }
+ auto current_module = std::string{"."};
+ if (main_ws_root) {
+ // module detection only works if main workspace is on the file system
+ current_module = DetermineCurrentModule(
+ *main_ws_root, *target_root, clargs.target_file_name);
+ }
+ auto config = ReadConfiguration(clargs);
+ if (clargs.target) {
+ auto entity = Base::ParseEntityNameFromJson(
+ *clargs.target,
+ Base::EntityName{main_repo, current_module, ""},
+ [&clargs](std::string const& parse_err) {
+ Logger::Log(LogLevel::Error,
+ "Parsing target name {} failed with:\n{}.",
+ clargs.target->dump(),
+ parse_err);
+ });
+ if (not entity) {
+ std::exit(kExitFailure);
+ }
+ return Target::ConfiguredTarget{std::move(*entity), std::move(config)};
+ }
+ std::string target_file_name =
+ clargs.target_file_name ? *clargs.target_file_name : "TARGETS";
+ auto const target_file =
+ (std::filesystem::path{current_module} / target_file_name).string();
+ auto file_content = target_root->ReadFile(target_file);
+ if (not file_content) {
+ Logger::Log(LogLevel::Error, "Cannot read file {}.", target_file);
+ std::exit(kExitFailure);
+ }
+ auto const json = nlohmann::json::parse(*file_content);
+ if (not json.is_object()) {
+ Logger::Log(
+ LogLevel::Error, "Invalid content in target file {}.", target_file);
+ std::exit(kExitFailure);
+ }
+ if (json.empty()) {
+ Logger::Log(LogLevel::Error,
+ "Missing target descriptions in file {}.",
+ target_file);
+ std::exit(kExitFailure);
+ }
+ return Target::ConfiguredTarget{
+ Base::EntityName{main_repo, current_module, json.begin().key()},
+ std::move(config)};
+}
+
+template <HasToString K, typename V>
+[[nodiscard]] auto DetectAndReportCycle(std::string const& name,
+ AsyncMapConsumer<K, V> const& map)
+ -> bool {
+ using namespace std::string_literals;
+ auto cycle = map.DetectCycle();
+ if (cycle) {
+ bool found{false};
+ std::ostringstream oss{};
+ oss << fmt::format("Cycle detected in {}:", name) << std::endl;
+ for (auto const& k : *cycle) {
+ auto match = (k == cycle->back());
+ auto prefix{match ? found ? "`-- "s : ".-> "s
+ : found ? "| "s : " "s};
+ oss << prefix << k.ToString() << std::endl;
+ found = found or match;
+ }
+ Logger::Log(LogLevel::Error, "{}", oss.str());
+ return true;
+ }
+ return false;
+}
+
+template <HasToString K, typename V>
+void DetectAndReportPending(std::string const& name,
+ AsyncMapConsumer<K, V> const& map) {
+ using namespace std::string_literals;
+ auto keys = map.GetPendingKeys();
+ if (not keys.empty()) {
+ std::ostringstream oss{};
+ oss << fmt::format("Internal error, failed to evaluate pending {}:",
+ name)
+ << std::endl;
+ for (auto const& k : keys) {
+ oss << " " << k.ToString() << std::endl;
+ }
+ Logger::Log(LogLevel::Error, "{}", oss.str());
+ }
+}
+
+std::vector<std::string> const kRootMarkers{"ROOT", "WORKSPACE", ".git"};
+
+[[nodiscard]] auto DetermineWorkspaceRoot(CommonArguments const& clargs)
+ -> std::filesystem::path {
+ if (clargs.workspace_root) {
+ return *clargs.workspace_root;
+ }
+ auto cwd = std::filesystem::current_path();
+ auto root = cwd.root_path();
+ cwd = std::filesystem::relative(cwd, root);
+ auto root_dir = FindRoot(cwd, FileRoot{root}, kRootMarkers);
+ if (not root_dir) {
+ Logger::Log(LogLevel::Error, "Could not determine workspace root.");
+ std::exit(kExitFailure);
+ }
+ return root / *root_dir;
+}
+
+// returns FileRoot and optional local path, if the root is local
+auto ParseRoot(nlohmann::json desc,
+ const std::string& repo,
+ const std::string& keyword)
+ -> std::pair<FileRoot, std::optional<std::filesystem::path>> {
+ nlohmann::json root = desc[keyword];
+ if ((not root.is_array()) or root.empty()) {
+ Logger::Log(LogLevel::Error,
+ "Expected {} for {} to be of the form [<scheme>, ...], but "
+ "found {}",
+ keyword,
+ repo,
+ root.dump());
+ std::exit(kExitFailure);
+ }
+ if (root[0] == "file") {
+ if (root.size() != 2 or (not root[1].is_string())) {
+ Logger::Log(LogLevel::Error,
+ "\"file\" scheme expects precisely one string "
+ "argument, but found {} for {} of repository {}",
+ root.dump(),
+ keyword,
+ repo);
+ std::exit(kExitFailure);
+ }
+ auto path = std::filesystem::path{root[1]};
+ return {FileRoot{path}, std::move(path)};
+ }
+ if (root[0] == "git tree") {
+ if (root.size() != 3 or (not root[1].is_string()) or
+ (not root[2].is_string())) {
+ Logger::Log(LogLevel::Error,
+ "\"git tree\" scheme expects two string arguments, "
+ "but found {} for {} of repository {}",
+ root.dump(),
+ keyword,
+ repo);
+ std::exit(kExitFailure);
+ }
+ if (auto git_root = FileRoot::FromGit(root[2], root[1])) {
+ return {std::move(*git_root), std::nullopt};
+ }
+ Logger::Log(LogLevel::Error,
+ "Could not create file root for git repository {} and tree "
+ "id {}",
+ root[2],
+ root[1]);
+ std::exit(kExitFailure);
+ }
+ Logger::Log(LogLevel::Error,
+ "Unknown scheme in the specification {} of {} of repository {}",
+ root.dump(),
+ keyword,
+ repo);
+ std::exit(kExitFailure);
+}
+
+// Set all roots and name mappings from the command-line arguments and
+// return the name of the main repository and main workspace path if local.
+auto DetermineRoots(CommonArguments cargs, AnalysisArguments aargs)
+ -> std::pair<std::string, std::optional<std::filesystem::path>> {
+ std::optional<std::filesystem::path> main_ws_root;
+ auto repo_config = nlohmann::json::object();
+ if (cargs.repository_config) {
+ try {
+ std::ifstream fs(*cargs.repository_config);
+ repo_config = nlohmann::json::parse(fs);
+ if (not repo_config.is_object()) {
+ Logger::Log(
+ LogLevel::Error,
+ "Repository configuration file {} does not contain a map.",
+ (*cargs.repository_config).string());
+ std::exit(kExitFailure);
+ }
+ } catch (std::exception const& e) {
+ Logger::Log(LogLevel::Error,
+ "Parsing repository configuration file {} failed with "
+ "error:\n{}",
+ (*cargs.repository_config).string(),
+ e.what());
+ std::exit(kExitFailure);
+ }
+ }
+
+ std::string main_repo;
+
+ auto main_it = repo_config.find("main");
+ if (main_it != repo_config.end()) {
+ if (not main_it->is_string()) {
+ Logger::Log(LogLevel::Error,
+ "Repository config: main has to be a string");
+ std::exit(kExitFailure);
+ }
+ main_repo = *main_it;
+ }
+ if (cargs.main) {
+ main_repo = *cargs.main;
+ }
+
+ auto repos = nlohmann::json::object();
+ auto repos_it = repo_config.find("repositories");
+ if (repos_it != repo_config.end()) {
+ if (not repos_it->is_object()) {
+ Logger::Log(LogLevel::Error,
+ "Repository config: repositories has to be a map");
+ std::exit(kExitFailure);
+ }
+ repos = *repos_it;
+ }
+ if (not repos.contains(main_repo)) {
+ repos[main_repo] = nlohmann::json::object();
+ }
+
+ for (auto const& [repo, desc] : repos.items()) {
+ FileRoot ws_root{};
+
+ if (desc.contains("workspace_root")) {
+ auto [root, path] = ParseRoot(desc, repo, "workspace_root");
+ ws_root = std::move(root);
+ if (repo == main_repo) {
+ main_ws_root = std::move(path);
+ }
+ }
+ else if (repo == main_repo) {
+ main_ws_root = DetermineWorkspaceRoot(cargs);
+ ws_root = FileRoot{*main_ws_root};
+ }
+ else {
+ Logger::Log(
+ LogLevel::Error, "Unknown root for repository {}", repo);
+ std::exit(kExitFailure);
+ }
+ // TODO(aehlig): Handle root-naming scheme. So far, we assume ["file",
+ // dir] without checking.
+ auto info = RepositoryConfig::RepositoryInfo{std::move(ws_root)};
+ if (desc.contains("target_root")) {
+ info.target_root = ParseRoot(desc, repo, "target_root").first;
+ }
+ if (repo == main_repo && aargs.target_root) {
+ info.target_root = FileRoot{*aargs.target_root};
+ }
+ info.rule_root = info.target_root;
+ if (desc.contains("rule_root")) {
+ info.rule_root = ParseRoot(desc, repo, "rule_root").first;
+ }
+ if (repo == main_repo && aargs.rule_root) {
+ info.rule_root = FileRoot{*aargs.rule_root};
+ }
+ info.expression_root = info.rule_root;
+ if (desc.contains("expression_root")) {
+ info.expression_root =
+ ParseRoot(desc, repo, "expression_root").first;
+ }
+ if (repo == main_repo && aargs.expression_root) {
+ info.expression_root = FileRoot{*aargs.expression_root};
+ }
+
+ if (desc.contains("bindings")) {
+ if (not desc["bindings"].is_object()) {
+ Logger::Log(
+ LogLevel::Error,
+ "bindings has to be a string-string map, but found {}",
+ desc["bindings"].dump());
+ std::exit(kExitFailure);
+ }
+ for (auto const& [local_name, global_name] :
+ desc["bindings"].items()) {
+ if (not repos.contains(global_name)) {
+ Logger::Log(LogLevel::Error,
+ "Binding {} for {} in {} does not refer to a "
+ "defined repository.",
+ global_name,
+ local_name,
+ repo);
+ std::exit(kExitFailure);
+ }
+ info.name_mapping[local_name] = global_name;
+ }
+ }
+
+ if (desc.contains("target_file_name")) {
+ info.target_file_name = desc["target_file_name"];
+ }
+ if (repo == main_repo && aargs.target_file_name) {
+ info.target_file_name = *aargs.target_file_name;
+ }
+ if (desc.contains("rule_file_name")) {
+ info.rule_file_name = desc["rule_file_name"];
+ }
+ if (repo == main_repo && aargs.rule_file_name) {
+ info.rule_file_name = *aargs.rule_file_name;
+ }
+ if (desc.contains("expression_file_name")) {
+ info.expression_file_name = desc["expression_file_name"];
+ }
+ if (repo == main_repo && aargs.expression_file_name) {
+ info.expression_file_name = *aargs.expression_file_name;
+ }
+
+ RepositoryConfig::Instance().SetInfo(repo, std::move(info));
+ }
+
+ return {main_repo, main_ws_root};
+}
+
+struct AnalysisResult {
+ Target::ConfiguredTarget id;
+ AnalysedTargetPtr target;
+};
+
+[[nodiscard]] auto AnalyseTarget(
+ gsl::not_null<Target::ResultTargetMap*> const& result_map,
+ std::string const& main_repo,
+ std::optional<std::filesystem::path> const& main_ws_root,
+ std::size_t jobs,
+ AnalysisArguments const& clargs) -> std::optional<AnalysisResult> {
+ auto directory_entries = Base::CreateDirectoryEntriesMap(jobs);
+ auto expressions_file_map = Base::CreateExpressionFileMap(jobs);
+ auto rule_file_map = Base::CreateRuleFileMap(jobs);
+ auto targets_file_map = Base::CreateTargetsFileMap(jobs);
+ auto expr_map = Base::CreateExpressionMap(&expressions_file_map, jobs);
+ auto rule_map = Base::CreateRuleMap(&rule_file_map, &expr_map, jobs);
+ auto source_targets = Base::CreateSourceTargetMap(&directory_entries, jobs);
+ auto target_map = Target::CreateTargetMap(
+ &source_targets, &targets_file_map, &rule_map, result_map, jobs);
+
+ auto id = ReadConfiguredTarget(clargs, main_repo, main_ws_root);
+ std::shared_ptr<AnalysedTarget> target{};
+
+ bool failed{false};
+ {
+ TaskSystem ts{jobs};
+ target_map.ConsumeAfterKeysReady(
+ &ts,
+ {id},
+ [&target](auto values) { target = *values[0]; },
+ [&failed](auto const& msg, bool fatal) {
+ Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning,
+ "While processing targets:\n{}",
+ msg);
+ failed = failed or fatal;
+ });
+ }
+
+ if (failed) {
+ return std::nullopt;
+ }
+
+ if (not target) {
+ Logger::Log(
+ LogLevel::Error, "Failed to analyse target: {}", id.ToString());
+ if (not(DetectAndReportCycle("expression imports", expr_map) or
+ DetectAndReportCycle("target dependencies", target_map))) {
+ DetectAndReportPending("expressions", expr_map);
+ DetectAndReportPending("targets", expr_map);
+ }
+ return std::nullopt;
+ }
+
+ // Clean up in parallel what is no longer needed
+ {
+ TaskSystem ts{jobs};
+ target_map.Clear(&ts);
+ source_targets.Clear(&ts);
+ directory_entries.Clear(&ts);
+ expressions_file_map.Clear(&ts);
+ rule_file_map.Clear(&ts);
+ targets_file_map.Clear(&ts);
+ expr_map.Clear(&ts);
+ rule_map.Clear(&ts);
+ }
+
+ return AnalysisResult{id, target};
+}
+
+[[nodiscard]] auto ResultToJson(TargetResult const& result) -> nlohmann::json {
+ std::vector<std::string> artifacts{};
+ std::vector<std::string> runfiles{};
+ artifacts.reserve(result.artifact_stage->Map().size());
+ runfiles.reserve(result.runfiles->Map().size());
+ auto get_key = [](std::pair<std::string, ExpressionPtr> const& entry) {
+ return entry.first;
+ };
+ std::transform(result.artifact_stage->Map().begin(),
+ result.artifact_stage->Map().end(),
+ std::back_inserter(artifacts),
+ get_key);
+ std::transform(result.runfiles->Map().begin(),
+ result.runfiles->Map().end(),
+ std::back_inserter(runfiles),
+ get_key);
+ return nlohmann::ordered_json{
+ {"artifacts", artifacts},
+ {"runfiles", runfiles},
+ {"provides",
+ result.provides->ToJson(Expression::JsonMode::SerializeAllButNodes)}};
+}
+
+[[nodiscard]] auto TargetActionsToJson(AnalysedTargetPtr const& target)
+ -> nlohmann::json {
+ auto actions = nlohmann::json::array();
+ std::for_each(
+ target->Actions().begin(),
+ target->Actions().end(),
+ [&actions](auto const& action) { actions.push_back(action.ToJson()); });
+ return actions;
+}
+
+[[nodiscard]] auto TreesToJson(AnalysedTargetPtr const& target)
+ -> nlohmann::json {
+ auto trees = nlohmann::json::object();
+ std::for_each(
+ target->Trees().begin(),
+ target->Trees().end(),
+ [&trees](auto const& tree) { trees[tree.Id()] = tree.ToJson(); });
+
+ return trees;
+}
+
+void DumpActions(std::string const& file_path, AnalysisResult const& result) {
+ auto const dump_string =
+ IndentListsOnlyUntilDepth(TargetActionsToJson(result.target), 2, 1);
+ if (file_path == "-") {
+ Logger::Log(
+ LogLevel::Info, "Actions for target {}:", result.id.ToString());
+ std::cout << dump_string << std::endl;
+ }
+ else {
+ Logger::Log(LogLevel::Info,
+ "Dumping actions for target {} to file '{}'.",
+ result.id.ToString(),
+ file_path);
+ std::ofstream os(file_path);
+ os << dump_string << std::endl;
+ }
+}
+
+void DumpBlobs(std::string const& file_path, AnalysisResult const& result) {
+ auto blobs = nlohmann::json::array();
+ for (auto const& s : result.target->Blobs()) {
+ blobs.push_back(s);
+ }
+ auto const dump_string = blobs.dump(2);
+ if (file_path == "-") {
+ Logger::Log(
+ LogLevel::Info, "Blobs for target {}:", result.id.ToString());
+ std::cout << dump_string << std::endl;
+ }
+ else {
+ Logger::Log(LogLevel::Info,
+ "Dumping blobs for target {} to file '{}'.",
+ result.id.ToString(),
+ file_path);
+ std::ofstream os(file_path);
+ os << dump_string << std::endl;
+ }
+}
+
+void DumpTrees(std::string const& file_path, AnalysisResult const& result) {
+ auto const dump_string = TreesToJson(result.target).dump(2);
+ if (file_path == "-") {
+ Logger::Log(
+ LogLevel::Info, "Trees for target {}:", result.id.ToString());
+ std::cout << dump_string << std::endl;
+ }
+ else {
+ Logger::Log(LogLevel::Info,
+ "Dumping trees for target {} to file '{}'.",
+ result.id.ToString(),
+ file_path);
+ std::ofstream os(file_path);
+ os << dump_string << std::endl;
+ }
+}
+
+void DumpTargets(std::string const& file_path,
+ std::vector<Target::ConfiguredTarget> const& target_ids) {
+ auto repo_map = nlohmann::json::object();
+ auto conf_list =
+ [&repo_map](Base::EntityName const& ref) -> nlohmann::json& {
+ if (ref.IsAnonymousTarget()) {
+ auto& anon_map = repo_map[Base::EntityName::kAnonymousMarker];
+ auto& rule_map = anon_map[ref.anonymous->rule_map.ToIdentifier()];
+ return rule_map[ref.anonymous->target_node.ToIdentifier()];
+ }
+ auto& location_map = repo_map[Base::EntityName::kLocationMarker];
+ auto& module_map = location_map[ref.repository];
+ auto& target_map = module_map[ref.module];
+ return target_map[ref.name];
+ };
+ std::for_each(
+ target_ids.begin(), target_ids.end(), [&conf_list](auto const& id) {
+ conf_list(id.target).push_back(id.config.ToJson());
+ });
+ auto const dump_string = IndentListsOnlyUntilDepth(repo_map, 2);
+ if (file_path == "-") {
+ Logger::Log(LogLevel::Info, "List of analysed targets:");
+ std::cout << dump_string << std::endl;
+ }
+ else {
+ Logger::Log(LogLevel::Info,
+ "Dumping list of analysed targets to file '{}'.",
+ file_path);
+ std::ofstream os(file_path);
+ os << dump_string << std::endl;
+ }
+}
+
+auto DumpExpressionToMap(gsl::not_null<nlohmann::json*> const& map,
+ ExpressionPtr const& expr) -> bool {
+ auto const& id = expr->ToIdentifier();
+ if (not map->contains(id)) {
+ (*map)[id] = expr->ToJson();
+ return true;
+ }
+ return false;
+}
+
+void DumpNodesInExpressionToMap(gsl::not_null<nlohmann::json*> const& map,
+ ExpressionPtr const& expr) {
+ if (expr->IsNode()) {
+ if (DumpExpressionToMap(map, expr)) {
+ auto const& node = expr->Node();
+ if (node.IsAbstract()) {
+ DumpNodesInExpressionToMap(map,
+ node.GetAbstract().target_fields);
+ }
+ else if (node.IsValue()) {
+ DumpNodesInExpressionToMap(map, node.GetValue());
+ }
+ }
+ }
+ else if (expr->IsList()) {
+ for (auto const& entry : expr->List()) {
+ DumpNodesInExpressionToMap(map, entry);
+ }
+ }
+ else if (expr->IsMap()) {
+ for (auto const& [_, value] : expr->Map()) {
+ DumpNodesInExpressionToMap(map, value);
+ }
+ }
+ else if (expr->IsResult()) {
+ DumpNodesInExpressionToMap(map, expr->Result().provides);
+ }
+}
+
+void DumpAnonymous(std::string const& file_path,
+ std::vector<Target::ConfiguredTarget> const& target_ids) {
+ auto anon_map = nlohmann::json{{"nodes", nlohmann::json::object()},
+ {"rule_maps", nlohmann::json::object()}};
+ std::for_each(
+ target_ids.begin(), target_ids.end(), [&anon_map](auto const& id) {
+ if (id.target.IsAnonymousTarget()) {
+ DumpExpressionToMap(&anon_map["rule_maps"],
+ id.target.anonymous->rule_map);
+ DumpNodesInExpressionToMap(&anon_map["nodes"],
+ id.target.anonymous->target_node);
+ }
+ });
+ auto const dump_string = IndentListsOnlyUntilDepth(anon_map, 2);
+ if (file_path == "-") {
+ Logger::Log(LogLevel::Info, "List of anonymous target data:");
+ std::cout << dump_string << std::endl;
+ }
+ else {
+ Logger::Log(LogLevel::Info,
+ "Dumping list of anonymous target data to file '{}'.",
+ file_path);
+ std::ofstream os(file_path);
+ os << dump_string << std::endl;
+ }
+}
+
+void DumpNodes(std::string const& file_path, AnalysisResult const& result) {
+ auto node_map = nlohmann::json::object();
+ DumpNodesInExpressionToMap(&node_map, result.target->Provides());
+ auto const dump_string = IndentListsOnlyUntilDepth(node_map, 2);
+ if (file_path == "-") {
+ Logger::Log(
+ LogLevel::Info, "Target nodes of target {}:", result.id.ToString());
+ std::cout << dump_string << std::endl;
+ }
+ else {
+ Logger::Log(LogLevel::Info,
+ "Dumping target nodes of target {} to file '{}'.",
+ result.id.ToString(),
+ file_path);
+ std::ofstream os(file_path);
+ os << dump_string << std::endl;
+ }
+}
+
+[[nodiscard]] auto DiagnoseResults(AnalysisResult const& result,
+ Target::ResultTargetMap const& result_map,
+ DiagnosticArguments const& clargs) {
+ Logger::Log(LogLevel::Info,
+ "Result of target {}: {}",
+ result.id.ToString(),
+ ResultToJson(result.target->Result()).dump(2));
+ if (clargs.dump_actions) {
+ DumpActions(*clargs.dump_actions, result);
+ }
+ if (clargs.dump_blobs) {
+ DumpBlobs(*clargs.dump_blobs, result);
+ }
+ if (clargs.dump_trees) {
+ DumpTrees(*clargs.dump_trees, result);
+ }
+ if (clargs.dump_targets) {
+ DumpTargets(*clargs.dump_targets, result_map.ConfiguredTargets());
+ }
+ if (clargs.dump_anonymous) {
+ DumpAnonymous(*clargs.dump_anonymous, result_map.ConfiguredTargets());
+ }
+ if (clargs.dump_nodes) {
+ DumpNodes(*clargs.dump_nodes, result);
+ }
+}
+
+// Return disjoint maps for artifacts and runfiles
+[[nodiscard]] auto ReadOutputArtifacts(AnalysedTargetPtr const& target)
+ -> std::pair<std::map<std::string, ArtifactDescription>,
+ std::map<std::string, ArtifactDescription>> {
+ std::map<std::string, ArtifactDescription> artifacts{};
+ std::map<std::string, ArtifactDescription> runfiles{};
+ for (auto const& [path, artifact] : target->Artifacts()->Map()) {
+ artifacts.emplace(path, artifact->Artifact());
+ }
+ for (auto const& [path, artifact] : target->RunFiles()->Map()) {
+ if (not artifacts.contains(path)) {
+ runfiles.emplace(path, artifact->Artifact());
+ }
+ }
+ return {artifacts, runfiles};
+}
+
+void ReportTaintedness(const AnalysisResult& result) {
+ if (result.target->Tainted().empty()) {
+ // Never report untainted targets
+ return;
+ }
+
+ // To ensure proper quoting, go through json.
+ nlohmann::json tainted{};
+ for (auto const& s : result.target->Tainted()) {
+ tainted.push_back(s);
+ }
+ Logger::Log(LogLevel::Info, "Target tainted {}.", tainted.dump());
+}
+
+#ifndef BOOTSTRAP_BUILD_TOOL
+[[nodiscard]] auto FetchAndInstallArtifacts(
+ gsl::not_null<IExecutionApi*> const& api,
+ FetchArguments const& clargs) -> bool {
+ auto object_info = Artifact::ObjectInfo::FromString(clargs.object_id);
+ if (not object_info) {
+ Logger::Log(
+ LogLevel::Error, "failed to parse object id {}.", clargs.object_id);
+ return false;
+ }
+
+ if (clargs.output_path) {
+ auto output_path = (*clargs.output_path / "").parent_path();
+ if (FileSystemManager::IsDirectory(output_path)) {
+ output_path /= object_info->digest.hash();
+ }
+
+ if (not FileSystemManager::CreateDirectory(output_path.parent_path()) or
+ not api->RetrieveToPaths({*object_info}, {output_path})) {
+ Logger::Log(LogLevel::Error, "failed to retrieve artifact.");
+ return false;
+ }
+
+ Logger::Log(LogLevel::Info,
+ "artifact {} was installed to {}",
+ object_info->ToString(),
+ output_path.string());
+ }
+ else { // dump to stdout
+ if (not api->RetrieveToFds({*object_info}, {dup(fileno(stdout))})) {
+ Logger::Log(LogLevel::Error, "failed to dump artifact.");
+ return false;
+ }
+ }
+
+ return true;
+}
+#endif
+
+void PrintDoc(const nlohmann::json& doc, const std::string& indent) {
+ if (not doc.is_array()) {
+ return;
+ }
+ for (auto const& line : doc) {
+ if (line.is_string()) {
+ std::cout << indent << line.get<std::string>() << "\n";
+ }
+ }
+}
+
+void PrintFields(nlohmann::json const& fields,
+ const nlohmann::json& fdoc,
+ const std::string& indent_field,
+ const std::string& indent_field_doc) {
+ for (auto const& f : fields) {
+ std::cout << indent_field << f << "\n";
+ auto doc = fdoc.find(f);
+ if (doc != fdoc.end()) {
+ PrintDoc(*doc, indent_field_doc);
+ }
+ }
+}
+
+auto DescribeTarget(std::string const& main_repo,
+ std::optional<std::filesystem::path> const& main_ws_root,
+ std::size_t jobs,
+ AnalysisArguments const& clargs) -> int {
+ auto id = ReadConfiguredTarget(clargs, main_repo, main_ws_root);
+ if (id.target.explicit_file_reference) {
+ std::cout << id.ToString() << " is a source file." << std::endl;
+ return kExitSuccess;
+ }
+ auto targets_file_map = Base::CreateTargetsFileMap(jobs);
+ nlohmann::json targets_file{};
+ bool failed{false};
+ {
+ TaskSystem ts{jobs};
+ targets_file_map.ConsumeAfterKeysReady(
+ &ts,
+ {id.target.ToModule()},
+ [&targets_file](auto values) { targets_file = *values[0]; },
+ [&failed](auto const& msg, bool fatal) {
+ Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning,
+ "While searching for target description:\n{}",
+ msg);
+ failed = failed or fatal;
+ });
+ }
+ if (failed) {
+ return kExitFailure;
+ }
+ auto desc_it = targets_file.find(id.target.name);
+ if (desc_it == targets_file.end()) {
+ std::cout << id.ToString() << " is implicitly a source file."
+ << std::endl;
+ return kExitSuccess;
+ }
+ nlohmann::json desc = *desc_it;
+ auto rule_it = desc.find("type");
+ if (rule_it == desc.end()) {
+ Logger::Log(LogLevel::Error,
+ "{} is a target without specified type.",
+ id.ToString());
+ return kExitFailure;
+ }
+ if (BuildMaps::Target::IsBuiltInRule(*rule_it)) {
+ std::cout << id.ToString() << " is defined by built-in rule "
+ << rule_it->dump() << "." << std::endl;
+ if (*rule_it == "export") {
+ // export targets may have doc fields of their own.
+ auto doc = desc.find("doc");
+ if (doc != desc.end()) {
+ PrintDoc(*doc, " | ");
+ }
+ auto config_doc = nlohmann::json::object();
+ auto config_doc_it = desc.find("config_doc");
+ if (config_doc_it != desc.end() and config_doc_it->is_object()) {
+ config_doc = *config_doc_it;
+ }
+ auto flexible_config = desc.find("flexible_config");
+ if (flexible_config != desc.end() and
+ (not flexible_config->empty())) {
+ std::cout << " Flexible configuration variables\n";
+ PrintFields(*flexible_config, config_doc, " - ", " | ");
+ }
+ }
+ return kExitSuccess;
+ }
+ auto rule_name = BuildMaps::Base::ParseEntityNameFromJson(
+ *rule_it, id.target, [&rule_it, &id](std::string const& parse_err) {
+ Logger::Log(LogLevel::Error,
+ "Parsing rule name {} for target {} failed with:\n{}.",
+ rule_it->dump(),
+ id.ToString(),
+ parse_err);
+ });
+ if (not rule_name) {
+ return kExitFailure;
+ }
+ auto rule_file_map = Base::CreateRuleFileMap(jobs);
+ nlohmann::json rules_file;
+ {
+ TaskSystem ts{jobs};
+ rule_file_map.ConsumeAfterKeysReady(
+ &ts,
+ {rule_name->ToModule()},
+ [&rules_file](auto values) { rules_file = *values[0]; },
+ [&failed](auto const& msg, bool fatal) {
+ Logger::Log(fatal ? LogLevel::Error : LogLevel::Warning,
+ "While searching for rule definition:\n{}",
+ msg);
+ failed = failed or fatal;
+ });
+ }
+ if (failed) {
+ return kExitFailure;
+ }
+ auto ruledesc_it = rules_file.find(rule_name->name);
+ if (ruledesc_it == rules_file.end()) {
+ Logger::Log(LogLevel::Error,
+ "Rule definition of {} is missing",
+ rule_name->ToString());
+ return kExitFailure;
+ }
+ std::cout << id.ToString() << " is defined by user-defined rule "
+ << rule_name->ToString() << ".\n\n";
+ auto const& rdesc = *ruledesc_it;
+ auto doc = rdesc.find("doc");
+ if (doc != rdesc.end()) {
+ PrintDoc(*doc, " | ");
+ }
+ auto field_doc = nlohmann::json::object();
+ auto field_doc_it = rdesc.find("field_doc");
+ if (field_doc_it != rdesc.end() and field_doc_it->is_object()) {
+ field_doc = *field_doc_it;
+ }
+ auto string_fields = rdesc.find("string_fields");
+ if (string_fields != rdesc.end() and (not string_fields->empty())) {
+ std::cout << " String fields\n";
+ PrintFields(*string_fields, field_doc, " - ", " | ");
+ }
+ auto target_fields = rdesc.find("target_fields");
+ if (target_fields != rdesc.end() and (not target_fields->empty())) {
+ std::cout << " Target fields\n";
+ PrintFields(*target_fields, field_doc, " - ", " | ");
+ }
+ auto config_doc = nlohmann::json::object();
+ auto config_doc_it = rdesc.find("config_doc");
+ if (config_doc_it != rdesc.end() and config_doc_it->is_object()) {
+ config_doc = *config_doc_it;
+ }
+ auto config_vars = rdesc.find("config_vars");
+ if (config_vars != rdesc.end() and (not config_vars->empty())) {
+ std::cout << " Variables taken from the configuration\n";
+ PrintFields(*config_vars, config_doc, " - ", " | ");
+ }
+ std::cout << std::endl;
+ return kExitSuccess;
+}
+
+void DumpArtifactsToBuild(
+ std::map<std::string, ArtifactDescription> const& artifacts,
+ std::map<std::string, ArtifactDescription> const& runfiles,
+ const std::filesystem::path& file_path) {
+ nlohmann::json to_build{};
+ for (auto const& [path, artifact] : runfiles) {
+ to_build[path] = artifact.ToJson();
+ }
+ for (auto const& [path, artifact] : artifacts) {
+ to_build[path] = artifact.ToJson();
+ }
+ auto const dump_string = IndentListsOnlyUntilDepth(to_build, 2, 1);
+ std::ofstream os(file_path);
+ os << dump_string << std::endl;
+}
+
+} // namespace
+
+auto main(int argc, char* argv[]) -> int {
+ try {
+ auto arguments = ParseCommandLineArguments(argc, argv);
+
+ SetupLogging(arguments.common);
+#ifndef BOOTSTRAP_BUILD_TOOL
+ SetupLocalExecution(arguments.endpoint, arguments.build);
+#endif
+
+ auto jobs = arguments.build.build_jobs > 0 ? arguments.build.build_jobs
+ : arguments.common.jobs;
+
+ auto stage_args = arguments.cmd == SubCommand::kInstall or
+ arguments.cmd == SubCommand::kInstallCas or
+ arguments.cmd == SubCommand::kTraverse
+ ? std::make_optional(std::move(arguments.stage))
+ : std::nullopt;
+
+ auto rebuild_args =
+ arguments.cmd == SubCommand::kRebuild
+ ? std::make_optional(std::move(arguments.rebuild))
+ : std::nullopt;
+
+#ifndef BOOTSTRAP_BUILD_TOOL
+ GraphTraverser const traverser{{jobs,
+ std::move(arguments.endpoint),
+ std::move(arguments.build),
+ std::move(stage_args),
+ std::move(rebuild_args)}};
+
+ if (arguments.cmd == SubCommand::kInstallCas) {
+ return FetchAndInstallArtifacts(traverser.ExecutionApi(),
+ arguments.fetch)
+ ? kExitSuccess
+ : kExitFailure;
+ }
+#endif
+
+ auto [main_repo, main_ws_root] =
+ DetermineRoots(arguments.common, arguments.analysis);
+
+#ifndef BOOTSTRAP_BUILD_TOOL
+ if (arguments.cmd == SubCommand::kTraverse) {
+ if (arguments.graph.git_cas) {
+ if (not RepositoryConfig::Instance().SetGitCAS(
+ *arguments.graph.git_cas)) {
+ Logger::Log(LogLevel::Warning,
+ "Failed set Git CAS {}.",
+ arguments.graph.git_cas->string());
+ }
+ }
+ if (traverser.BuildAndStage(arguments.graph.graph_file,
+ arguments.graph.artifacts)) {
+ return kExitSuccess;
+ }
+ }
+ else if (arguments.cmd == SubCommand::kDescribe) {
+ return DescribeTarget(main_repo,
+ main_ws_root,
+ arguments.common.jobs,
+ arguments.analysis);
+ }
+
+ else {
+#endif
+ BuildMaps::Target::ResultTargetMap result_map{
+ arguments.common.jobs};
+ auto result = AnalyseTarget(&result_map,
+ main_repo,
+ main_ws_root,
+ arguments.common.jobs,
+ arguments.analysis);
+ if (result) {
+ if (arguments.analysis.graph_file) {
+ result_map.ToFile(*arguments.analysis.graph_file);
+ }
+ auto const [artifacts, runfiles] =
+ ReadOutputArtifacts(result->target);
+ if (arguments.analysis.artifacts_to_build_file) {
+ DumpArtifactsToBuild(
+ artifacts,
+ runfiles,
+ *arguments.analysis.artifacts_to_build_file);
+ }
+ if (arguments.cmd == SubCommand::kAnalyse) {
+ DiagnoseResults(*result, result_map, arguments.diagnose);
+ ReportTaintedness(*result);
+ // Clean up in parallel
+ {
+ TaskSystem ts;
+ result_map.Clear(&ts);
+ }
+ return kExitSuccess;
+ }
+#ifndef BOOTSTRAP_BUILD_TOOL
+ auto const& [actions, blobs, trees] = result_map.ToResult();
+
+ // Clean up result map, now that it is no longer needed
+ {
+ TaskSystem ts;
+ result_map.Clear(&ts);
+ }
+
+ Logger::Log(
+ LogLevel::Info,
+ "{}ing {}.",
+ arguments.cmd == SubCommand::kRebuild ? "Rebuild" : "Build",
+ result->id.ToString());
+ ReportTaintedness(*result);
+
+ auto build_result = traverser.BuildAndStage(
+ artifacts, runfiles, actions, blobs, trees);
+ if (build_result) {
+ // Repeat taintedness message to make the user aware that
+ // the artifacts are not for production use.
+ ReportTaintedness(*result);
+ return build_result->second ? kExitSuccessFailedArtifacts
+ : kExitSuccess;
+ }
+ }
+#endif
+ }
+ } catch (std::exception const& ex) {
+ Logger::Log(
+ LogLevel::Error, "Caught exception with message: {}", ex.what());
+ }
+ return kExitFailure;
+}
diff --git a/src/buildtool/main/main.hpp b/src/buildtool/main/main.hpp
new file mode 100644
index 00000000..6212d86e
--- /dev/null
+++ b/src/buildtool/main/main.hpp
@@ -0,0 +1,10 @@
+#ifndef INCLUDED_SRC_BUILDOOL_MAIN_MAIN_HPP
+#define INCLUDED_SRC_BUILDOOL_MAIN_MAIN_HPP
+
+enum ExitCodes {
+ kExitSuccess = 0,
+ kExitFailure = 1,
+ kExitSuccessFailedArtifacts = 2
+};
+
+#endif
diff --git a/src/buildtool/multithreading/TARGETS b/src/buildtool/multithreading/TARGETS
new file mode 100644
index 00000000..f14d42bb
--- /dev/null
+++ b/src/buildtool/multithreading/TARGETS
@@ -0,0 +1,54 @@
+{ "task":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["task"]
+ , "hdrs": ["task.hpp"]
+ , "stage": ["src", "buildtool", "multithreading"]
+ }
+, "notification_queue":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["notification_queue"]
+ , "hdrs": ["notification_queue.hpp"]
+ , "deps": ["task", ["src/utils/cpp", "atomic"]]
+ , "stage": ["src", "buildtool", "multithreading"]
+ }
+, "task_system":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["task_system"]
+ , "hdrs": ["task_system.hpp"]
+ , "srcs": ["task_system.cpp"]
+ , "deps": ["notification_queue", "task", ["@", "gsl-lite", "", "gsl-lite"]]
+ , "stage": ["src", "buildtool", "multithreading"]
+ }
+, "async_map_node":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["async_map_node"]
+ , "hdrs": ["async_map_node.hpp"]
+ , "deps": ["task", "task_system", ["@", "gsl-lite", "", "gsl-lite"]]
+ , "stage": ["src", "buildtool", "multithreading"]
+ }
+, "async_map":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["async_map"]
+ , "hdrs": ["async_map.hpp"]
+ , "deps":
+ [ "task"
+ , "task_system"
+ , "async_map_node"
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "multithreading"]
+ }
+, "async_map_consumer":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["async_map_consumer"]
+ , "hdrs": ["async_map_consumer.hpp"]
+ , "deps":
+ [ "task"
+ , "task_system"
+ , "async_map_node"
+ , "async_map"
+ , ["@", "gsl-lite", "", "gsl-lite"]
+ ]
+ , "stage": ["src", "buildtool", "multithreading"]
+ }
+} \ No newline at end of file
diff --git a/src/buildtool/multithreading/async_map.hpp b/src/buildtool/multithreading/async_map.hpp
new file mode 100644
index 00000000..80d5b0a3
--- /dev/null
+++ b/src/buildtool/multithreading/async_map.hpp
@@ -0,0 +1,109 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_HPP
+
+#include <memory>
+#include <mutex> // unique_lock
+#include <shared_mutex>
+#include <thread>
+#include <unordered_map>
+#include <utility> // std::make_pair to use std::unordered_map's emplace()
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/multithreading/async_map_node.hpp"
+#include "src/buildtool/multithreading/task.hpp"
+#include "src/buildtool/multithreading/task_system.hpp"
+
+// Wrapper around map data structure for KeyT->AsyncMapNode<ValueT> that only
+// exposes the possibility to retrieve the node for a certain key, adding it in
+// case of the key not yet being present. Thread-safe. Map look-ups happen under
+// a shared lock, and only in the case that key needs to be added to the
+// underlying map we uniquely lock. This is the default map class used inside
+// AsyncMapConsumer
+template <typename KeyT, typename ValueT>
+class AsyncMap {
+ public:
+ using Node = AsyncMapNode<KeyT, ValueT>;
+ // Nodes will be passed onto tasks. Nodes are owned by this map. Nodes are
+ // alive as long as this map lives.
+ using NodePtr = Node*;
+
+ explicit AsyncMap(std::size_t jobs) : width_{ComputeWidth(jobs)} {}
+
+ AsyncMap() = default;
+
+ /// \brief Retrieve node for certain key. Key and new node are emplaced in
+ /// the map in case that the key does not exist already.
+ /// \returns shared pointer to the Node associated to given key
+ [[nodiscard]] auto GetOrCreateNode(KeyT const& key) -> NodePtr {
+ auto* node_or_null = GetNodeOrNullFromSharedMap(key);
+ return node_or_null != nullptr ? node_or_null : AddKey(key);
+ }
+
+ [[nodiscard]] auto GetPendingKeys() const -> std::vector<KeyT> {
+ std::vector<KeyT> keys{};
+ size_t s = 0;
+ for (auto& i : map_) {
+ s += i.size();
+ }
+
+ keys.reserve(s);
+ for (auto& i : map_) {
+ for (auto const& [key, node] : i) {
+ if (not node->IsReady()) {
+ keys.emplace_back(key);
+ }
+ }
+ }
+ return keys;
+ }
+
+ void Clear(gsl::not_null<TaskSystem*> const& ts) {
+ for (std::size_t i = 0; i < width_; ++i) {
+ ts->QueueTask([i, this]() { map_[i].clear(); });
+ }
+ }
+
+ private:
+ constexpr static std::size_t kScalingFactor = 2;
+ std::size_t width_{ComputeWidth(0)};
+ std::vector<std::shared_mutex> m_{width_};
+ std::vector<std::unordered_map<KeyT, std::unique_ptr<Node>>> map_{width_};
+
+ constexpr static auto ComputeWidth(std::size_t jobs) -> std::size_t {
+ if (jobs <= 0) {
+ // Non-positive indicates to use the default value
+ return ComputeWidth(
+ std::max(1U, std::thread::hardware_concurrency()));
+ }
+ return jobs * kScalingFactor + 1;
+ }
+
+ [[nodiscard]] auto GetNodeOrNullFromSharedMap(KeyT const& key) -> NodePtr {
+ auto part = std::hash<KeyT>{}(key) % width_;
+ std::shared_lock sl{m_[part]};
+ auto it_to_key_pair = map_[part].find(key);
+ if (it_to_key_pair != map_[part].end()) {
+ // we know if the key is in the map then
+ // the pair {key, node} is read only
+ return it_to_key_pair->second.get();
+ }
+ return nullptr;
+ }
+
+ [[nodiscard]] auto AddKey(KeyT const& key) -> NodePtr {
+ auto part = std::hash<KeyT>{}(key) % width_;
+ std::unique_lock ul{m_[part]};
+ auto it_to_key_pair = map_[part].find(key);
+ if (it_to_key_pair != map_[part].end()) {
+ return it_to_key_pair->second.get();
+ }
+ auto new_node = std::make_unique<Node>(key);
+ bool unused{};
+ std::tie(it_to_key_pair, unused) =
+ map_[part].emplace(std::make_pair(key, std::move(new_node)));
+ return it_to_key_pair->second.get();
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_HPP
diff --git a/src/buildtool/multithreading/async_map_consumer.hpp b/src/buildtool/multithreading/async_map_consumer.hpp
new file mode 100644
index 00000000..eb965d4e
--- /dev/null
+++ b/src/buildtool/multithreading/async_map_consumer.hpp
@@ -0,0 +1,331 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_CONSUMER_HPP
+#define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_CONSUMER_HPP
+
+#include <atomic>
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <shared_mutex>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/multithreading/async_map.hpp"
+#include "src/buildtool/multithreading/async_map_node.hpp"
+#include "src/buildtool/multithreading/task.hpp"
+#include "src/buildtool/multithreading/task_system.hpp"
+
+using AsyncMapConsumerLogger = std::function<void(std::string const&, bool)>;
+using AsyncMapConsumerLoggerPtr = std::shared_ptr<AsyncMapConsumerLogger>;
+
+// Thread safe class that enables us to add tasks to the queue system that
+// depend on values being ready. Value constructors are only queued once per key
+// and tasks that depend on such values are only queued once the values are
+// ready. As template parameters, it takes the type that keys will have, the
+// type that their corresponding values will have and the type of the underlying
+// thread-safe associative container. The default thread-safe associative
+// container is AsyncMap<Key, Value> and any substite must have the same public
+// interface to be used in AsyncMapConsumer.
+template <typename Key, typename Value, typename Map = AsyncMap<Key, Value>>
+class AsyncMapConsumer {
+ public:
+ using Node = typename Map::Node;
+ using NodePtr = typename Map::NodePtr;
+
+ using Setter = std::function<void(Value&&)>;
+ using SetterPtr = std::shared_ptr<Setter>;
+
+ using Logger = AsyncMapConsumerLogger;
+ using LoggerPtr = AsyncMapConsumerLoggerPtr;
+
+ using FailureFunction = std::function<void()>;
+ using FailureFunctionPtr = std::shared_ptr<FailureFunction>;
+
+ using Consumer = std::function<void(std::vector<Value const*> const&)>;
+ using ConsumerPtr = std::shared_ptr<Consumer>;
+
+ using SubCaller =
+ std::function<void(std::vector<Key> const&, Consumer, LoggerPtr)>;
+ using SubCallerPtr = std::shared_ptr<SubCaller>;
+
+ using ValueCreator = std::function<void(gsl::not_null<TaskSystem*> const&,
+ SetterPtr,
+ LoggerPtr,
+ SubCallerPtr,
+ Key const&)>;
+
+ explicit AsyncMapConsumer(ValueCreator vc, std::size_t jobs = 0)
+ : value_creator_{std::make_shared<ValueCreator>(std::move(vc))},
+ map_{jobs} {}
+
+ /// \brief Makes sure that the consumer will be executed once the values for
+ /// all the keys are available, and that the value creators for those keys
+ /// are queued (if they weren't queued already).
+ /// \param[in] ts task system
+ /// \param[in] keys keys for the values that consumer requires
+ /// \param[in] consumer function-like object that takes a vector of values
+ /// and returns void that will be queued to be called with the values
+ /// associated to keys once they are ready
+ /// \param[in] logger function-like object that takes a string and a bool
+ /// indicating that the event was fatal and returns
+ /// void. This will be passed around and can be used to report errors
+ /// (possibly with side effects outside AsyncMapConsumer) in the value
+ /// creator
+ /// \param[in] fail function to call instead of the consumer if the
+ /// creation of this node failed
+ void ConsumeAfterKeysReady(gsl::not_null<TaskSystem*> const& ts,
+ std::vector<Key> const& keys,
+ Consumer&& consumer,
+ Logger&& logger,
+ FailureFunction&& fail) {
+ ConsumeAfterKeysReady(
+ ts,
+ std::nullopt,
+ keys,
+ std::move(consumer),
+ std::make_shared<Logger>(std::move(logger)),
+ std::make_shared<FailureFunction>(std::move(fail)));
+ }
+
+ // Similar to the previous method, but without failure function
+ void ConsumeAfterKeysReady(gsl::not_null<TaskSystem*> const& ts,
+ std::vector<Key> const& keys,
+ Consumer&& consumer,
+ Logger&& logger) {
+ ConsumeAfterKeysReady(ts,
+ std::nullopt,
+ keys,
+ std::move(consumer),
+ std::make_shared<Logger>(std::move(logger)),
+ nullptr);
+ }
+
+ [[nodiscard]] auto GetPendingKeys() const -> std::vector<Key> {
+ return map_.GetPendingKeys();
+ }
+
+ // Returns call order of the first cycle found in the requests map.
+ [[nodiscard]] auto DetectCycle() const -> std::optional<std::vector<Key>> {
+ auto const& requests = GetPendingRequests();
+ std::vector<Key> calls{};
+ std::unordered_set<Key> known{};
+ calls.resize(requests.size() + 1, Key{});
+ known.reserve(requests.size());
+ for (auto const& [caller, _] : requests) {
+ if (DetectCycleForCaller(&calls, &known, requests, caller)) {
+ return calls;
+ }
+ }
+ return std::nullopt;
+ }
+
+ void Clear(gsl::not_null<TaskSystem*> const& ts) { map_.Clear(ts); }
+
+ private:
+ using NodeRequests = std::unordered_map<Key, std::unordered_set<NodePtr>>;
+
+ std::shared_ptr<ValueCreator> value_creator_{};
+ Map map_{};
+ mutable std::shared_mutex requests_m_{};
+ std::unordered_map<std::thread::id, NodeRequests> requests_by_thread_{};
+
+ // Similar to previous methods, but in this case the logger and failure
+ // function are already std::shared_ptr type.
+ void ConsumeAfterKeysReady(gsl::not_null<TaskSystem*> const& ts,
+ std::optional<Key> const& consumer_id,
+ std::vector<Key> const& keys,
+ Consumer&& consumer,
+ LoggerPtr&& logger,
+ FailureFunctionPtr&& fail) {
+ auto consumerptr = std::make_shared<Consumer>(std::move(consumer));
+ if (keys.empty()) {
+ ts->QueueTask([consumerptr = std::move(consumerptr)]() {
+ (*consumerptr)({});
+ });
+ return;
+ }
+
+ auto nodes = EnsureValuesEventuallyPresent(ts, keys, std::move(logger));
+ auto first_node = nodes->at(0);
+ if (fail) {
+ first_node->QueueOnFailure(ts, [fail]() { (*fail)(); });
+ }
+ auto const queued = first_node->AddOrQueueAwaitingTask(
+ ts,
+ [ts,
+ consumerptr,
+ nodes = std::move(nodes),
+ fail,
+ this,
+ consumer_id]() {
+ QueueTaskWhenAllReady(
+ ts, consumer_id, consumerptr, fail, nodes, 1);
+ });
+ if (consumer_id and not queued) {
+ RecordNodeRequest(*consumer_id, first_node);
+ }
+ }
+
+ [[nodiscard]] auto EnsureValuesEventuallyPresent(
+ gsl::not_null<TaskSystem*> const& ts,
+ std::vector<Key> const& keys,
+ LoggerPtr&& logger) -> std::shared_ptr<std::vector<NodePtr>> {
+ std::vector<NodePtr> nodes{};
+ nodes.reserve(keys.size());
+ std::transform(std::begin(keys),
+ std::end(keys),
+ std::back_inserter(nodes),
+ [this, ts, logger](Key const& key) {
+ return EnsureValuePresent(ts, key, logger);
+ });
+ return std::make_shared<std::vector<NodePtr>>(std::move(nodes));
+ }
+
+ // Retrieves node from map associated to given key and queues its processing
+ // task (i.e. a task that executes the value creator) to the task system.
+ // Note that the node will only queue a processing task once.
+ [[nodiscard]] auto EnsureValuePresent(gsl::not_null<TaskSystem*> const& ts,
+ Key const& key,
+ LoggerPtr const& logger) -> NodePtr {
+ auto node = map_.GetOrCreateNode(key);
+ auto setterptr = std::make_shared<Setter>([ts, node](Value&& value) {
+ node->SetAndQueueAwaitingTasks(ts, std::move(value));
+ });
+ auto failptr =
+ std::make_shared<FailureFunction>([node, ts]() { node->Fail(ts); });
+ auto subcallerptr = std::make_shared<SubCaller>(
+ [ts, failptr = std::move(failptr), this, key](
+ std::vector<Key> const& keys,
+ Consumer&& consumer,
+ LoggerPtr&& logger) {
+ ConsumeAfterKeysReady(ts,
+ key,
+ keys,
+ std::move(consumer),
+ std::move(logger),
+ FailureFunctionPtr{failptr});
+ });
+ auto wrappedLogger =
+ std::make_shared<Logger>([logger, node, ts](auto msg, auto fatal) {
+ if (fatal) {
+ node->Fail(ts);
+ }
+ (*logger)(msg, fatal);
+ });
+ node->QueueOnceProcessingTask(
+ ts,
+ [vc = value_creator_,
+ ts,
+ key,
+ setterptr = std::move(setterptr),
+ wrappedLogger = std::move(wrappedLogger),
+ subcallerptr = std::move(subcallerptr)]() {
+ (*vc)(ts, setterptr, wrappedLogger, subcallerptr, key);
+ });
+ return node;
+ }
+
+ // Queues tasks for each node making sure that the task that calls the
+ // consumer on the values is only queued once all the values are ready
+ void QueueTaskWhenAllReady(
+ gsl::not_null<TaskSystem*> const& ts,
+ std::optional<Key> const& consumer_id,
+ ConsumerPtr const& consumer,
+ // NOLINTNEXTLINE(performance-unnecessary-value-param)
+ FailureFunctionPtr const& fail,
+ std::shared_ptr<std::vector<NodePtr>> const& nodes,
+ std::size_t pos) {
+ if (pos == nodes->size()) {
+ ts->QueueTask([nodes, consumer]() {
+ std::vector<Value const*> values{};
+ values.reserve(nodes->size());
+ std::transform(
+ nodes->begin(),
+ nodes->end(),
+ std::back_inserter(values),
+ [](NodePtr const& node) { return &node->GetValue(); });
+ (*consumer)(values);
+ });
+ }
+ else {
+ auto current = nodes->at(pos);
+ if (fail) {
+ current->QueueOnFailure(ts, [fail]() { (*fail)(); });
+ }
+ auto const queued = current->AddOrQueueAwaitingTask(
+ ts, [ts, consumer, fail, nodes, pos, this, consumer_id]() {
+ QueueTaskWhenAllReady(
+ ts, consumer_id, consumer, fail, nodes, pos + 1);
+ });
+ if (consumer_id and not queued) {
+ RecordNodeRequest(*consumer_id, current);
+ }
+ }
+ }
+
+ void RecordNodeRequest(Key const& consumer_id,
+ gsl::not_null<NodePtr> const& node) {
+ auto tid = std::this_thread::get_id();
+ std::shared_lock shared(requests_m_);
+ auto local_requests_it = requests_by_thread_.find(tid);
+ if (local_requests_it == requests_by_thread_.end()) {
+ shared.unlock();
+ std::unique_lock lock(requests_m_);
+ // create new requests map for thread
+ requests_by_thread_[tid] = NodeRequests{{consumer_id, {node}}};
+ return;
+ }
+ // every thread writes to separate local requests map
+ local_requests_it->second[consumer_id].emplace(node);
+ }
+
+ [[nodiscard]] auto GetPendingRequests() const -> NodeRequests {
+ NodeRequests requests{};
+ std::unique_lock lock(requests_m_);
+ for (auto const& [_, local_requests] : requests_by_thread_) {
+ requests.reserve(requests.size() + local_requests.size());
+ for (auto const& [consumer, deps] : local_requests) {
+ auto& nodes = requests[consumer];
+ std::copy_if( // filter out nodes that are ready by now
+ deps.begin(),
+ deps.end(),
+ std::inserter(nodes, nodes.end()),
+ [](auto const& node) { return not node->IsReady(); });
+ }
+ }
+ return requests;
+ }
+
+ [[nodiscard]] static auto DetectCycleForCaller(
+ gsl::not_null<std::vector<Key>*> const& calls,
+ gsl::not_null<std::unordered_set<Key>*> const& known,
+ NodeRequests const& requests,
+ Key const& caller,
+ std::size_t pos = 0) -> bool {
+ if (not known->contains(caller)) {
+ auto it = requests.find(caller);
+ if (it != requests.end()) {
+ (*calls)[pos++] = caller;
+ for (auto const& dep : it->second) {
+ auto const& dep_key = dep->GetKey();
+ auto last = calls->begin() + static_cast<int>(pos);
+ if (std::find(calls->begin(), last, dep_key) != last) {
+ (*calls)[pos++] = dep_key;
+ calls->resize(pos);
+ return true;
+ }
+ if (DetectCycleForCaller(
+ calls, known, requests, dep_key, pos)) {
+ return true;
+ }
+ }
+ }
+ known->emplace(caller);
+ }
+ return false;
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_CONSUMER_HPP
diff --git a/src/buildtool/multithreading/async_map_node.hpp b/src/buildtool/multithreading/async_map_node.hpp
new file mode 100644
index 00000000..31a33512
--- /dev/null
+++ b/src/buildtool/multithreading/async_map_node.hpp
@@ -0,0 +1,173 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_NODE_HPP
+#define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_NODE_HPP
+
+#include <atomic>
+#include <mutex>
+#include <optional>
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/multithreading/task.hpp"
+#include "src/buildtool/multithreading/task_system.hpp"
+
+// Wrapper around Value to enable async access to it in a continuation-style
+// programming way
+template <typename Key, typename Value>
+class AsyncMapNode {
+ public:
+ explicit AsyncMapNode(Key key) : key_{std::move(key)} {}
+
+ /// \brief Set value and queue awaiting tasks to the task system under a
+ /// unique lock. Awaiting tasks are cleared to ensure node does not hold
+ /// (shared) ownership of any data related to the task once they are given
+ /// to the task system
+ /// \param[in] ts task system to which tasks will be queued
+ /// \param[in] value value to set
+ void SetAndQueueAwaitingTasks(gsl::not_null<TaskSystem*> const& ts,
+ Value&& value) {
+ std::unique_lock lock{m_};
+ if (failed_) {
+ // The node is failed already; no value can be set.
+ return;
+ }
+ value_ = std::move(value);
+ for (auto& task : awaiting_tasks_) {
+ ts->QueueTask(std::move(task));
+ }
+ // After tasks are queued we need to release them and any other
+ // information we are keeping about the tasks
+ awaiting_tasks_.clear();
+ failure_tasks_.clear();
+ }
+
+ /// \brief If node is not marked as queued to be processed, task is queued
+ /// to the task system. A task to process the node (that is, set its value)
+ /// can only be queued once. Lock free
+ /// \param[in] ts task system
+ /// \param[in] task processing task. Function type must have
+ /// operator()()
+ template <typename Function>
+ void QueueOnceProcessingTask(gsl::not_null<TaskSystem*> const& ts,
+ Function&& task) {
+ // if node was already queued to be processed, nothing to do
+ if (GetAndMarkQueuedToBeProcessed()) {
+ return;
+ }
+ ts->QueueTask(std::forward<Function>(task));
+ }
+
+ /// \brief Ensure task will be queued to the task system once the value of
+ /// the node is ready. This operation is lock free once the value is ready
+ /// before that node is uniquely locked while task is being added to
+ /// awaiting tasks
+ /// \param[in] ts task system
+ /// \param[in] task task awaiting for value. Function type must have
+ /// operator()()
+ /// \returns boolean indicating whether task was immediately queued.
+ template <typename Function>
+ [[nodiscard]] auto AddOrQueueAwaitingTask(
+ gsl::not_null<TaskSystem*> const& ts,
+ Function&& task) -> bool {
+ if (IsReady()) {
+ ts->QueueTask(std::forward<Function>(task));
+ return true;
+ }
+ {
+ std::unique_lock ul{m_};
+ if (failed_) {
+ // If the node is failed (and hence will never get ready), do
+ // not queue any more tasks.
+ return false;
+ }
+ // Check again in case the node was made ready after the lock-free
+ // check by another thread
+ if (IsReady()) {
+ ts->QueueTask(std::forward<Function>(task));
+ return true;
+ }
+ awaiting_tasks_.emplace_back(std::forward<Function>(task));
+ return false;
+ }
+ }
+
+ /// \brief Ensure task will be queued to the task system once the value of
+ /// the node is ready. This operation is lock free once the value is ready
+ /// before that node is uniquely locked while task is being added to
+ /// awaiting tasks
+ /// \param[in] ts task system
+ /// \param[in] task task awaiting for value. Function type must have
+ /// operator()()
+ template <typename Function>
+ void QueueOnFailure(gsl::not_null<TaskSystem*> const& ts, Function&& task) {
+ if (IsReady()) {
+ // The node is ready, so it won't fail any more.
+ return;
+ }
+ {
+ std::unique_lock ul{m_};
+ if (failed_) {
+ ts->QueueTask(std::forward<Function>(task));
+ }
+ else {
+ failure_tasks_.emplace_back(std::forward<Function>(task));
+ }
+ }
+ }
+
+ /// \brief Mark the node as failed and schedule the cleanup tasks.
+ /// \param[in] ts task system
+ void Fail(gsl::not_null<TaskSystem*> const& ts) {
+ std::unique_lock ul{m_};
+ if (IsReady()) {
+ // The node has a value already, so it can't be marked as failed any
+ // more
+ return;
+ }
+ if (failed_) {
+ // The was already marked as failed and the failure handled.
+ // So there is nothing more to do.
+ return;
+ }
+ failed_ = true;
+ // As the node will never become ready, we have to clean up all tasks
+ // and schedule the failure tasks.
+ for (auto& task : failure_tasks_) {
+ ts->QueueTask(std::move(task));
+ }
+ awaiting_tasks_.clear();
+ failure_tasks_.clear();
+ }
+
+ // Not thread safe, do not use unless the value has been already set
+ [[nodiscard]] auto GetValue() const& noexcept -> Value const& {
+ // Will only be checked in debug build
+ gsl_ExpectsAudit(value_.has_value());
+ return *value_;
+ }
+ [[nodiscard]] auto GetValue() && noexcept = delete;
+
+ [[nodiscard]] auto GetKey() const& noexcept -> Key const& { return key_; }
+ [[nodiscard]] auto GetKey() && noexcept -> Key { return std::move(key_); }
+
+ [[nodiscard]] auto IsReady() const noexcept -> bool {
+ return value_.has_value();
+ }
+
+ private:
+ Key key_;
+ std::optional<Value> value_{};
+ std::vector<Task> awaiting_tasks_{};
+ std::vector<Task> failure_tasks_{};
+ std::mutex m_{};
+ std::atomic<bool> is_queued_to_be_processed_{false};
+ bool failed_{false};
+
+ /// \brief Sets node as queued to be processed
+ /// \returns True if it was already queued to be processed, false
+ /// otherwise
+ /// Note: this is an atomic, lock-free operation
+ [[nodiscard]] auto GetAndMarkQueuedToBeProcessed() noexcept -> bool {
+ return std::atomic_exchange(&is_queued_to_be_processed_, true);
+ }
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_ASYNC_MAP_NODE_HPP
diff --git a/src/buildtool/multithreading/notification_queue.hpp b/src/buildtool/multithreading/notification_queue.hpp
new file mode 100644
index 00000000..7e79aa43
--- /dev/null
+++ b/src/buildtool/multithreading/notification_queue.hpp
@@ -0,0 +1,188 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_NOTIFICATION_QUEUE_HPP
+#define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_NOTIFICATION_QUEUE_HPP
+
+#include <condition_variable>
+#include <deque>
+#include <mutex>
+#include <optional>
+#include <utility> // std::forward
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/multithreading/task.hpp"
+#include "src/utils/cpp/atomic.hpp"
+
+// Flag that can block the caller until it is set. Cannot be cleared after set.
+class WaitableOneWayFlag {
+ public:
+ // Clear flag. Essentially a noop, if it was set before.
+ void Clear() {
+ if (not was_set_) {
+ initial_ = false;
+ }
+ }
+
+ // Set flag. Essentially a noop, if it was set before.
+ void Set() {
+ if (not was_set_) {
+ was_set_ = true;
+ was_set_.notify_all();
+ }
+ }
+
+ // Blocks caller until it is set, if it was ever cleared.
+ void WaitForSet() {
+ if (not was_set_ and not initial_) {
+ was_set_.wait(false);
+ }
+ }
+
+ private:
+ atomic<bool> was_set_{};
+ bool initial_{true};
+};
+
+// Counter that can block the caller until it reaches zero.
+class WaitableZeroCounter {
+ enum class Status { Init, Wait, Reached };
+
+ public:
+ explicit WaitableZeroCounter(std::size_t init = 0) : count_{init} {}
+
+ // Essentially a noop, if count reached zero since last wait call.
+ void Decrement() {
+ if (status_ != Status::Reached and --count_ == 0) {
+ if (status_ == Status::Wait) {
+ status_ = Status::Reached;
+ status_.notify_all();
+ }
+ }
+ }
+
+ // Essentially a noop, if count reached zero since last wait call.
+ void Increment() {
+ if (status_ != Status::Reached) {
+ ++count_;
+ }
+ }
+
+ // Blocks caller until count reached zero, since last call to this method.
+ void WaitForZero() {
+ status_ = Status::Wait;
+ if (count_ != 0) {
+ status_.wait(Status::Wait);
+ }
+ status_ = Status::Reached;
+ }
+
+ private:
+ std::atomic<std::size_t> count_{};
+ atomic<Status> status_{Status::Init};
+};
+
+class NotificationQueue {
+ public:
+ NotificationQueue(gsl::not_null<WaitableOneWayFlag*> queues_read,
+ gsl::not_null<WaitableZeroCounter*> num_threads_running)
+ : queues_read_{std::move(queues_read)},
+ num_threads_running_{std::move(num_threads_running)} {}
+
+ NotificationQueue(NotificationQueue const& other) = delete;
+ NotificationQueue(NotificationQueue&& other) noexcept
+ : queue_{std::move(other.queue_)},
+ done_{other.done_},
+ queues_read_{std::move(other.queues_read_)},
+ num_threads_running_{std::move(other.num_threads_running_)} {}
+ ~NotificationQueue() = default;
+
+ [[nodiscard]] auto operator=(NotificationQueue const& other)
+ -> NotificationQueue& = delete;
+ [[nodiscard]] auto operator=(NotificationQueue&& other)
+ -> NotificationQueue& = delete;
+
+ // Blocks the thread until it's possible to pop or we are done.
+ // Note that the lock releases ownership of the mutex while waiting
+ // for the queue to have some element or for the notification queue
+ // state to be set to "done".
+ // Returns task popped or nullopt if no task was popped
+ [[nodiscard]] auto pop() -> std::optional<Task> {
+ std::unique_lock lock{mutex_};
+ auto there_is_something_to_pop_or_we_are_done = [&]() {
+ return !queue_.empty() || done_;
+ };
+ if (not there_is_something_to_pop_or_we_are_done()) {
+ num_threads_running_->Decrement();
+ ready_.wait(lock, there_is_something_to_pop_or_we_are_done);
+ num_threads_running_->Increment();
+ }
+
+ if (queue_.empty()) {
+ return std::nullopt;
+ }
+ auto t = std::move(queue_.front());
+ queue_.pop_front();
+ queues_read_->Set();
+ return t;
+ }
+
+ // Returns nullopt if the mutex is already locked or the queue is empty,
+ // otherwise pops the front element of the queue and returns it
+ [[nodiscard]] auto try_pop() -> std::optional<Task> {
+ std::unique_lock lock{mutex_, std::try_to_lock};
+ if (!lock || queue_.empty()) {
+ return std::nullopt;
+ }
+ auto t = std::move(queue_.front());
+ queue_.pop_front();
+ queues_read_->Set();
+ return t;
+ }
+
+ // Push task once the mutex is available (locking it until addition is
+ // finished)
+ template <typename FunctionType>
+ void push(FunctionType&& f) {
+ {
+ std::unique_lock lock{mutex_};
+ queue_.emplace_back(std::forward<FunctionType>(f));
+ }
+ queues_read_->Clear();
+ ready_.notify_one();
+ }
+
+ // Returns false if mutex is locked without pushing the task, pushes task
+ // and returns true otherwise
+ template <typename FunctionType>
+ [[nodiscard]] auto try_push(FunctionType&& f) -> bool {
+ {
+ std::unique_lock lock{mutex_, std::try_to_lock};
+ if (!lock) {
+ return false;
+ }
+ queue_.emplace_back(std::forward<FunctionType>(f));
+ }
+ queues_read_->Clear();
+ ready_.notify_one();
+ return true;
+ }
+
+ // Method to communicate to the notification queue that there will not be
+ // any more queries. Queries after calling this method are not guaratied to
+ // work as expected
+ void done() {
+ {
+ std::unique_lock lock{mutex_};
+ done_ = true;
+ }
+ ready_.notify_all();
+ }
+
+ private:
+ std::deque<Task> queue_{};
+ bool done_{false};
+ std::mutex mutex_{};
+ std::condition_variable ready_{};
+ gsl::not_null<WaitableOneWayFlag*> queues_read_;
+ gsl::not_null<WaitableZeroCounter*> num_threads_running_;
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_NOTIFICATION_QUEUE_HPP
diff --git a/src/buildtool/multithreading/task.hpp b/src/buildtool/multithreading/task.hpp
new file mode 100644
index 00000000..49eb20a9
--- /dev/null
+++ b/src/buildtool/multithreading/task.hpp
@@ -0,0 +1,38 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_HPP
+#define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_HPP
+
+#include <functional>
+#include <type_traits>
+
+class Task {
+ public:
+ using TaskFunc = std::function<void()>;
+
+ Task() noexcept = default;
+
+ // NOLINTNEXTLINE(modernize-pass-by-value)
+ explicit Task(TaskFunc const& function) noexcept : f_{function} {}
+ explicit Task(TaskFunc&& function) noexcept : f_{std::move(function)} {}
+
+ void operator()() { f_(); }
+
+ // To be able to discern whether the internal f_ has been set or not,
+ // allowing us to write code such as:
+ /*
+ Task t;
+ while (!t) {
+ t = TryGetTaskFromQueue(); // (*)
+ }
+ t(); // (**)
+ */
+ // (*) does `return Task();` or `return {};` if queue is empty or locked)
+ // (**) we can now surely execute the task (and be sure it won't throw any
+ // exception) (for the sake of the example, imagine we are sure that the
+ // queue wasn't empty, otherwise this would be an infinite loop)
+ explicit operator bool() const noexcept { return f_.operator bool(); }
+
+ private:
+ TaskFunc f_{};
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_HPP
diff --git a/src/buildtool/multithreading/task_system.cpp b/src/buildtool/multithreading/task_system.cpp
new file mode 100644
index 00000000..8c976a2f
--- /dev/null
+++ b/src/buildtool/multithreading/task_system.cpp
@@ -0,0 +1,56 @@
+#include "src/buildtool/multithreading/task_system.hpp"
+
+#include "gsl-lite/gsl-lite.hpp"
+#include "src/buildtool/multithreading/task.hpp"
+
+TaskSystem::TaskSystem() : TaskSystem(std::thread::hardware_concurrency()) {}
+
+TaskSystem::TaskSystem(std::size_t number_of_threads)
+ : thread_count_{std::max(1UL, number_of_threads)},
+ num_threads_running_{thread_count_} {
+ for (std::size_t index = 0; index < thread_count_; ++index) {
+ queues_.emplace_back(&queues_read_, &num_threads_running_);
+ }
+ for (std::size_t index = 0; index < thread_count_; ++index) {
+ threads_.emplace_back([&, index]() { Run(index); });
+ }
+}
+
+TaskSystem::~TaskSystem() {
+ // When starting a new task system all spawned threads will immediately go
+ // to sleep and wait for tasks. Even after adding some tasks, it can take a
+ // while until the first thread wakes up. Therefore, we first need to wait
+ // for the queues being read, before we can wait for all threads to become
+ // idle.
+ queues_read_.WaitForSet();
+ num_threads_running_.WaitForZero();
+ for (auto& q : queues_) {
+ q.done();
+ }
+ for (auto& t : threads_) {
+ t.join();
+ }
+}
+
+void TaskSystem::Run(std::size_t idx) {
+ gsl_Expects(thread_count_ > 0);
+
+ while (true) {
+ std::optional<Task> t{};
+ for (std::size_t i = 0; i < thread_count_; ++i) {
+ t = queues_[(idx + i) % thread_count_].try_pop();
+ if (t) {
+ break;
+ }
+ }
+
+ // NOLINTNEXTLINE(clang-analyzer-core.DivideZero)
+ t = t ? t : queues_[idx % thread_count_].pop();
+
+ if (!t) {
+ break;
+ }
+
+ (*t)();
+ }
+}
diff --git a/src/buildtool/multithreading/task_system.hpp b/src/buildtool/multithreading/task_system.hpp
new file mode 100644
index 00000000..c2e46779
--- /dev/null
+++ b/src/buildtool/multithreading/task_system.hpp
@@ -0,0 +1,65 @@
+#ifndef INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_SYSTEM_HPP
+#define INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_SYSTEM_HPP
+
+#include <algorithm>
+#include <atomic>
+#include <thread>
+#include <vector>
+
+#include "src/buildtool/multithreading/notification_queue.hpp"
+
+class TaskSystem {
+ public:
+ // Constructors create as many threads as specified (or
+ // std::thread::hardware_concurrency() many if not specified) running
+ // `TaskSystem::Run(index)` on them, where `index` is their position in
+ // `threads_`
+ TaskSystem();
+ explicit TaskSystem(std::size_t number_of_threads);
+
+ TaskSystem(TaskSystem const&) = delete;
+ TaskSystem(TaskSystem&&) = delete;
+ auto operator=(TaskSystem const&) -> TaskSystem& = delete;
+ auto operator=(TaskSystem &&) -> TaskSystem& = delete;
+
+ // Destructor calls sets to "done" all notification queues and joins the
+ // threads. Note that joining the threads will wait until the Run method
+ // they are running is finished
+ ~TaskSystem();
+
+ // Queue a task. Task will be added to the first notification queue that is
+ // found to be unlocked or, if none is found (after kNumberOfAttemps
+ // iterations), to the one in `index+1` position waiting until it's
+ // unlocked.
+ template <typename FunctionType>
+ void QueueTask(FunctionType&& f) noexcept {
+ auto idx = index_++;
+
+ for (std::size_t i = 0; i < thread_count_ * kNumberOfAttempts; ++i) {
+ if (queues_[(idx + i) % thread_count_].try_push(
+ std::forward<FunctionType>(f))) {
+ return;
+ }
+ }
+ queues_[idx % thread_count_].push(std::forward<FunctionType>(f));
+ }
+
+ [[nodiscard]] auto NumberOfThreads() const noexcept -> std::size_t {
+ return thread_count_;
+ }
+
+ private:
+ std::size_t const thread_count_{
+ std::max(1U, std::thread::hardware_concurrency())};
+ std::vector<std::thread> threads_{};
+ std::vector<NotificationQueue> queues_{};
+ std::atomic<std::size_t> index_{0};
+ WaitableOneWayFlag queues_read_{};
+ WaitableZeroCounter num_threads_running_{};
+
+ static constexpr std::size_t kNumberOfAttempts = 5;
+
+ void Run(std::size_t idx);
+};
+
+#endif // INCLUDED_SRC_BUILDTOOL_MULTITHREADING_TASK_SYSTEM_HPP
diff --git a/src/utils/TARGETS b/src/utils/TARGETS
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/utils/TARGETS
@@ -0,0 +1 @@
+{} \ No newline at end of file
diff --git a/src/utils/cpp/TARGETS b/src/utils/cpp/TARGETS
new file mode 100644
index 00000000..6b4347a2
--- /dev/null
+++ b/src/utils/cpp/TARGETS
@@ -0,0 +1,40 @@
+{ "hash_combine":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["hash_combine"]
+ , "hdrs": ["hash_combine.hpp"]
+ , "deps": [["@", "gsl-lite", "", "gsl-lite"]]
+ , "stage": ["src", "utils", "cpp"]
+ }
+, "type_safe_arithmetic":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["type_safe_arithmetic"]
+ , "hdrs": ["type_safe_arithmetic.hpp"]
+ , "deps": [["@", "gsl-lite", "", "gsl-lite"]]
+ , "stage": ["src", "utils", "cpp"]
+ }
+, "json":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["json"]
+ , "hdrs": ["json.hpp"]
+ , "deps": [["@", "json", "", "json"], ["@", "gsl-lite", "", "gsl-lite"]]
+ , "stage": ["src", "utils", "cpp"]
+ }
+, "concepts":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["concepts"]
+ , "hdrs": ["concepts.hpp"]
+ , "stage": ["src", "utils", "cpp"]
+ }
+, "atomic":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["atomic"]
+ , "hdrs": ["atomic.hpp"]
+ , "stage": ["src", "utils", "cpp"]
+ }
+, "hex_string":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["hex_string"]
+ , "hdrs": ["hex_string.hpp"]
+ , "stage": ["src", "utils", "cpp"]
+ }
+} \ No newline at end of file
diff --git a/src/utils/cpp/atomic.hpp b/src/utils/cpp/atomic.hpp
new file mode 100644
index 00000000..7f7631d0
--- /dev/null
+++ b/src/utils/cpp/atomic.hpp
@@ -0,0 +1,119 @@
+#ifndef INCLUDED_SRC_UTILS_CPP_ATOMIC_HPP
+#define INCLUDED_SRC_UTILS_CPP_ATOMIC_HPP
+
+#include <atomic>
+#include <condition_variable>
+#include <shared_mutex>
+
+// Atomic wrapper with notify/wait capabilities.
+// TODO(modernize): Replace any use this class by C++20's std::atomic<T>, once
+// libcxx adds support for notify_*() and wait().
+// [https://libcxx.llvm.org/docs/Cxx2aStatus.html]
+template <class T>
+class atomic {
+ public:
+ atomic() = default;
+ explicit atomic(T value) : value_{std::move(value)} {}
+ atomic(atomic const& other) = delete;
+ atomic(atomic&& other) = delete;
+ ~atomic() = default;
+
+ auto operator=(atomic const& other) -> atomic& = delete;
+ auto operator=(atomic&& other) -> atomic& = delete;
+ auto operator=(T desired) -> T { // NOLINT
+ std::shared_lock lock(mutex_);
+ value_ = desired;
+ return desired;
+ }
+ operator T() const { return static_cast<T>(value_); } // NOLINT
+
+ void store(T desired, std::memory_order order = std::memory_order_seq_cst) {
+ std::shared_lock lock(mutex_);
+ value_.store(std::move(desired), order);
+ }
+ [[nodiscard]] auto load(
+ std::memory_order order = std::memory_order_seq_cst) const -> T {
+ return value_.load(order);
+ }
+
+ template <class U = T, class = std::enable_if_t<std::is_integral_v<U>>>
+ auto operator++() -> T {
+ std::shared_lock lock(mutex_);
+ return ++value_;
+ }
+ template <class U = T, class = std::enable_if_t<std::is_integral_v<U>>>
+ [[nodiscard]] auto operator++(int) -> T {
+ std::shared_lock lock(mutex_);
+ return value_++;
+ }
+ template <class U = T, class = std::enable_if_t<std::is_integral_v<U>>>
+ auto operator--() -> T {
+ std::shared_lock lock(mutex_);
+ return --value_;
+ }
+ template <class U = T, class = std::enable_if_t<std::is_integral_v<U>>>
+ [[nodiscard]] auto operator--(int) -> T {
+ std::shared_lock lock(mutex_);
+ return value_--;
+ }
+
+ void notify_one() { cv_.notify_one(); }
+ void notify_all() { cv_.notify_all(); }
+ void wait(T old,
+ std::memory_order order = std::memory_order::seq_cst) const {
+ std::unique_lock lock(mutex_);
+ cv_.wait(lock,
+ [this, &old, order]() { return value_.load(order) != old; });
+ }
+
+ private:
+ std::atomic<T> value_{};
+ mutable std::shared_mutex mutex_{};
+ mutable std::condition_variable_any cv_{};
+};
+
+// Atomic shared_pointer with notify/wait capabilities.
+// TODO(modernize): Replace any use this class by C++20's
+// std::atomic<std::shared_ptr<T>>, once libcxx adds support for it.
+// [https://libcxx.llvm.org/docs/Cxx2aStatus.html]
+template <class T>
+class atomic_shared_ptr {
+ using ptr_t = std::shared_ptr<T>;
+
+ public:
+ atomic_shared_ptr() = default;
+ explicit atomic_shared_ptr(ptr_t value) : value_{std::move(value)} {}
+ atomic_shared_ptr(atomic_shared_ptr const& other) = delete;
+ atomic_shared_ptr(atomic_shared_ptr&& other) = delete;
+ ~atomic_shared_ptr() = default;
+
+ auto operator=(atomic_shared_ptr const& other)
+ -> atomic_shared_ptr& = delete;
+ auto operator=(atomic_shared_ptr&& other) -> atomic_shared_ptr& = delete;
+ auto operator=(ptr_t desired) -> ptr_t { // NOLINT
+ std::shared_lock lock(mutex_);
+ value_ = desired;
+ return desired;
+ }
+ operator ptr_t() const { value_; } // NOLINT
+
+ void store(ptr_t desired) {
+ std::shared_lock lock(mutex_);
+ value_ = std::move(desired);
+ }
+ [[nodiscard]] auto load() const -> ptr_t { return value_; }
+
+ void notify_one() { cv_.notify_one(); }
+ void notify_all() { cv_.notify_all(); }
+ void wait(ptr_t old) const {
+ std::unique_lock lock(mutex_);
+ cv_.wait(lock, [this, &old]() { return value_ != old; });
+ }
+
+ private:
+ ptr_t value_{};
+ mutable std::shared_mutex mutex_{};
+ mutable std::condition_variable_any cv_{};
+};
+
+#endif // INCLUDED_SRC_UTILS_CPP_ATOMIC_HPP
diff --git a/src/utils/cpp/concepts.hpp b/src/utils/cpp/concepts.hpp
new file mode 100644
index 00000000..92718b43
--- /dev/null
+++ b/src/utils/cpp/concepts.hpp
@@ -0,0 +1,55 @@
+#ifndef INCLUDED_SRC_UTILS_CPP_CONCEPTS_HPP
+#define INCLUDED_SRC_UTILS_CPP_CONCEPTS_HPP
+
+#include <string>
+#include <type_traits>
+
+// TODO(modernize): remove this once std::derived_from is shipped with libcxx
+template <class T, class U>
+concept derived_from = std::is_base_of_v<U, T>&&
+ std::is_convertible_v<const volatile T*, const volatile U*>;
+
+// TODO(modernize): remove this once std::same_as is shipped with libcxx
+template <class T, class U>
+concept same_as = std::is_same_v<T, U>and std::is_same_v<U, T>;
+
+template <class T>
+concept ContainsString = requires {
+ typename T::value_type;
+}
+and std::is_same_v<typename T::value_type, std::string>;
+
+template <class T>
+concept HasSize = requires(T const c) {
+ { c.size() }
+ ->same_as<std::size_t>; // TODO(modernize): replace by std::same_as
+};
+
+template <typename T>
+concept HasToString = requires(T const t) {
+ { t.ToString() }
+ ->same_as<std::string>; // TODO(modernize): replace by std::same_as
+};
+
+template <class T>
+concept InputIterableContainer = requires(T const c) {
+ { c.begin() }
+ ->same_as<typename T::const_iterator>; // TODO(modernize): replace by
+ // std::input_iterator
+ { c.end() }
+ ->same_as<typename T::const_iterator>; // TODO(modernize): replace by
+ // std::input_iterator
+};
+
+template <class T>
+concept OutputIterableContainer = InputIterableContainer<T>and requires(T c) {
+ { std::inserter(c, c.begin()) }
+ ->same_as<std::insert_iterator<T>>; // TODO(modernize): replace by
+ // std::output_iterator
+};
+
+template <class T>
+concept InputIterableStringContainer =
+ InputIterableContainer<T>and ContainsString<T>;
+
+#endif // INCLUDED_SRC_UTILS_CPP_CONCEPTS_HPP
diff --git a/src/utils/cpp/hash_combine.hpp b/src/utils/cpp/hash_combine.hpp
new file mode 100644
index 00000000..65c0c8ad
--- /dev/null
+++ b/src/utils/cpp/hash_combine.hpp
@@ -0,0 +1,15 @@
+#ifndef INCLUDED_SRC_UTILS_CPP_HASH_COMBINE_HPP
+#define INCLUDED_SRC_UTILS_CPP_HASH_COMBINE_HPP
+
+#include "gsl-lite/gsl-lite.hpp"
+
+// Taken from Boost, as hash_combine did not yet make it to STL.
+// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2017/p0814r0.pdf
+template <class T>
+inline auto hash_combine(gsl::not_null<std::size_t*> const& seed, T const& v)
+ -> void {
+ *seed ^=
+ std::hash<T>{}(v) + 0x9e3779b9 + (*seed << 6) + (*seed >> 2); // NOLINT
+}
+
+#endif
diff --git a/src/utils/cpp/hex_string.hpp b/src/utils/cpp/hex_string.hpp
new file mode 100644
index 00000000..86ea1b9e
--- /dev/null
+++ b/src/utils/cpp/hex_string.hpp
@@ -0,0 +1,19 @@
+#ifndef INCLUDED_SRC_UTILS_CPP_HEX_STRING_HPP
+#define INCLUDED_SRC_UTILS_CPP_HEX_STRING_HPP
+
+#include <iomanip>
+#include <sstream>
+#include <string>
+
+[[nodiscard]] static inline auto ToHexString(std::string const& bytes)
+ -> std::string {
+ std::ostringstream ss{};
+ ss << std::hex << std::setfill('0');
+ for (auto const& b : bytes) {
+ ss << std::setw(2)
+ << static_cast<int>(static_cast<unsigned char const>(b));
+ }
+ return ss.str();
+}
+
+#endif // INCLUDED_SRC_UTILS_CPP_HEX_STRING_HPP
diff --git a/src/utils/cpp/json.hpp b/src/utils/cpp/json.hpp
new file mode 100644
index 00000000..8945e975
--- /dev/null
+++ b/src/utils/cpp/json.hpp
@@ -0,0 +1,83 @@
+#ifndef INCLUDED_SRC_UTILS_CPP_JSON_HPP
+#define INCLUDED_SRC_UTILS_CPP_JSON_HPP
+
+#include <algorithm>
+#include <optional>
+#include <sstream>
+#include <string>
+
+#include "nlohmann/json.hpp"
+#include "gsl-lite/gsl-lite.hpp"
+
+template <typename ValueT>
+auto ExtractValueAs(
+ nlohmann::json const& j,
+ std::string const& key,
+ std::function<void(std::string const& error)>&& logger =
+ [](std::string const & /*unused*/) -> void {}) noexcept
+ -> std::optional<ValueT> {
+ try {
+ auto it = j.find(key);
+ if (it == j.end()) {
+ logger("key " + key + " cannot be found in JSON object");
+ return std::nullopt;
+ }
+ return it.value().template get<ValueT>();
+ } catch (std::exception& e) {
+ logger(e.what());
+ return std::nullopt;
+ }
+}
+
+namespace detail {
+
+[[nodiscard]] static inline auto IndentListsOnlyUntilDepth(
+ nlohmann::json const& json,
+ std::string const& indent,
+ std::size_t until,
+ std::size_t depth) -> std::string {
+ using iterator = std::ostream_iterator<std::string>;
+ if (json.is_object()) {
+ std::size_t i{};
+ std::ostringstream oss{};
+ oss << '{' << std::endl;
+ for (auto const& [key, value] : json.items()) {
+ std::fill_n(iterator{oss}, depth + 1, indent);
+ oss << nlohmann::json(key).dump() << ": "
+ << IndentListsOnlyUntilDepth(value, indent, until, depth + 1)
+ << (++i == json.size() ? "" : ",") << std::endl;
+ }
+ std::fill_n(iterator{oss}, depth, indent);
+ oss << '}';
+ gsl_EnsuresAudit(nlohmann::json::parse(oss.str()) == json);
+ return oss.str();
+ }
+ if (json.is_array() and depth < until) {
+ std::size_t i{};
+ std::ostringstream oss{};
+ oss << '[' << std::endl;
+ for (auto const& value : json) {
+ std::fill_n(iterator{oss}, depth + 1, indent);
+ oss << IndentListsOnlyUntilDepth(value, indent, until, depth + 1)
+ << (++i == json.size() ? "" : ",") << std::endl;
+ }
+ std::fill_n(iterator{oss}, depth, indent);
+ oss << ']';
+ gsl_EnsuresAudit(nlohmann::json::parse(oss.str()) == json);
+ return oss.str();
+ }
+ return json.dump();
+}
+
+} // namespace detail
+
+/// \brief Dump json with indent. Indent lists only until specified depth.
+[[nodiscard]] static inline auto IndentListsOnlyUntilDepth(
+ nlohmann::json const& json,
+ std::size_t indent,
+ std::size_t until_depth = 0) -> std::string {
+ return detail::IndentListsOnlyUntilDepth(
+ json, std::string(indent, ' '), until_depth, 0);
+}
+
+#endif // INCLUDED_SRC_UTILS_CPP_JSON_HPP
diff --git a/src/utils/cpp/type_safe_arithmetic.hpp b/src/utils/cpp/type_safe_arithmetic.hpp
new file mode 100644
index 00000000..21bba0b5
--- /dev/null
+++ b/src/utils/cpp/type_safe_arithmetic.hpp
@@ -0,0 +1,197 @@
+#ifndef INCLUDED_SRC_UTILS_CPP_TYPE_SAFE_ARITHMETIC_HPP
+#define INCLUDED_SRC_UTILS_CPP_TYPE_SAFE_ARITHMETIC_HPP
+
+#include <limits>
+#include <type_traits>
+
+#include "gsl-lite/gsl-lite.hpp"
+
+/// \struct type_safe_arithmetic_tag
+/// \brief Abstract tag defining types and limits for custom arithmetic types.
+/// Usage example:
+/// struct my_type_tag : type_safe_arithmetic_tag<int, -2, +3> {};
+/// using my_type_t = type_safe_arithmetic<my_type_tag>;
+template <typename T,
+ T MIN_VALUE = std::numeric_limits<T>::lowest(),
+ T MAX_VALUE = std::numeric_limits<T>::max(),
+ T SMALLEST_VALUE = std::numeric_limits<T>::min()>
+struct type_safe_arithmetic_tag {
+ static_assert(std::is_arithmetic<T>::value,
+ "T must be an arithmetic type (integer or floating-point)");
+
+ using value_t = T;
+ using reference_t = T&;
+ using const_reference_t = T const&;
+ using pointer_t = T*;
+ using const_pointer_t = T const*;
+
+ static constexpr value_t max_value = MAX_VALUE;
+ static constexpr value_t min_value = MIN_VALUE;
+ static constexpr value_t smallest_value = SMALLEST_VALUE;
+};
+
+/// \class type_safe_arithmetic
+/// \brief Abstract class for defining custom arithmetic types.
+/// \tparam TAG The actual \ref type_safe_arithmetic_tag
+template <typename TAG>
+class type_safe_arithmetic {
+ typename TAG::value_t m_value{};
+
+ public:
+ using tag_t = TAG;
+ using value_t = typename tag_t::value_t;
+ using reference_t = typename tag_t::reference_t;
+ using const_reference_t = typename tag_t::const_reference_t;
+ using pointer_t = typename tag_t::pointer_t;
+ using const_pointer_t = typename tag_t::const_pointer_t;
+
+ static constexpr value_t max_value = tag_t::max_value;
+ static constexpr value_t min_value = tag_t::min_value;
+ static constexpr value_t smallest_value = tag_t::smallest_value;
+
+ constexpr type_safe_arithmetic() = default;
+
+ // NOLINTNEXTLINE
+ constexpr /*explicit*/ type_safe_arithmetic(value_t value) { set(value); }
+
+ type_safe_arithmetic(type_safe_arithmetic const&) = default;
+ type_safe_arithmetic(type_safe_arithmetic&&) noexcept = default;
+ auto operator=(type_safe_arithmetic const&)
+ -> type_safe_arithmetic& = default;
+ auto operator=(type_safe_arithmetic&&) noexcept
+ -> type_safe_arithmetic& = default;
+ ~type_safe_arithmetic() = default;
+
+ auto operator=(value_t value) -> type_safe_arithmetic& {
+ set(value);
+ return *this;
+ }
+
+ // NOLINTNEXTLINE
+ constexpr /*explicit*/ operator value_t() const { return m_value; }
+
+ constexpr auto get() const -> value_t { return m_value; }
+
+ constexpr void set(value_t value) {
+ gsl_Expects(value >= min_value && value <= max_value &&
+ "value output of range");
+ m_value = value;
+ }
+
+ auto pointer() const -> const_pointer_t { return &m_value; }
+};
+
+// template <typename TAG>
+// bool operator==(type_safe_arithmetic<TAG> lhs, type_safe_arithmetic<TAG> rhs)
+// {
+// return lhs.get() == rhs.get();
+// }
+//
+// template <typename TAG>
+// bool operator!=(type_safe_arithmetic<TAG> lhs, type_safe_arithmetic<TAG> rhs)
+// {
+// return !(lhs == rhs);
+// }
+//
+// template <typename TAG>
+// bool operator>(type_safe_arithmetic<TAG> lhs, type_safe_arithmetic<TAG> rhs)
+// {
+// return lhs.get() > rhs.get();
+// }
+//
+// template <typename TAG>
+// bool operator>=(type_safe_arithmetic<TAG> lhs, type_safe_arithmetic<TAG> rhs)
+// {
+// return lhs.get() >= rhs.get();
+// }
+//
+// template <typename TAG>
+// bool operator<(type_safe_arithmetic<TAG> lhs, type_safe_arithmetic<TAG> rhs)
+// {
+// return lhs.get() < rhs.get();
+// }
+//
+// template <typename TAG>
+// bool operator<=(type_safe_arithmetic<TAG> lhs, type_safe_arithmetic<TAG> rhs)
+// {
+// return lhs.get() <= rhs.get();
+// }
+//
+// template <typename TAG>
+// type_safe_arithmetic<TAG> operator+(type_safe_arithmetic<TAG> lhs,
+// type_safe_arithmetic<TAG> rhs) {
+// return type_safe_arithmetic<TAG>{lhs.get() + rhs.get()};
+// }
+
+template <typename TAG>
+auto operator+=(type_safe_arithmetic<TAG>& lhs, type_safe_arithmetic<TAG> rhs)
+ -> type_safe_arithmetic<TAG>& {
+ lhs.set(lhs.get() + rhs.get());
+ return lhs;
+}
+
+// template <typename TAG>
+// type_safe_arithmetic<TAG> operator-(type_safe_arithmetic<TAG> lhs,
+// type_safe_arithmetic<TAG> rhs) {
+// return type_safe_arithmetic<TAG>{lhs.get() - rhs.get()};
+// }
+//
+// template <typename TAG>
+// type_safe_arithmetic<TAG>& operator-=(type_safe_arithmetic<TAG>& lhs,
+// type_safe_arithmetic<TAG> rhs) {
+// lhs.set(lhs.get() - rhs.get());
+// return lhs;
+// }
+//
+// template <typename TAG>
+// type_safe_arithmetic<TAG> operator*(type_safe_arithmetic<TAG> lhs,
+// typename TAG::value_t rhs) {
+// return type_safe_arithmetic<TAG>{lhs.get() - rhs};
+// }
+//
+// template <typename TAG>
+// type_safe_arithmetic<TAG>& operator*=(type_safe_arithmetic<TAG>& lhs,
+// typename TAG::value_t rhs) {
+// lhs.set(lhs.get() * rhs);
+// return lhs;
+// }
+//
+// template <typename TAG>
+// type_safe_arithmetic<TAG> operator/(type_safe_arithmetic<TAG> lhs,
+// typename TAG::value_t rhs) {
+// return type_safe_arithmetic<TAG>{lhs.get() / rhs};
+// }
+//
+// template <typename TAG>
+// type_safe_arithmetic<TAG>& operator/=(type_safe_arithmetic<TAG>& lhs,
+// typename TAG::value_t rhs) {
+// lhs.set(lhs.get() / rhs);
+// return lhs;
+// }
+//
+// template <typename TAG>
+// type_safe_arithmetic<TAG>& operator++(type_safe_arithmetic<TAG>& a) {
+// return a += type_safe_arithmetic<TAG>{1};
+// }
+
+template <typename TAG>
+auto operator++(type_safe_arithmetic<TAG>& a, int)
+ -> type_safe_arithmetic<TAG> {
+ auto r = a;
+ a += type_safe_arithmetic<TAG>{1};
+ return r;
+}
+
+// template <typename TAG>
+// type_safe_arithmetic<TAG>& operator--(type_safe_arithmetic<TAG>& a) {
+// return a -= type_safe_arithmetic<TAG>{1};
+// }
+//
+// template <typename TAG>
+// type_safe_arithmetic<TAG> operator--(type_safe_arithmetic<TAG>& a, int) {
+// auto r = a;
+// a += type_safe_arithmetic<TAG>{1};
+// return r;
+// }
+
+#endif // INCLUDED_SRC_UTILS_CPP_TYPE_SAFE_ARITHMETIC_HPP