diff options
6 files changed, 341 insertions, 58 deletions
diff --git a/src/buildtool/build_engine/target_map/target_map.cpp b/src/buildtool/build_engine/target_map/target_map.cpp index 634db0dc..1e1436d3 100644 --- a/src/buildtool/build_engine/target_map/target_map.cpp +++ b/src/buildtool/build_engine/target_map/target_map.cpp @@ -328,6 +328,38 @@ auto ListDependencies( return deps.str(); } +auto ExprToTree(std::string const& context, + ExpressionPtr const& val) -> Tree::Ptr { + if (not val->IsMap()) { + throw Evaluator::EvaluationError{ + fmt::format("{} has to be a map of artifacts, " + "but found {}", + context, + val->ToString())}; + } + std::unordered_map<std::string, ArtifactDescription> artifacts; + artifacts.reserve(val->Map().size()); + for (auto const& [input_path, artifact] : val->Map()) { + if (not artifact->IsArtifact()) { + throw Evaluator::EvaluationError{ + fmt::format("{} has to be a map of artifacts, " + "but found {} for {}", + artifact->ToString(), + input_path, + context)}; + } + auto norm_path = ToNormalPath(std::filesystem::path{input_path}); + artifacts.emplace(std::move(norm_path), artifact->Artifact()); + } + auto conflict = BuildMaps::Target::Utils::tree_conflict(val); + if (conflict) { + throw Evaluator::EvaluationError{fmt::format( + "Tree conflicts on subtree {} of {}", *conflict, context)}; + } + auto tree = std::make_shared<Tree>(std::move(artifacts)); + return tree; +} + void withDependencies( const gsl::not_null<AnalyseContext*>& context, const std::vector<BuildMaps::Target::ConfiguredTarget>& transition_keys, @@ -491,6 +523,7 @@ void withDependencies( std::vector<ActionDescription::Ptr> actions{}; std::vector<std::string> blobs{}; std::vector<Tree::Ptr> trees{}; + std::vector<TreeOverlay::Ptr> tree_overlays{}; auto main_exp_fcts = FunctionMap::MakePtr( {{"FIELD", [¶ms](auto&& eval, auto const& expr, auto const& env) { @@ -813,36 +846,67 @@ void withDependencies( {"TREE", [&trees](auto&& eval, auto const& expr, auto const& env) { auto val = eval(expr->Get("$1", Expression::kEmptyMapExpr), env); - if (not val->IsMap()) { + auto tree = ExprToTree("TREE argument", val); + auto tree_id = tree->Id(); + trees.emplace_back(std::move(tree)); + return ExpressionPtr{ArtifactDescription::CreateTree(tree_id)}; + }}, + + {"TREE_OVERLAY", + [&tree_overlays, &trees]( + auto&& eval, auto const& expr, auto const& env) { + auto val = eval(expr->Get("$1", Expression::kEmptyList), env); + if (not val->IsList()) { throw Evaluator::EvaluationError{ - fmt::format("TREE argument has to be a map of artifacts, " - "but found {}", + fmt::format("TREE_OVERLAY argument has to be a list of " + "stages, but found {}", val->ToString())}; } - std::unordered_map<std::string, ArtifactDescription> artifacts; - artifacts.reserve(val->Map().size()); - for (auto const& [input_path, artifact] : val->Map()) { - if (not artifact->IsArtifact()) { - throw Evaluator::EvaluationError{fmt::format( - "TREE argument has to be a map of artifacts, " - "but found {} for {}", - artifact->ToString(), - input_path)}; - } - auto norm_path = - ToNormalPath(std::filesystem::path{input_path}); - artifacts.emplace(std::move(norm_path), artifact->Artifact()); + TreeOverlay::to_overlay_t parts{}; + for (std::size_t i = 0; i < val->List().size(); i++) { + auto& entry = val->List()[i]; + auto context = + fmt::format("Entry {} of TREE_OVERLAY argument", i); + auto tree = ExprToTree(context, entry); + auto tree_id = tree->Id(); + trees.emplace_back(std::move(tree)); + parts.emplace_back(ArtifactDescription::CreateTree(tree_id)); } - auto conflict = BuildMaps::Target::Utils::tree_conflict(val); - if (conflict) { - throw Evaluator::EvaluationError{ - fmt::format("TREE conflicts on subtree {}", *conflict)}; + auto overlay = std::make_shared<TreeOverlay>(std::move(parts), + /*disjoint=*/false); + auto overlay_id = overlay->Id(); + tree_overlays.emplace_back(std::move(overlay)); + return ExpressionPtr{ + ArtifactDescription::CreateTreeOverlay(overlay_id)}; + }}, + {"DISJOINT_TREE_OVERLAY", + [&tree_overlays, &trees]( + auto&& eval, auto const& expr, auto const& env) { + auto val = eval(expr->Get("$1", Expression::kEmptyList), env); + if (not val->IsList()) { + throw Evaluator::EvaluationError{fmt::format( + "DISJOINT_TREE_OVERLAY argument has to be a list of " + "stages, but found {}", + val->ToString())}; } - auto tree = std::make_shared<Tree>(std::move(artifacts)); - auto tree_id = tree->Id(); - trees.emplace_back(std::move(tree)); - return ExpressionPtr{ArtifactDescription::CreateTree(tree_id)}; + TreeOverlay::to_overlay_t parts{}; + for (std::size_t i = 0; i < val->List().size(); i++) { + auto& entry = val->List()[i]; + auto context = fmt::format( + "Entry {} of DISJOINT_TREE_OVERLAY argument", i); + auto tree = ExprToTree(context, entry); + auto tree_id = tree->Id(); + trees.emplace_back(std::move(tree)); + parts.emplace_back(ArtifactDescription::CreateTree(tree_id)); + } + auto overlay = std::make_shared<TreeOverlay>(std::move(parts), + /*disjoint=*/true); + auto overlay_id = overlay->Id(); + tree_overlays.emplace_back(std::move(overlay)); + return ExpressionPtr{ + ArtifactDescription::CreateTreeOverlay(overlay_id)}; }}, + {"VALUE_NODE", [](auto&& eval, auto const& expr, auto const& env) { auto val = eval(expr->Get("$1", Expression::kNone), env); @@ -1044,7 +1108,7 @@ void withDependencies( std::move(actions), std::move(blobs), std::move(trees), - std::vector<TreeOverlay::Ptr>{}, + std::move(tree_overlays), std::move(effective_vars), std::move(tainted), std::move(implied_export), diff --git a/test/buildtool/build_engine/target_map/TARGETS b/test/buildtool/build_engine/target_map/TARGETS index e477393e..67d35ade 100644 --- a/test/buildtool/build_engine/target_map/TARGETS +++ b/test/buildtool/build_engine/target_map/TARGETS @@ -70,6 +70,7 @@ , ["@", "src", "src/buildtool/common", "config"] , ["@", "src", "src/buildtool/common", "statistics"] , ["@", "src", "src/buildtool/common", "tree"] + , ["@", "src", "src/buildtool/common", "tree_overlay"] , ["@", "src", "src/buildtool/common/remote", "remote_common"] , ["@", "src", "src/buildtool/common/remote", "retry_config"] , ["@", "src", "src/buildtool/crypto", "hash_function"] @@ -117,39 +118,9 @@ , "test_data": { "type": ["@", "rules", "data", "staged"] , "srcs": - [ "data_src/a/b/targets_here/c/d/foo" - , "data_src/file_reference/hello.txt" - , "data_src/foo" - , "data_src/simple_rules/implicit_script.sh" - , "data_src/simple_targets/bar.txt" - , "data_src/simple_targets/baz.txt" - , "data_src/simple_targets/foo.txt" - , "data_src/tree/foo.txt" - , "data_src/tree/tree/foo.txt" - , "data_src/x/foo" - , "data_src/x/x/foo" - , "data_src/x/x/x/foo" - , "data_src/x/x/x/x/foo" - , "data_src/x/x/x/x/x/foo" - , "data_targets/TARGETS" - , "data_targets/a/b/targets_here/TARGETS" - , "data_targets/bad_targets/TARGETS" - , "data_targets/config_targets/TARGETS" - , "data_targets/file_reference/TARGETS" - , "data_targets/result/TARGETS" - , "data_targets/simple_rules/TARGETS" - , "data_targets/simple_targets/TARGETS" - , "data_targets/symlink_reference/TARGETS" - , "data_targets/tree/TARGETS" - , "data_targets/x/TARGETS" - , "data_targets/x/x/TARGETS" - , "data_targets/x/x/x/TARGETS" - , "data_targets/x/x/x/x/TARGETS" - , "data_targets/x/x/x/x/x/TARGETS" - , "data_rules/result/RULES" - , "data_rules/rule/RULES" - , "data_rules/simple_rules/RULES" - , "data_rules/tree/RULES" + [ ["TREE", null, "data_src"] + , ["TREE", null, "data_targets"] + , ["TREE", null, "data_rules"] ] , "stage": ["test", "buildtool", "build_engine", "target_map"] } diff --git a/test/buildtool/build_engine/target_map/data_rules/tree_overlay/RULES b/test/buildtool/build_engine/target_map/data_rules/tree_overlay/RULES new file mode 100644 index 00000000..eb809da4 --- /dev/null +++ b/test/buildtool/build_engine/target_map/data_rules/tree_overlay/RULES @@ -0,0 +1,55 @@ +{ "overlay": + { "target_fields": ["deps"] + , "expression": + { "type": "let*" + , "bindings": + [ [ "deps" + , { "type": "foreach" + , "range": {"type": "FIELD", "name": "deps"} + , "body": + {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "_"}} + } + ] + , [ "overlay tree" + , {"type": "TREE_OVERLAY", "$1": {"type": "var", "name": "deps"}} + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": + { "type": "singleton_map" + , "key": "it" + , "value": {"type": "var", "name": "overlay tree"} + } + } + } + } +, "disjoint overlay": + { "target_fields": ["deps"] + , "expression": + { "type": "let*" + , "bindings": + [ [ "deps" + , { "type": "foreach" + , "range": {"type": "FIELD", "name": "deps"} + , "body": + {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "_"}} + } + ] + , [ "overlay tree" + , { "type": "DISJOINT_TREE_OVERLAY" + , "$1": {"type": "var", "name": "deps"} + } + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": + { "type": "singleton_map" + , "key": "it" + , "value": {"type": "var", "name": "overlay tree"} + } + } + } + } +} diff --git a/test/buildtool/build_engine/target_map/data_src/tree_overlay/x b/test/buildtool/build_engine/target_map/data_src/tree_overlay/x new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/buildtool/build_engine/target_map/data_src/tree_overlay/x diff --git a/test/buildtool/build_engine/target_map/data_targets/tree_overlay/TARGETS b/test/buildtool/build_engine/target_map/data_targets/tree_overlay/TARGETS new file mode 100644 index 00000000..088d47de --- /dev/null +++ b/test/buildtool/build_engine/target_map/data_targets/tree_overlay/TARGETS @@ -0,0 +1,6 @@ +{ "empty": {"type": ["tree_overlay", "overlay"]} +, "one stage": {"type": ["tree_overlay", "overlay"], "deps": ["x"]} +, "disjoint empty": {"type": ["tree_overlay", "disjoint overlay"]} +, "disjoint one stage": + {"type": ["tree_overlay", "disjoint overlay"], "deps": ["x"]} +} diff --git a/test/buildtool/build_engine/target_map/target_map.test.cpp b/test/buildtool/build_engine/target_map/target_map.test.cpp index 87698b13..0b51355b 100644 --- a/test/buildtool/build_engine/target_map/target_map.test.cpp +++ b/test/buildtool/build_engine/target_map/target_map.test.cpp @@ -46,6 +46,7 @@ #include "src/buildtool/common/repository_config.hpp" #include "src/buildtool/common/statistics.hpp" #include "src/buildtool/common/tree.hpp" +#include "src/buildtool/common/tree_overlay.hpp" #include "src/buildtool/crypto/hash_function.hpp" #include "src/buildtool/execution_api/common/api_bundle.hpp" #include "src/buildtool/execution_api/local/config.hpp" @@ -1500,6 +1501,192 @@ TEST_CASE("trees", "[target_map]") { } } +TEST_CASE("tree_overlays", "[target_map]") { + auto const storage_config = TestStorageConfig::Create(); + auto const storage = Storage::Create(&storage_config.Get()); + + auto repo_config = SetupConfig(); + auto directory_entries = + BuildMaps::Base::CreateDirectoryEntriesMap(&repo_config); + auto source = BuildMaps::Base::CreateSourceTargetMap( + &directory_entries, + &repo_config, + storage_config.Get().hash_function.GetType()); + auto targets_file_map = + BuildMaps::Base::CreateTargetsFileMap(&repo_config, 0); + auto rule_file_map = BuildMaps::Base::CreateRuleFileMap(&repo_config, 0); + static auto expressions_file_map = + BuildMaps::Base::CreateExpressionFileMap(&repo_config, 0); + auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map, + &repo_config); + auto rule_map = + BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map, &repo_config); + BuildMaps::Target::ResultTargetMap result_map{0}; + Statistics stats{}; + Progress exports_progress{}; + + auto serve_config = TestServeConfig::ReadFromEnvironment(); + REQUIRE(serve_config); + + LocalExecutionConfig local_exec_config{}; + + // pack the local context instances to be passed to ApiBundle + LocalContext const local_context{.exec_config = &local_exec_config, + .storage_config = &storage_config.Get(), + .storage = &storage}; + + Auth auth{}; + RetryConfig retry_config{}; + RemoteExecutionConfig remote_exec_config{}; + + // pack the remote context instances to be passed to ApiBundle + RemoteContext const remote_context{.auth = &auth, + .retry_config = &retry_config, + .exec_config = &remote_exec_config}; + + auto const apis = ApiBundle::Create(&local_context, + &remote_context, + /*repo_config=*/nullptr); + + auto serve = + ServeApi::Create(*serve_config, &local_context, &remote_context, &apis); + + AnalyseContext ctx{.repo_config = &repo_config, + .storage = &storage, + .statistics = &stats, + .progress = &exports_progress, + .serve = serve ? &*serve : nullptr}; + + auto absent_target_variables_map = + BuildMaps::Target::CreateAbsentTargetVariablesMap(&ctx, 0); + + auto absent_target_map = BuildMaps::Target::CreateAbsentTargetMap( + &ctx, &result_map, &absent_target_variables_map, 0); + + auto target_map = BuildMaps::Target::CreateTargetMap(&ctx, + &source, + &targets_file_map, + &rule_map, + &directory_entries, + &absent_target_map, + &result_map); + + AnalysedTargetPtr result; + bool error{false}; + std::string error_msg; + auto empty_config = Configuration{Expression::FromJson(R"({})"_json)}; + + SECTION("empty") { + error = false; + error_msg = "NONE"; + { + TaskSystem ts; + target_map.ConsumeAfterKeysReady( + &ts, + {BuildMaps::Target::ConfiguredTarget{ + .target = + BuildMaps::Base::EntityName{ + "", "tree_overlay", "empty"}, + .config = empty_config}}, + [&result](auto values) { result = *values[0]; }, + [&error, &error_msg](std::string const& msg, bool /*unused*/) { + error = true; + error_msg = msg; + }); + } + CHECK(not error); + CHECK(error_msg == "NONE"); + CHECK(result->TreeOverlays().size() == 1); + CHECK(result->TreeOverlays()[0]->ToJson()["trees"] == + nlohmann::json::array()); + CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == false); + } + + SECTION("implicit tree") { + error = false; + error_msg = "NONE"; + { + TaskSystem ts; + target_map.ConsumeAfterKeysReady( + &ts, + {BuildMaps::Target::ConfiguredTarget{ + .target = + BuildMaps::Base::EntityName{ + "", "tree_overlay", "one stage"}, + .config = empty_config}}, + [&result](auto values) { result = *values[0]; }, + [&error, &error_msg](std::string const& msg, bool /*unused*/) { + error = true; + error_msg = msg; + }); + } + CHECK(not error); + CHECK(error_msg == "NONE"); + CHECK(result->Trees().size() == 1); + CHECK(result->Trees()[0]->ToJson()["x"]["type"] == "LOCAL"); + CHECK(result->TreeOverlays().size() == 1); + CHECK(result->TreeOverlays()[0]->ToJson()["trees"].size() == 1); + CHECK(result->TreeOverlays()[0]->ToJson()["trees"][0]["type"] == + "TREE"); + CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == false); + } + + SECTION("disjoint empty") { + error = false; + error_msg = "NONE"; + { + TaskSystem ts; + target_map.ConsumeAfterKeysReady( + &ts, + {BuildMaps::Target::ConfiguredTarget{ + .target = + BuildMaps::Base::EntityName{ + "", "tree_overlay", "disjoint empty"}, + .config = empty_config}}, + [&result](auto values) { result = *values[0]; }, + [&error, &error_msg](std::string const& msg, bool /*unused*/) { + error = true; + error_msg = msg; + }); + } + CHECK(not error); + CHECK(error_msg == "NONE"); + CHECK(result->TreeOverlays().size() == 1); + CHECK(result->TreeOverlays()[0]->ToJson()["trees"] == + nlohmann::json::array()); + CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == true); + } + + SECTION("disjoint implicit tree") { + error = false; + error_msg = "NONE"; + { + TaskSystem ts; + target_map.ConsumeAfterKeysReady( + &ts, + {BuildMaps::Target::ConfiguredTarget{ + .target = + BuildMaps::Base::EntityName{ + "", "tree_overlay", "disjoint one stage"}, + .config = empty_config}}, + [&result](auto values) { result = *values[0]; }, + [&error, &error_msg](std::string const& msg, bool /*unused*/) { + error = true; + error_msg = msg; + }); + } + CHECK(not error); + CHECK(error_msg == "NONE"); + CHECK(result->Trees().size() == 1); + CHECK(result->Trees()[0]->ToJson()["x"]["type"] == "LOCAL"); + CHECK(result->TreeOverlays().size() == 1); + CHECK(result->TreeOverlays()[0]->ToJson()["trees"].size() == 1); + CHECK(result->TreeOverlays()[0]->ToJson()["trees"][0]["type"] == + "TREE"); + CHECK(result->TreeOverlays()[0]->ToJson()["disjoint"] == true); + } +} + TEST_CASE("RESULT error reporting", "[target_map]") { auto const storage_config = TestStorageConfig::Create(); auto const storage = Storage::Create(&storage_config.Get()); |