define AbsentTargetMap for handling export targets in absent repositories

7 files changed, 310 insertions, 17 deletions

diff --git a/src/buildtool/build_engine/target_map/TARGETS b/src/buildtool/build_engine/target_map/TARGETS
index e2c76c8e..0ddbdf27 100644
--- a/src/buildtool/build_engine/target_map/TARGETS
+++ b/src/buildtool/build_engine/target_map/TARGETS
@@ -40,6 +40,7 @@
   , "deps":
     [ "configured_target"
     , "result_map"
+    , "absent_target_map"
     , ["src/buildtool/build_engine/analysed_target", "target"]
     , ["src/buildtool/build_engine/base_maps", "rule_map"]
     , ["src/buildtool/build_engine/base_maps", "source_map"]
@@ -63,6 +64,8 @@
     , ["src/utils/cpp", "path"]
     , ["src/utils/cpp", "path_hash"]
     , ["src/utils/cpp", "vector"]
+    , ["src/buildtool/serve_api/remote", "serve_api"]
+    , ["src/buildtool/serve_api/remote", "config"]
     ]
   }
 , "target_map_testable_internals":
@@ -81,4 +84,26 @@
     ]
   , "stage": ["src", "buildtool", "build_engine", "target_map"]
   }
+, "absent_target_map":
+  { "type": ["@", "rules", "CC", "library"]
+  , "name": ["absent_target_map"]
+  , "hdrs": ["absent_target_map.hpp"]
+  , "srcs": ["absent_target_map.cpp"]
+  , "private-hdrs": ["built_in_rules.hpp", "export.hpp", "utils.hpp"]
+  , "deps":
+    [ "configured_target"
+    , "result_map"
+    , ["src/buildtool/build_engine/analysed_target", "target"]
+    , ["src/buildtool/multithreading", "async_map_consumer"]
+    , ["@", "gsl", "", "gsl"]
+    ]
+  , "stage": ["src", "buildtool", "build_engine", "target_map"]
+  , "private-deps":
+    [ ["src/buildtool/common", "common"]
+    , ["src/buildtool/common", "config"]
+    , ["src/buildtool/storage", "storage"]
+    , ["src/buildtool/logging", "logging"]
+    , ["src/buildtool/serve_api/remote", "serve_api"]
+    ]
+  }
 }
diff --git a/src/buildtool/build_engine/target_map/absent_target_map.cpp b/src/buildtool/build_engine/target_map/absent_target_map.cpp
new file mode 100644
index 00000000..38ab6cfa
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/absent_target_map.cpp
@@ -0,0 +1,137 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "src/buildtool/build_engine/target_map/absent_target_map.hpp"
+
+#ifndef BOOTSTRAP_BUILD_TOOL
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/serve_api/remote/serve_api.hpp"
+#include "src/buildtool/storage/target_cache_key.hpp"
+#endif
+auto BuildMaps::Target::CreateAbsentTargetMap(
+    const gsl::not_null<ResultTargetMap*>& result_map,
+    std::size_t jobs) -> AbsentTargetMap {
+#ifndef BOOTSTRAP_BUILD_TOOL
+    auto target_reader = [result_map](auto /*ts*/,
+                                      auto setter,
+                                      auto logger,
+                                      auto /*subcaller*/,
+                                      auto key) {
+        // assumptions:
+        // - target with absent targets file requested
+        // - ServeApi correctly configured
+        auto const& repo_name = key.target.ToModule().repository;
+        auto target_root_id = RepositoryConfig::Instance()
+                                  .TargetRoot(repo_name)
+                                  ->GetAbsentTreeId();
+        if (!target_root_id) {
+            (*logger)(fmt::format("Failed to get the target root id for "
+                                  "repository \"{}\"",
+                                  repo_name),
+                      true);
+            return;
+        }
+        auto flexible_vars = ServeApi::ServeTargetVariables(
+            *target_root_id,
+            *RepositoryConfig::Instance().TargetFileName(repo_name),
+            key.target.GetNamedTarget().name);
+        if (!flexible_vars) {
+            (*logger)(fmt::format("Failed to obtain flexible config variables "
+                                  "for target \"{}\"",
+                                  key.target.ToString()),
+                      true);
+            return;
+        }
+        // TODO(asartori): avoid code duplication in export.cpp
+        auto effective_config = key.config.Prune(*flexible_vars);
+        auto target_name = key.target.GetNamedTarget();
+        auto repo_key =
+            RepositoryConfig::Instance().RepositoryKey(target_name.repository);
+        if (!repo_key) {
+            (*logger)(
+                fmt::format("Failed to obtain repository key for repo \"{}\"",
+                            target_name.repository),
+                true);
+            return;
+        }
+        auto target_cache_key =
+            TargetCacheKey::Create(*repo_key, target_name, effective_config);
+        std::optional<std::pair<TargetCacheEntry, Artifact::ObjectInfo>>
+            target_cache_value{std::nullopt};
+        target_cache_value =
+            Storage::Instance().TargetCache().Read(*target_cache_key);
+        bool from_just_serve = false;
+        if (!target_cache_value) {
+            Logger::Log(LogLevel::Debug,
+                        "Querying just serve for export target {}",
+                        key.target.ToString());
+            target_cache_value = ServeApi::ServeTarget(*target_cache_key);
+            from_just_serve = true;
+        }
+
+        if (!target_cache_value) {
+            (*logger)(fmt::format("Could not get target cache value for key {}",
+                                  target_cache_key->Id().ToString()),
+                      true);
+            return;
+        }
+        auto const& [entry, info] = *target_cache_value;
+        if (auto result = entry.ToResult()) {
+            auto deps_info = TargetGraphInformation{
+                std::make_shared<BuildMaps::Target::ConfiguredTarget>(
+                    BuildMaps::Target::ConfiguredTarget{
+                        .target = key.target, .config = effective_config}),
+                {},
+                {},
+                {}};
+
+            auto analysis_result = std::make_shared<AnalysedTarget const>(
+                *result,
+                std::vector<ActionDescription::Ptr>{},
+                std::vector<std::string>{},
+                std::vector<Tree::Ptr>{},
+                std::unordered_set<std::string>{flexible_vars->begin(),
+                                                flexible_vars->end()},
+                std::set<std::string>{},
+                deps_info);
+
+            analysis_result = result_map->Add(key.target,
+                                              effective_config,
+                                              analysis_result,
+                                              std::nullopt,
+                                              true);
+
+            Logger::Log(LogLevel::Performance,
+                        "Export target {} served from {}: {} -> {}",
+                        key.target.ToString(),
+                        (from_just_serve ? "just-serve" : "cache"),
+                        target_cache_key->Id().ToString(),
+                        info.ToString());
+
+            (*setter)(std::move(analysis_result));
+            Statistics::Instance().IncrementExportsCachedCounter();
+            return;
+        }
+        (*logger)(fmt::format("Reading target entry for key {} failed",
+                              target_cache_key->Id().ToString()),
+                  true);
+    };
+#else
+    auto target_reader = [](auto /*ts*/,
+                            auto /*setter*/,
+                            auto /*logger*/,
+                            auto /*subcaller*/,
+                            auto /*key*/) {};
+#endif
+    return AbsentTargetMap{target_reader, jobs};
+}
diff --git a/src/buildtool/build_engine/target_map/absent_target_map.hpp b/src/buildtool/build_engine/target_map/absent_target_map.hpp
new file mode 100644
index 00000000..11959b2f
--- /dev/null
+++ b/src/buildtool/build_engine/target_map/absent_target_map.hpp
@@ -0,0 +1,30 @@
+// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_ABSENT_TARGET_MAP_HPP
+#define INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_ABSENT_TARGET_MAP_HPP
+#include "gsl/gsl"
+#include "src/buildtool/build_engine/analysed_target/analysed_target.hpp"
+#include "src/buildtool/build_engine/target_map/configured_target.hpp"
+#include "src/buildtool/build_engine/target_map/result_map.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+
+namespace BuildMaps::Target {
+using AbsentTargetMap = AsyncMapConsumer<ConfiguredTarget, AnalysedTargetPtr>;
+
+auto CreateAbsentTargetMap(const gsl::not_null<ResultTargetMap*>&,
+                           std::size_t jobs = 0) -> AbsentTargetMap;
+
+}  // namespace BuildMaps::Target
+#endif  // INCLUDED_SRC_BUILDTOOL_BUILD_ENGINE_TARGET_MAP_ABSENT_TARGET_MAP_HPP
diff --git a/src/buildtool/build_engine/target_map/target_map.cpp b/src/buildtool/build_engine/target_map/target_map.cpp
index 96d25b5e..ca8fdddd 100644
--- a/src/buildtool/build_engine/target_map/target_map.cpp
+++ b/src/buildtool/build_engine/target_map/target_map.cpp
@@ -37,6 +37,10 @@
 #include "src/utils/cpp/gsl.hpp"
 #include "src/utils/cpp/path.hpp"
 #include "src/utils/cpp/vector.hpp"
+#ifndef BOOTSTRAP_BUILD_TOOL
+#include "src/buildtool/serve_api/remote/config.hpp"
+#include "src/buildtool/serve_api/remote/serve_api.hpp"
+#endif  // BOOTSTRAP_BUILD_TOOL
 
 namespace {
 
@@ -1595,6 +1599,21 @@ void GlobTargetWithDirEntry(
         });
 }
 
+#ifndef BOOTSTRAP_BUILD_TOOL
+// The remote execution endpoint specified on the command line must be the same
+// used by the provided just serve instance.
+[[nodiscard]] auto CheckServeAndExecutionEndpoints() -> bool {
+    auto sadd = RemoteServeConfig::RemoteAddress();
+    if (!sadd) {
+        Logger::Log(LogLevel::Error,
+                    "Absent root detected. Please provide "
+                    "--remote-serve-address and retry.");
+        return false;
+    }
+    return ServeApi::CheckServeRemoteExecution();
+}
+#endif  // BOOTSTRAP_BUILD_TOOL
+
 }  // namespace
 
 namespace BuildMaps::Target {
@@ -1604,12 +1623,14 @@ auto CreateTargetMap(
     const gsl::not_null<BuildMaps::Base::UserRuleMap*>& rule_map,
     const gsl::not_null<BuildMaps::Base::DirectoryEntriesMap*>&
         directory_entries_map,
+    const gsl::not_null<AbsentTargetMap*>& absent_target_map,
     const gsl::not_null<ResultTargetMap*>& result_map,
     std::size_t jobs) -> TargetMap {
     auto target_reader = [source_target_map,
                           targets_file_map,
                           rule_map,
                           result_map,
+                          absent_target_map,
                           directory_entries_map](auto ts,
                                                  auto setter,
                                                  auto logger,
@@ -1703,6 +1724,34 @@ auto CreateTargetMap(
 
             );
         }
+#ifndef BOOTSTRAP_BUILD_TOOL
+        else if (RepositoryConfig::Instance()
+                     .TargetRoot(key.target.ToModule().repository)
+                     ->IsAbsent()) {
+            static auto consistent_serve_and_remote_execution =
+                CheckServeAndExecutionEndpoints();
+            if (!consistent_serve_and_remote_execution) {
+                (*logger)(
+                    "Inconsistent remote execution endpoint and just serve "
+                    "configuration detected.",
+                    true);
+                return;
+            }
+            absent_target_map->ConsumeAfterKeysReady(
+                ts,
+                {key},
+                [setter](auto values) {
+                    (*setter)(AnalysedTargetPtr{*values[0]});
+                },
+                [logger, key](auto msg, auto fatal) {
+                    (*logger)(
+                        fmt::format("While processing absent target {}: {}",
+                                    key.target.ToString(),
+                                    msg),
+                        fatal);
+                });
+        }
+#endif
         else {
             targets_file_map->ConsumeAfterKeysReady(
                 ts,
diff --git a/src/buildtool/build_engine/target_map/target_map.hpp b/src/buildtool/build_engine/target_map/target_map.hpp
index 4baf07e3..d86c8ab1 100644
--- a/src/buildtool/build_engine/target_map/target_map.hpp
+++ b/src/buildtool/build_engine/target_map/target_map.hpp
@@ -20,6 +20,7 @@
 #include "src/buildtool/build_engine/base_maps/rule_map.hpp"
 #include "src/buildtool/build_engine/base_maps/source_map.hpp"
 #include "src/buildtool/build_engine/base_maps/targets_file_map.hpp"
+#include "src/buildtool/build_engine/target_map/absent_target_map.hpp"
 #include "src/buildtool/build_engine/target_map/configured_target.hpp"
 #include "src/buildtool/build_engine/target_map/result_map.hpp"
 #include "src/buildtool/multithreading/async_map_consumer.hpp"
@@ -33,6 +34,7 @@ auto CreateTargetMap(
     const gsl::not_null<BuildMaps::Base::TargetsFileMap*>&,
     const gsl::not_null<BuildMaps::Base::UserRuleMap*>&,
     const gsl::not_null<BuildMaps::Base::DirectoryEntriesMap*>&,
+    [[maybe_unused]] const gsl::not_null<AbsentTargetMap*>&,
     const gsl::not_null<ResultTargetMap*>&,
     std::size_t jobs = 0) -> TargetMap;
 
diff --git a/src/buildtool/main/analyse.cpp b/src/buildtool/main/analyse.cpp
index 96972703..6cccd1c3 100644
--- a/src/buildtool/main/analyse.cpp
+++ b/src/buildtool/main/analyse.cpp
@@ -20,6 +20,7 @@
 #include "src/buildtool/build_engine/base_maps/rule_map.hpp"
 #include "src/buildtool/build_engine/base_maps/source_map.hpp"
 #include "src/buildtool/build_engine/base_maps/targets_file_map.hpp"
+#include "src/buildtool/build_engine/target_map/absent_target_map.hpp"
 #include "src/buildtool/build_engine/target_map/target_map.hpp"
 #include "src/buildtool/multithreading/async_map_consumer.hpp"
 #include "src/buildtool/multithreading/task_system.hpp"
@@ -133,10 +134,12 @@ void DetectAndReportPending(std::string const& name,
     auto expr_map = Base::CreateExpressionMap(&expressions_file_map, jobs);
     auto rule_map = Base::CreateRuleMap(&rule_file_map, &expr_map, jobs);
     auto source_targets = Base::CreateSourceTargetMap(&directory_entries, jobs);
+    auto absent_target_map = Target::CreateAbsentTargetMap(result_map, jobs);
     auto target_map = Target::CreateTargetMap(&source_targets,
                                               &targets_file_map,
                                               &rule_map,
                                               &directory_entries,
+                                              &absent_target_map,
                                               result_map,
                                               jobs);
     Logger::Log(LogLevel::Info, "Requested target is {}", id.ToString());
diff --git a/test/buildtool/build_engine/target_map/target_map.test.cpp b/test/buildtool/build_engine/target_map/target_map.test.cpp
index 6397ba1d..d9144dfb 100644
--- a/test/buildtool/build_engine/target_map/target_map.test.cpp
+++ b/test/buildtool/build_engine/target_map/target_map.test.cpp
@@ -74,8 +74,14 @@ TEST_CASE("simple targets") {
     auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map);
     auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map);
     BuildMaps::Target::ResultTargetMap result_map{0};
-    auto target_map = BuildMaps::Target::CreateTargetMap(
-        &source, &targets_file_map, &rule_map, &directory_entries, &result_map);
+    auto absent_target_map =
+        BuildMaps::Target::CreateAbsentTargetMap(&result_map, 0);
+    auto target_map = BuildMaps::Target::CreateTargetMap(&source,
+                                                         &targets_file_map,
+                                                         &rule_map,
+                                                         &directory_entries,
+                                                         &absent_target_map,
+                                                         &result_map);
 
     AnalysedTargetPtr result;
     bool error{false};
@@ -407,8 +413,14 @@ TEST_CASE("configuration deduplication") {
     auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map);
     auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map);
     BuildMaps::Target::ResultTargetMap result_map{0};
-    auto target_map = BuildMaps::Target::CreateTargetMap(
-        &source, &targets_file_map, &rule_map, &directory_entries, &result_map);
+    auto absent_target_map =
+        BuildMaps::Target::CreateAbsentTargetMap(&result_map, 0);
+    auto target_map = BuildMaps::Target::CreateTargetMap(&source,
+                                                         &targets_file_map,
+                                                         &rule_map,
+                                                         &directory_entries,
+                                                         &absent_target_map,
+                                                         &result_map);
 
     std::vector<AnalysedTargetPtr> result;
     bool error{false};
@@ -462,8 +474,14 @@ TEST_CASE("generator functions in string arguments") {
     auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map);
     auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map);
     BuildMaps::Target::ResultTargetMap result_map{0};
-    auto target_map = BuildMaps::Target::CreateTargetMap(
-        &source, &targets_file_map, &rule_map, &directory_entries, &result_map);
+    auto absent_target_map =
+        BuildMaps::Target::CreateAbsentTargetMap(&result_map, 0);
+    auto target_map = BuildMaps::Target::CreateTargetMap(&source,
+                                                         &targets_file_map,
+                                                         &rule_map,
+                                                         &directory_entries,
+                                                         &absent_target_map,
+                                                         &result_map);
 
     AnalysedTargetPtr result;
     bool error{false};
@@ -530,9 +548,14 @@ TEST_CASE("built-in rules") {
     auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map);
     auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map);
     BuildMaps::Target::ResultTargetMap result_map{0};
-    auto target_map = BuildMaps::Target::CreateTargetMap(
-        &source, &targets_file_map, &rule_map, &directory_entries, &result_map);
-
+    auto absent_target_map =
+        BuildMaps::Target::CreateAbsentTargetMap(&result_map, 0);
+    auto target_map = BuildMaps::Target::CreateTargetMap(&source,
+                                                         &targets_file_map,
+                                                         &rule_map,
+                                                         &directory_entries,
+                                                         &absent_target_map,
+                                                         &result_map);
     AnalysedTargetPtr result;
     bool error{false};
     std::string error_msg;
@@ -709,8 +732,14 @@ TEST_CASE("target reference") {
     auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map);
     auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map);
     BuildMaps::Target::ResultTargetMap result_map{0};
-    auto target_map = BuildMaps::Target::CreateTargetMap(
-        &source, &targets_file_map, &rule_map, &directory_entries, &result_map);
+    auto absent_target_map =
+        BuildMaps::Target::CreateAbsentTargetMap(&result_map, 0);
+    auto target_map = BuildMaps::Target::CreateTargetMap(&source,
+                                                         &targets_file_map,
+                                                         &rule_map,
+                                                         &directory_entries,
+                                                         &absent_target_map,
+                                                         &result_map);
 
     AnalysedTargetPtr result;
     bool error{false};
@@ -820,8 +849,14 @@ TEST_CASE("trees") {
     auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map);
     auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map);
     BuildMaps::Target::ResultTargetMap result_map{0};
-    auto target_map = BuildMaps::Target::CreateTargetMap(
-        &source, &targets_file_map, &rule_map, &directory_entries, &result_map);
+    auto absent_target_map =
+        BuildMaps::Target::CreateAbsentTargetMap(&result_map, 0);
+    auto target_map = BuildMaps::Target::CreateTargetMap(&source,
+                                                         &targets_file_map,
+                                                         &rule_map,
+                                                         &directory_entries,
+                                                         &absent_target_map,
+                                                         &result_map);
 
     AnalysedTargetPtr result;
     bool error{false};
@@ -895,8 +930,14 @@ TEST_CASE("RESULT error reporting") {
     auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map);
     auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map);
     BuildMaps::Target::ResultTargetMap result_map{0};
-    auto target_map = BuildMaps::Target::CreateTargetMap(
-        &source, &targets_file_map, &rule_map, &directory_entries, &result_map);
+    auto absent_target_map =
+        BuildMaps::Target::CreateAbsentTargetMap(&result_map, 0);
+    auto target_map = BuildMaps::Target::CreateTargetMap(&source,
+                                                         &targets_file_map,
+                                                         &rule_map,
+                                                         &directory_entries,
+                                                         &absent_target_map,
+                                                         &result_map);
 
     AnalysedTargetPtr result;
     bool error{false};
@@ -1029,8 +1070,14 @@ TEST_CASE("wrong arguments") {
     auto expr_map = BuildMaps::Base::CreateExpressionMap(&expressions_file_map);
     auto rule_map = BuildMaps::Base::CreateRuleMap(&rule_file_map, &expr_map);
     BuildMaps::Target::ResultTargetMap result_map{0};
-    auto target_map = BuildMaps::Target::CreateTargetMap(
-        &source, &targets_file_map, &rule_map, &directory_entries, &result_map);
+    auto absent_target_map =
+        BuildMaps::Target::CreateAbsentTargetMap(&result_map, 0);
+    auto target_map = BuildMaps::Target::CreateTargetMap(&source,
+                                                         &targets_file_map,
+                                                         &rule_map,
+                                                         &directory_entries,
+                                                         &absent_target_map,
+                                                         &result_map);
 
     AnalysedTargetPtr result;
     bool error{false};