computed roots: enforce export targets of content-fixed repos

... and look up values in cache, if possible.

6 files changed, 254 insertions, 7 deletions

diff --git a/src/buildtool/computed_roots/TARGETS b/src/buildtool/computed_roots/TARGETS
index 0bf40d6c..46f9ea0b 100644
--- a/src/buildtool/computed_roots/TARGETS
+++ b/src/buildtool/computed_roots/TARGETS
@@ -46,7 +46,8 @@
   { "type": ["@", "rules", "CC", "library"]
   , "name": ["evaluate"]
   , "hdrs": ["evaluate.hpp"]
-  , "srcs": ["evaluate.cpp"]
+  , "private-hdrs": ["lookup_cache.hpp"]
+  , "srcs": ["evaluate.cpp", "lookup_cache.cpp"]
   , "stage": ["src", "buildtool", "computed_roots"]
   , "deps":
     [ ["@", "gsl", "", "gsl"]
@@ -57,10 +58,15 @@
     ]
   , "private-deps":
     [ "analyse_and_build"
+    , "artifacts_root"
     , ["@", "fmt", "", "fmt"]
     , ["@", "json", "", "json"]
     , ["src/buildtool/build_engine/base_maps", "entity_name_data"]
+    , ["src/buildtool/build_engine/base_maps", "field_reader"]
+    , ["src/buildtool/build_engine/base_maps", "module_name"]
+    , ["src/buildtool/build_engine/base_maps", "targets_file_map"]
     , ["src/buildtool/build_engine/expression", "expression"]
+    , ["src/buildtool/build_engine/expression", "expression_ptr_interface"]
     , ["src/buildtool/build_engine/target_map", "configured_target"]
     , ["src/buildtool/common", "cli"]
     , ["src/buildtool/common", "common"]
diff --git a/src/buildtool/computed_roots/evaluate.cpp b/src/buildtool/computed_roots/evaluate.cpp
index b20f144e..06beccec 100644
--- a/src/buildtool/computed_roots/evaluate.cpp
+++ b/src/buildtool/computed_roots/evaluate.cpp
@@ -26,6 +26,7 @@
 #include <shared_mutex>
 #include <sstream>
 #include <utility>
+#include <variant>
 #include <vector>
 
 #include "fmt/core.h"
@@ -38,6 +39,7 @@
 #include "src/buildtool/common/cli.hpp"
 #include "src/buildtool/common/statistics.hpp"
 #include "src/buildtool/computed_roots/analyse_and_build.hpp"
+#include "src/buildtool/computed_roots/lookup_cache.hpp"
 #include "src/buildtool/file_system/file_root.hpp"
 #include "src/buildtool/file_system/file_system_manager.hpp"
 #include "src/buildtool/file_system/git_cas.hpp"
@@ -235,6 +237,27 @@ auto ImportToGitCas(std::filesystem::path root_dir,
     return *std::move(res);
 }
 
+auto IsTreePresent(std::string const& tree_id,
+                   gsl::not_null<const StorageConfig*> const& storage_config,
+                   GitRepo::anon_logger_ptr const& logger)
+    -> std::optional<bool> {
+    auto just_git_cas = GitCAS::Open(storage_config->GitRoot());
+    if (not just_git_cas) {
+        (*logger)(fmt::format("Failed to open Git ODB at {}",
+                              storage_config->GitRoot().string()),
+                  true);
+        return std::nullopt;
+    }
+    auto just_git_repo = GitRepo::Open(just_git_cas);
+    if (not just_git_repo) {
+        (*logger)(fmt::format("Failed to open Git repository at {}",
+                              storage_config->GitRoot().string()),
+                  true);
+        return std::nullopt;
+    }
+    return just_git_repo->CheckTreeExists(tree_id, logger);
+}
+
 void ComputeAndFill(
     FileRoot::ComputedRoot const& key,
     gsl::not_null<RepositoryConfig*> const& repository_config,
@@ -277,15 +300,59 @@ void ComputeAndFill(
                                               context->remote_context,
                                               &statistics,
                                               &progress};
+
+    auto cache_lookup =
+        expected<std::optional<std::string>, std::monostate>(std::nullopt);
+    {
+        std::shared_lock computing{*config_lock};
+        cache_lookup = LookupCache(target, repository_config, storage, logger);
+    }
+    if (not cache_lookup) {
+        // prerequisite failure; fatal logger call already handled by
+        // LookupCache
+        return;
+    }
+    if (*cache_lookup) {
+        std::string root = **cache_lookup;
+        auto wrapped_logger = std::make_shared<GitRepo::anon_logger_t>(
+            [&logger, &root](auto const& msg, bool fatal) {
+                (*logger)(fmt::format("While checking presence of tree {} in "
+                                      "local git repo:\n{}",
+                                      root,
+                                      msg),
+                          fatal);
+            });
+        auto tree_present = IsTreePresent(root, storage_config, wrapped_logger);
+        if (not tree_present) {
+            // fatal error; logger already called by IsTreePresent
+            return;
+        }
+        if (*tree_present) {
+            Logger::Log(LogLevel::Performance,
+                        "Root {} taken from cache to be {}",
+                        target.ToString(),
+                        root);
+            auto root_result =
+                FileRoot::FromGit(storage_config->GitRoot(), root);
+            if (not root_result) {
+                (*logger)(fmt::format("Failed to create git root for {}", root),
+                          /*fatal=*/true);
+                return;
+            }
+            {
+                std::unique_lock setting{*config_lock};
+                repository_config->SetComputedRoot(key, *root_result);
+            }
+            (*setter)(std::move(root));
+            return;
+        }
+    }
+
     GraphTraverser traverser{
         root_build_args, &root_exec_context, reporter, &build_logger};
     std::optional<AnalyseAndBuildResult> build_result{};
     {
         std::shared_lock computing{*config_lock};
-        // TODO(aehlig): ensure that target is an export target of a
-        // content-fixed repo
-        // TODO(aehlig): avoid installing and importing to git if the
-        // resulting root is available already
         build_result = AnalyseAndBuild(&analyse_context,
                                        traverser,
                                        target,
diff --git a/src/buildtool/computed_roots/lookup_cache.cpp b/src/buildtool/computed_roots/lookup_cache.cpp
new file mode 100644
index 00000000..9021463c
--- /dev/null
+++ b/src/buildtool/computed_roots/lookup_cache.cpp
@@ -0,0 +1,138 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/buildtool/computed_roots/lookup_cache.hpp"
+
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "fmt/core.h"
+#include "nlohmann/json.hpp"
+#include "src/buildtool/build_engine/base_maps/entity_name_data.hpp"
+#include "src/buildtool/build_engine/base_maps/field_reader.hpp"
+#include "src/buildtool/build_engine/base_maps/module_name.hpp"
+#include "src/buildtool/build_engine/base_maps/targets_file_map.hpp"
+#include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/build_engine/expression/target_result.hpp"
+#include "src/buildtool/computed_roots/artifacts_root.hpp"
+#include "src/buildtool/file_system/file_root.hpp"
+#include "src/buildtool/multithreading/task_system.hpp"
+#include "src/buildtool/storage/target_cache_entry.hpp"
+#include "src/buildtool/storage/target_cache_key.hpp"
+
+auto LookupCache(BuildMaps::Target::ConfiguredTarget const& ctarget,
+                 gsl::not_null<RepositoryConfig*> const& repository_config,
+                 Storage const& storage,
+                 AsyncMapConsumerLoggerPtr const& logger)
+    -> expected<std::optional<std::string>, std::monostate> {
+    auto const* target_root =
+        repository_config->TargetRoot(ctarget.target.ToModule().repository);
+    if ((target_root == nullptr) or target_root->IsAbsent()) {
+        // TODO(aehlig): avoid local installing in case of absent target root of
+        // the base repository
+        return expected<std::optional<std::string>, std::monostate>(
+            std::nullopt);
+    }
+    auto repo_key = repository_config->RepositoryKey(
+        storage, ctarget.target.GetNamedTarget().repository);
+    if (not repo_key) {
+        (*logger)(fmt::format("Repository {} is not content-fixed",
+                              nlohmann::json(
+                                  ctarget.target.GetNamedTarget().repository)),
+                  /*fatal=*/true);
+        return unexpected(std::monostate{});
+    }
+    auto targets_file_map =
+        BuildMaps::Base::CreateTargetsFileMap(repository_config, 1);
+    nlohmann::json targets_file{};
+    bool failed{false};
+    {
+        TaskSystem ts{1};
+        targets_file_map.ConsumeAfterKeysReady(
+            &ts,
+            {ctarget.target.ToModule()},
+            [&targets_file](auto values) { targets_file = *values[0]; },
+            [&logger, &failed](auto const& msg, bool fatal) {
+                (*logger)(
+                    fmt::format("While searching for target description:\n{}",
+                                msg),
+                    fatal);
+                failed = failed or fatal;
+            });
+    }
+    if (failed) {
+        return unexpected(std::monostate{});
+    }
+    auto desc_it = targets_file.find(ctarget.target.GetNamedTarget().name);
+    if (desc_it == targets_file.end()) {
+        (*logger)("Not referring to a defined target", /*fatal=*/true);
+        return unexpected(std::monostate{});
+    }
+    nlohmann::json const& desc = *desc_it;
+    auto rule_it = desc.find("type");
+    if (rule_it == desc.end()) {
+        (*logger)(fmt::format("No type specified in target-description {}",
+                              desc.dump()),
+                  true);
+        return unexpected(std::monostate{});
+    }
+    auto const& rule = *rule_it;
+    if (not(rule.is_string() and rule.get<std::string>() == "export")) {
+        (*logger)(fmt::format("Target not an export target, but of type {}",
+                              rule.dump()),
+                  true);
+        return unexpected(std::monostate{});
+    }
+    auto reader = BuildMaps::Base::FieldReader::CreatePtr(
+        desc, ctarget.target, "export target", logger);
+    auto flexible_vars = reader->ReadStringList("flexible_config");
+    if (not flexible_vars) {
+        return unexpected(std::monostate{});
+    }
+    auto effective_config = ctarget.config.Prune(*flexible_vars);
+    auto cache_key = storage.TargetCache().ComputeKey(
+        *repo_key, ctarget.target.GetNamedTarget(), effective_config);
+    if (not cache_key) {
+        (*logger)("Target-cache key generation failed", true);
+        return unexpected(std::monostate{});
+    }
+    auto target_cache_value = storage.TargetCache().Read(*cache_key);
+    if (not target_cache_value) {
+        return expected<std::optional<std::string>, std::monostate>(
+            std::nullopt);
+    }
+    auto const& [entry, info] = *target_cache_value;
+    auto result = entry.ToResult();
+    if (not result) {
+        (*logger)(fmt::format("Failed to deserialize cache entry {} for key {}",
+                              info.ToString(),
+                              cache_key->Id().ToString()),
+                  true);
+        return unexpected(std::monostate{});
+    }
+
+    auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+        [&logger](auto const& msg, bool fatal) {
+            (*logger)(
+                fmt::format("While computing git tree for artifacts stage:\n{}",
+                            msg),
+                fatal);
+        });
+    auto root = ArtifactsRoot(result->artifact_stage, wrapped_logger);
+    if (not root) {
+        return unexpected(std::monostate{});
+    }
+    return root;
+}
diff --git a/src/buildtool/computed_roots/lookup_cache.hpp b/src/buildtool/computed_roots/lookup_cache.hpp
new file mode 100644
index 00000000..0f8096d1
--- /dev/null
+++ b/src/buildtool/computed_roots/lookup_cache.hpp
@@ -0,0 +1,35 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_LOOKUP_CACHE_HPP
+#define INCLUDED_SRC_BUILDTOOL_COMPUTED_ROOT_LOOKUP_CACHE_HPP
+
+#include <optional>
+#include <string>
+#include <variant>
+
+#include "gsl/gsl"
+#include "src/buildtool/build_engine/target_map/configured_target.hpp"
+#include "src/buildtool/common/repository_config.hpp"
+#include "src/buildtool/multithreading/async_map_consumer.hpp"
+#include "src/buildtool/storage/storage.hpp"
+#include "src/utils/cpp/expected.hpp"
+
+auto LookupCache(BuildMaps::Target::ConfiguredTarget const& ctarget,
+                 gsl::not_null<RepositoryConfig*> const& repository_config,
+                 Storage const& storage,
+                 AsyncMapConsumerLoggerPtr const& logger)
+    -> expected<std::optional<std::string>, std::monostate>;
+
+#endif
diff --git a/test/end-to-end/computed-roots/basic.sh b/test/end-to-end/computed-roots/basic.sh
index 59470ef4..16e363c1 100644
--- a/test/end-to-end/computed-roots/basic.sh
+++ b/test/end-to-end/computed-roots/basic.sh
@@ -140,6 +140,6 @@ echo
     --log-limit 4 -f "${OUT}/log2" \
     --main 'other derived' 2>&1
 echo
-grep '[Ee]xport.*from cache' "${OUT}/log2"
+grep '[Rr]oot.*from cache' "${OUT}/log2"
 
 echo OK
diff --git a/test/end-to-end/computed-roots/mr_computed_setup.sh b/test/end-to-end/computed-roots/mr_computed_setup.sh
index 02b2e0ce..47b17e42 100644
--- a/test/end-to-end/computed-roots/mr_computed_setup.sh
+++ b/test/end-to-end/computed-roots/mr_computed_setup.sh
@@ -73,6 +73,7 @@ cat > repo-config.json <<EOF
         { "repository":
             { "type": "file"
             , "path": "${BASE_ROOT}"
+            , "pragma": {"to_git": true}
             }
         }
     , "derived":
@@ -127,4 +128,4 @@ echo
 
 [ "$(cat "${OUT}/other-derived/out" | wc -l)" -eq 78 ]
 
-echo OK
\ No newline at end of file
+echo OK