just serve: add remote execution endpoint and --fetch-absent option

The serve service will communicate with this endpoint when needed,
as well as ensure artifacts it provides are synced with the remote
execution CAS, if requested by the client.

If just-mr is given the --fetch-absent option, it Always produce
present roots irrespective of the 'absent' pragma. For Git repositories
marked with the 'absent' pragma, first try to fetch any commit
trees provided by the serve endpoint from the execution endpoint
CAS, before reverting to a network fetch.

Co-authored-by: Klaus Aehlig <klaus.aehlig@huawei.com>
Co-authored-by: Alberto Sartori <alberto.sartori@huawei.com>

20 files changed, 442 insertions, 74 deletions

diff --git a/share/man/just-serve-config.5.md b/share/man/just-serve-config.5.md
index ee42692c..c742e207 100644
--- a/share/man/just-serve-config.5.md
+++ b/share/man/just-serve-config.5.md
@@ -19,6 +19,10 @@ The just-serve configuration format
 
 The configuration file is given by a JSON object.
 
+ - The value for the key *`"local build root"`* is a string specifying the path
+   to use as the root for local CAS, cache, and build directories. The path will
+   be created if it does not exist already. 
+
  - The value for the key *`"repositories"`* is a list of strings specifying
    paths to Git repositories for **`just`** **`serve`** to use as additional
    object lookup locations. The paths are to be used in the order given and
@@ -59,6 +63,15 @@ The configuration file is given by a JSON object.
    For subkey *`"server key"`* the value is a string specifying the path to a
    TLS server key.  
 
+ - The value for the key *`"execution endpoint"`* is a JSON object specifying
+   the arguments of a remote execution endpoint to be used by **`just`**
+   **`serve`**.  
+   For subkey *`"address"`* the value is a string specifying the remote
+   execution address in a NAME:PORT format.  
+   For subkey *`"compatible"`* the value is a flag which specifies whether
+   the remote endpoint uses the original remote execution protocol.
+
+
 See also
 ========
 
diff --git a/src/buildtool/main/serve.cpp b/src/buildtool/main/serve.cpp
index da7ed62a..74d78bdc 100644
--- a/src/buildtool/main/serve.cpp
+++ b/src/buildtool/main/serve.cpp
@@ -57,6 +57,18 @@ void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs) {
                     clargs->serve.config.string());
         std::exit(kExitFailure);
     }
+    // read local build root
+    auto local_root = serve_config["local build root"];
+    if (local_root.IsNotNull()) {
+        if (not local_root->IsString()) {
+            Logger::Log(LogLevel::Error,
+                        "just serve: configuration-file provided local root "
+                        "has to be a string, but found {}",
+                        local_root->ToString());
+            std::exit(kExitFailure);
+        }
+        clargs->endpoint.local_root = local_root->String();
+    }
     // read paths of additional lookup repositories
     auto repositories = serve_config["repositories"];
     if (repositories.IsNotNull()) {
@@ -176,6 +188,30 @@ void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs) {
             }
             clargs->auth.tls_ca_cert = cacert->String();
         }
+        // read the TLS client certificate
+        auto client_cert = auth_args->Get("client cert", Expression::none_t{});
+        if (client_cert.IsNotNull()) {
+            if (not client_cert->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided TLS client certificate has "
+                            "to be a string, but found {}",
+                            client_cert->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->cauth.tls_client_cert = client_cert->String();
+        }
+        // read the TLS client key
+        auto client_key = auth_args->Get("client key", Expression::none_t{});
+        if (client_key.IsNotNull()) {
+            if (not client_key->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided TLS client key has to be a "
+                            "string, but found {}",
+                            client_key->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->cauth.tls_client_key = client_key->String();
+        }
     }
     // read remote service arguments
     auto remote_service = serve_config["remote service"];
@@ -271,6 +307,45 @@ void ReadJustServeConfig(gsl::not_null<CommandLineArguments*> const& clargs) {
             clargs->sauth.tls_server_key = server_key->String();
         }
     }
+    // read execution endpoint arguments
+    auto exec_endpoint = serve_config["execution endpoint"];
+    if (exec_endpoint.IsNotNull()) {
+        if (not exec_endpoint->IsMap()) {
+            Logger::Log(LogLevel::Error,
+                        "just-serve: configuration-file provided execution "
+                        "endpoint has to be a map, but found {}",
+                        exec_endpoint->ToString());
+            std::exit(kExitFailure);
+        }
+        // read the compatible flag
+        auto compatible =
+            exec_endpoint->Get("compatible", Expression::none_t{});
+        if (compatible.IsNotNull()) {
+            if (not compatible->IsBool()) {
+                Logger::Log(LogLevel::Error,
+                            "just-serve: expected execution endpoint "
+                            "compatible to be a flag, but found {}",
+                            compatible->ToString());
+                std::exit(kExitFailure);
+            }
+            // compatibility is set immediately if flag is true
+            if (compatible->Bool()) {
+                Compatibility::SetCompatible();
+            }
+        }
+        // read the address
+        auto address = exec_endpoint->Get("address", Expression::none_t{});
+        if (address.IsNotNull()) {
+            if (not address->IsString()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-provided execution endpoint address "
+                            "has to be a string, but found {}",
+                            address->ToString());
+                std::exit(kExitFailure);
+            }
+            clargs->endpoint.remote_execution_address = address->String();
+        }
+    }
 }
 
 #endif  // BOOTSTRAP_BUILD_TOOL
diff --git a/src/buildtool/serve_api/remote/serve_api.cpp b/src/buildtool/serve_api/remote/serve_api.cpp
index 67b5a8d4..7b57d7d0 100644
--- a/src/buildtool/serve_api/remote/serve_api.cpp
+++ b/src/buildtool/serve_api/remote/serve_api.cpp
@@ -24,7 +24,8 @@ ServeApi::ServeApi(ServeApi&& other) noexcept = default;
 ServeApi::~ServeApi() = default;
 
 auto ServeApi::RetrieveTreeFromCommit(std::string const& commit,
-                                      std::string const& subdir)
+                                      std::string const& subdir,
+                                      bool sync_tree)
     -> std::optional<std::string> {
-    return tlc_->ServeCommitTree(commit, subdir);
+    return tlc_->ServeCommitTree(commit, subdir, sync_tree);
 }
diff --git a/src/buildtool/serve_api/remote/serve_api.hpp b/src/buildtool/serve_api/remote/serve_api.hpp
index f1724cf9..d3fc29d1 100644
--- a/src/buildtool/serve_api/remote/serve_api.hpp
+++ b/src/buildtool/serve_api/remote/serve_api.hpp
@@ -37,7 +37,8 @@ class ServeApi final {
     ~ServeApi();
 
     [[nodiscard]] auto RetrieveTreeFromCommit(std::string const& commit,
-                                              std::string const& subdir = ".")
+                                              std::string const& subdir = ".",
+                                              bool sync_tree = false)
         -> std::optional<std::string>;
 
   private:
diff --git a/src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp b/src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp
index e09b3d03..3109d4d9 100644
--- a/src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp
+++ b/src/buildtool/serve_api/remote/serve_target_level_cache_client.cpp
@@ -24,11 +24,13 @@ ServeTargetLevelCacheClient::ServeTargetLevelCacheClient(
 }
 
 auto ServeTargetLevelCacheClient::ServeCommitTree(std::string const& commit_id,
-                                                  std::string const& subdir)
+                                                  std::string const& subdir,
+                                                  bool sync_tree)
     -> std::optional<std::string> {
     justbuild::just_serve::ServeCommitTreeRequest request{};
     request.set_commit(commit_id);
     request.set_subdir(subdir);
+    request.set_sync_tree(sync_tree);
 
     grpc::ClientContext context;
     justbuild::just_serve::ServeCommitTreeResponse response;
@@ -38,8 +40,11 @@ auto ServeTargetLevelCacheClient::ServeCommitTree(std::string const& commit_id,
         LogStatus(&logger_, LogLevel::Debug, status);
         return std::nullopt;
     }
-    if (response.status().code() != grpc::StatusCode::OK) {
-        LogStatus(&logger_, LogLevel::Debug, response.status());
+    if (response.status() !=
+        ::justbuild::just_serve::ServeCommitTreeStatus::OK) {
+        logger_.Emit(LogLevel::Debug,
+                     "ServeCommitTree response returned with {}",
+                     static_cast<int>(response.status()));
         return std::nullopt;
     }
     return response.tree();
diff --git a/src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp b/src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp
index fa89c24b..a9b2a3d2 100644
--- a/src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp
+++ b/src/buildtool/serve_api/remote/serve_target_level_cache_client.hpp
@@ -31,9 +31,11 @@ class ServeTargetLevelCacheClient {
     /// \brief Retrieve the Git tree of a given commit, if known by the remote.
     /// \param[in] commit_id Hash of the Git commit to look up.
     /// \param[in] subdir Relative path of the tree inside commit.
+    /// \param[in] sync_tree Sync tree to the remote-execution end point
     /// \returns The hash of the tree if commit found, nullopt otherwise.
     [[nodiscard]] auto ServeCommitTree(std::string const& commit_id,
-                                       std::string const& subdir)
+                                       std::string const& subdir,
+                                       bool sync_tree)
         -> std::optional<std::string>;
 
   private:
diff --git a/src/buildtool/serve_api/serve_service/TARGETS b/src/buildtool/serve_api/serve_service/TARGETS
index 93155ae7..984806c9 100644
--- a/src/buildtool/serve_api/serve_service/TARGETS
+++ b/src/buildtool/serve_api/serve_service/TARGETS
@@ -12,10 +12,18 @@
   , "hdrs": ["target_level_cache_server.hpp"]
   , "srcs": ["target_level_cache_server.cpp"]
   , "proto": ["just_serve_proto"]
-  , "deps": [["src/buildtool/logging", "logging"]]
+  , "deps":
+    [ ["src/buildtool/logging", "logging"]
+    , ["src/buildtool/execution_api/common", "common"]
+    , ["src/buildtool/execution_api/remote", "config"]
+    ]
   , "stage": ["src", "buildtool", "serve_api", "serve_service"]
   , "private-deps":
     [ ["@", "fmt", "", "fmt"]
+    , ["src/buildtool/common", "common"]
+    , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
+    , ["src/buildtool/execution_api/local", "local"]
+    , ["src/buildtool/execution_api/remote", "bazel"]
     , ["src/buildtool/file_system", "git_repo"]
     , ["src/buildtool/serve_api/remote", "config"]
     , ["src/buildtool/storage", "config"]
diff --git a/src/buildtool/serve_api/serve_service/just_serve.proto b/src/buildtool/serve_api/serve_service/just_serve.proto
index 3eca2954..c20f77aa 100644
--- a/src/buildtool/serve_api/serve_service/just_serve.proto
+++ b/src/buildtool/serve_api/serve_service/just_serve.proto
@@ -16,8 +16,6 @@ syntax = "proto3";
 
 package justbuild.just_serve;
 
-import "google/rpc/status.proto";
-
 // A request message for
 // [TargetLevelCache.ServeCommitTree][justbuild.just_serve.TargetLevelCache.ServeCommitTree].
 message ServeCommitTreeRequest {
@@ -26,21 +24,36 @@ message ServeCommitTreeRequest {
 
   // Relative path of requested tree with respect to the commit root.
   string subdir = 2;
+
+  // If set to true and the tree is found, it will be uploaded to the
+  // remote-execution end point.
+  bool sync_tree = 3;
 }
 
+enum ServeCommitTreeStatus{
+  // All good
+  OK = 0;
+
+  // Failed to upload tree remotely
+  SYNC_ERROR = 1;
+
+  // Tree not found
+  NOT_FOUND = 2;
+
+  // Internaly, something is very broken
+  INTERNAL_ERROR = 3;
+}
+
+
 // A response message for
 // [TargetLevelCache.ServeCommitTree][justbuild.just_serve.TargetLevelCache.ServeCommitTree].
 message ServeCommitTreeResponse {
   // The requested Git tree hash.
   string tree = 1;
 
-  // If the status has a code other than `OK`, it indicates that the tree hash
-  // could not be computed. In this case, the `tree` field is optional.
-  //
-  // If the status code is `NOT_FOUND`, it indicates that either the commit was
-  // not found, or the commit root tree does not contain the given relative
-  // path.
-  google.rpc.Status status = 2;
+  // If the status has a code `OK` or `SYNC_ERROR`, the tree is correct.
+  // For any other value, the `tree` field is not set.
+  ServeCommitTreeStatus status = 2;
 }
 
 // Services for improved interaction with the target-level cache.
diff --git a/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp b/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp
index 688a5c32..e0451c63 100644
--- a/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp
+++ b/src/buildtool/serve_api/serve_service/target_level_cache_server.cpp
@@ -15,10 +15,28 @@
 #include "src/buildtool/serve_api/serve_service/target_level_cache_server.hpp"
 
 #include "fmt/core.h"
+#include "src/buildtool/common/artifact.hpp"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_api/local/local_api.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
 #include "src/buildtool/file_system/git_repo.hpp"
 #include "src/buildtool/serve_api/remote/config.hpp"
 #include "src/buildtool/storage/config.hpp"
 
+auto TargetLevelCacheService::CreateExecutionApi(
+    std::optional<RemoteExecutionConfig::ServerAddress> const& address)
+    -> gsl::not_null<IExecutionApi::Ptr> {
+    if (address) {
+        ExecutionConfiguration config;
+        config.skip_cache_lookup = false;
+
+        return std::make_unique<BazelApi>(
+            "remote-execution", address->host, address->port, config);
+    }
+    return std::make_unique<LocalApi>();
+}
+
 auto TargetLevelCacheService::GetTreeFromCommit(
     std::filesystem::path const& repo_path,
     std::string const& commit,
@@ -53,26 +71,74 @@ auto TargetLevelCacheService::ServeCommitTree(
     const ::justbuild::just_serve::ServeCommitTreeRequest* request,
     ::justbuild::just_serve::ServeCommitTreeResponse* response)
     -> ::grpc::Status {
+    using ServeCommitTreeStatus =
+        ::justbuild::just_serve::ServeCommitTreeStatus;
     auto const& commit{request->commit()};
     auto const& subdir{request->subdir()};
     // try in local build root Git cache
     if (auto tree_id = GetTreeFromCommit(
             StorageConfig::GitRoot(), commit, subdir, logger_)) {
+        if (request->sync_tree()) {
+            // sync tree with remote CAS
+            auto digest = ArtifactDigest{*tree_id, 0, /*is_tree=*/true};
+            auto repo = RepositoryConfig{};
+            if (not repo.SetGitCAS(StorageConfig::GitRoot())) {
+                auto str = fmt::format("Failed to SetGitCAS at {}",
+                                       StorageConfig::GitRoot().string());
+                logger_->Emit(LogLevel::Debug, str);
+                response->set_status(ServeCommitTreeStatus::INTERNAL_ERROR);
+                return ::grpc::Status::OK;
+            }
+            auto git_api = GitApi{&repo};
+            if (not git_api.RetrieveToCas(
+                    {Artifact::ObjectInfo{.digest = digest,
+                                          .type = ObjectType::Tree}},
+                    &(*remote_api_))) {
+                auto str = fmt::format("Failed to sync tree {}", *tree_id);
+                logger_->Emit(LogLevel::Debug, str);
+                *(response->mutable_tree()) = std::move(*tree_id);
+                response->set_status(ServeCommitTreeStatus::SYNC_ERROR);
+                return ::grpc::Status::OK;
+            }
+        }
+        // set response
         *(response->mutable_tree()) = std::move(*tree_id);
-        response->mutable_status()->CopyFrom(google::rpc::Status{});
+        response->set_status(ServeCommitTreeStatus::OK);
         return ::grpc::Status::OK;
     }
     // try given extra repositories, in order
     for (auto const& path : RemoteServeConfig::KnownRepositories()) {
         if (auto tree_id = GetTreeFromCommit(path, commit, subdir, logger_)) {
+
+            if (request->sync_tree()) {
+                // sync tree with remote CAS
+                auto digest = ArtifactDigest{*tree_id, 0, /*is_tree=*/true};
+                auto repo = RepositoryConfig{};
+                if (not repo.SetGitCAS(path)) {
+                    auto str =
+                        fmt::format("Failed to SetGitCAS at {}", path.string());
+                    logger_->Emit(LogLevel::Debug, str);
+                    response->set_status(ServeCommitTreeStatus::INTERNAL_ERROR);
+                    return ::grpc::Status::OK;
+                }
+                auto git_api = GitApi{&repo};
+                if (not git_api.RetrieveToCas(
+                        {Artifact::ObjectInfo{.digest = digest,
+                                              .type = ObjectType::Tree}},
+                        &(*remote_api_))) {
+                    auto str = fmt::format("Failed to sync tree {}", *tree_id);
+                    logger_->Emit(LogLevel::Debug, str);
+                    *(response->mutable_tree()) = std::move(*tree_id);
+                    response->set_status(ServeCommitTreeStatus::SYNC_ERROR);
+                }
+            }
+            // set response
             *(response->mutable_tree()) = std::move(*tree_id);
-            response->mutable_status()->CopyFrom(google::rpc::Status{});
+            response->set_status(ServeCommitTreeStatus::OK);
             return ::grpc::Status::OK;
         }
     }
     // commit not found
-    google::rpc::Status status;
-    status.set_code(grpc::StatusCode::NOT_FOUND);
-    response->mutable_status()->CopyFrom(status);
+    response->set_status(ServeCommitTreeStatus::NOT_FOUND);
     return ::grpc::Status::OK;
 }
diff --git a/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp b/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp
index 80d09647..826d4031 100644
--- a/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp
+++ b/src/buildtool/serve_api/serve_service/target_level_cache_server.hpp
@@ -21,7 +21,10 @@
 #include <string>
 #include <vector>
 
+#include "gsl/gsl"
 #include "justbuild/just_serve/just_serve.grpc.pb.h"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/remote/config.hpp"
 #include "src/buildtool/logging/logger.hpp"
 
 class TargetLevelCacheService final
@@ -46,6 +49,17 @@ class TargetLevelCacheService final
   private:
     std::shared_ptr<Logger> logger_{std::make_shared<Logger>("serve-service")};
 
+    // remote execution endpoint
+    gsl::not_null<IExecutionApi::Ptr> const remote_api_{
+        CreateExecutionApi(RemoteExecutionConfig::RemoteAddress())};
+    // local api
+    gsl::not_null<IExecutionApi::Ptr> const local_api_{
+        CreateExecutionApi(std::nullopt)};
+
+    [[nodiscard]] static auto CreateExecutionApi(
+        std::optional<RemoteExecutionConfig::ServerAddress> const& address)
+        -> gsl::not_null<IExecutionApi::Ptr>;
+
     [[nodiscard]] static auto GetTreeFromCommit(
         std::filesystem::path const& repo_path,
         std::string const& commit,
diff --git a/src/other_tools/just_mr/cli.hpp b/src/other_tools/just_mr/cli.hpp
index 22c401ba..328ab666 100644
--- a/src/other_tools/just_mr/cli.hpp
+++ b/src/other_tools/just_mr/cli.hpp
@@ -48,6 +48,7 @@ struct MultiRepoCommonArguments {
     std::optional<std::string> remote_execution_address;
     std::optional<bool> compatible{std::nullopt};
     std::optional<std::string> remote_serve_address;
+    bool fetch_absent{false};
 };
 
 struct MultiRepoLogArguments {
@@ -192,6 +193,11 @@ static inline void SetupMultiRepoCommonArguments(
                     clargs->remote_serve_address,
                     "Address of a remote 'just serve' service.")
         ->type_name("NAME:PORT");
+    app->add_flag("--fetch-absent",
+                  clargs->fetch_absent,
+                  "Do not produce absent roots. For Git repositories, try to "
+                  "fetch served commit trees from the remote execution "
+                  "endpoint before reverting to the network.");
 }
 
 static inline auto SetupMultiRepoLogArguments(
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index ed09bb61..54295c6b 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -116,10 +116,14 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
 
     auto commit_git_map =
         CreateCommitGitMap(&critical_git_op_map,
+                           &import_to_git_map,
                            common_args.just_mr_paths,
                            common_args.git_path->string(),
                            *common_args.local_launcher,
                            serve_api ? &(*serve_api) : nullptr,
+                           local_api ? &(*local_api) : nullptr,
+                           remote_api ? &(*remote_api) : nullptr,
+                           common_args.fetch_absent,
                            common_args.jobs);
     auto content_git_map = CreateContentGitMap(&content_cas_map,
                                                &import_to_git_map,
@@ -147,6 +151,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
                                                     &fpath_git_map,
                                                     &distdir_git_map,
                                                     &tree_id_git_map,
+                                                    common_args.fetch_absent,
                                                     common_args.jobs);
 
     // set up progress observer
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index 5aefe3b1..f51e48ec 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -150,6 +150,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
                      std::string const& repo_name,
                      std::string const& repo_type,
                      gsl::not_null<ContentGitMap*> const& content_git_map,
+                     bool fetch_absent,
                      gsl::not_null<TaskSystem*> const& ts,
                      ReposToSetupMap::SetterPtr const& setter,
                      ReposToSetupMap::LoggerPtr const& logger) {
@@ -224,7 +225,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
         .repo_type = repo_type,
         .subdir = subdir.empty() ? "." : subdir.string(),
         .pragma_special = pragma_special_value,
-        .absent = pragma_absent_value};
+        .absent = not fetch_absent and pragma_absent_value};
     // get the WS root as git tree
     content_git_map->ConsumeAfterKeysReady(
         ts,
@@ -258,6 +259,7 @@ void FileCheckout(ExpressionPtr const& repo_desc,
                   ExpressionPtr&& repos,
                   std::string const& repo_name,
                   gsl::not_null<FilePathGitMap*> const& fpath_git_map,
+                  bool fetch_absent,
                   gsl::not_null<TaskSystem*> const& ts,
                   ReposToSetupMap::SetterPtr const& setter,
                   ReposToSetupMap::LoggerPtr const& logger) {
@@ -305,9 +307,10 @@ void FileCheckout(ExpressionPtr const& repo_desc,
                                    pragma_absent->get()->IsBool() and
                                    pragma_absent->get()->Bool();
         // get the WS root as git tree
-        FpathInfo fpath_info = {.fpath = fpath,
-                                .pragma_special = pragma_special_value,
-                                .absent = pragma_absent_value};
+        FpathInfo fpath_info = {
+            .fpath = fpath,
+            .pragma_special = pragma_special_value,
+            .absent = not fetch_absent and pragma_absent_value};
         fpath_git_map->ConsumeAfterKeysReady(
             ts,
             {std::move(fpath_info)},
@@ -349,6 +352,7 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
                      ExpressionPtr&& repos,
                      std::string const& repo_name,
                      gsl::not_null<DistdirGitMap*> const& distdir_git_map,
+                     bool fetch_absent,
                      gsl::not_null<TaskSystem*> const& ts,
                      ReposToSetupMap::SetterPtr const& setter,
                      ReposToSetupMap::LoggerPtr const& logger) {
@@ -509,11 +513,12 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
         HashFunction::ComputeBlobHash(nlohmann::json(*distdir_content).dump())
             .HexString();
     // get the WS root as git tree
-    DistdirInfo distdir_info = {.content_id = distdir_content_id,
-                                .content_list = distdir_content,
-                                .repos_to_fetch = dist_repos_to_fetch,
-                                .origin = repo_name,
-                                .absent = pragma_absent_value};
+    DistdirInfo distdir_info = {
+        .content_id = distdir_content_id,
+        .content_list = distdir_content,
+        .repos_to_fetch = dist_repos_to_fetch,
+        .origin = repo_name,
+        .absent = not fetch_absent and pragma_absent_value};
     distdir_git_map->ConsumeAfterKeysReady(
         ts,
         {std::move(distdir_info)},
@@ -545,6 +550,7 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc,
                      ExpressionPtr&& repos,
                      std::string const& repo_name,
                      gsl::not_null<TreeIdGitMap*> const& tree_id_git_map,
+                     bool fetch_absent,
                      gsl::not_null<TaskSystem*> const& ts,
                      ReposToSetupMap::SetterPtr const& setter,
                      ReposToSetupMap::LoggerPtr const& logger) {
@@ -628,7 +634,7 @@ void GitTreeCheckout(ExpressionPtr const& repo_desc,
         .env_vars = std::move(env),
         .command = std::move(cmd),
         .ignore_special = pragma_special_value == PragmaSpecial::Ignore,
-        .absent = pragma_absent_value};
+        .absent = not fetch_absent and pragma_absent_value};
     // get the WS root as git tree
     tree_id_git_map->ConsumeAfterKeysReady(
         ts,
@@ -665,6 +671,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
                            gsl::not_null<FilePathGitMap*> const& fpath_git_map,
                            gsl::not_null<DistdirGitMap*> const& distdir_git_map,
                            gsl::not_null<TreeIdGitMap*> const& tree_id_git_map,
+                           bool fetch_absent,
                            std::size_t jobs) -> ReposToSetupMap {
     auto setup_repo = [config,
                        main,
@@ -673,11 +680,12 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
                        content_git_map,
                        fpath_git_map,
                        distdir_git_map,
-                       tree_id_git_map](auto ts,
-                                        auto setter,
-                                        auto logger,
-                                        auto /* unused */,
-                                        auto const& key) {
+                       tree_id_git_map,
+                       fetch_absent](auto ts,
+                                     auto setter,
+                                     auto logger,
+                                     auto /* unused */,
+                                     auto const& key) {
         auto repos = (*config)["repositories"];
         if (main && (key == *main) && interactive) {
             // no repository checkout required
@@ -765,6 +773,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
                                     key,
                                     repo_type_str,
                                     content_git_map,
+                                    fetch_absent,
                                     ts,
                                     setter,
                                     wrapped_logger);
@@ -775,6 +784,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
                                  std::move(repos),
                                  key,
                                  fpath_git_map,
+                                 fetch_absent,
                                  ts,
                                  setter,
                                  wrapped_logger);
@@ -785,6 +795,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
                                     std::move(repos),
                                     key,
                                     distdir_git_map,
+                                    fetch_absent,
                                     ts,
                                     setter,
                                     wrapped_logger);
@@ -795,6 +806,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
                                     std::move(repos),
                                     key,
                                     tree_id_git_map,
+                                    fetch_absent,
                                     ts,
                                     setter,
                                     wrapped_logger);
diff --git a/src/other_tools/repo_map/repos_to_setup_map.hpp b/src/other_tools/repo_map/repos_to_setup_map.hpp
index 3306800d..1fbf9687 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.hpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.hpp
@@ -34,6 +34,7 @@ auto CreateReposToSetupMap(std::shared_ptr<Configuration> const& config,
                            gsl::not_null<FilePathGitMap*> const& fpath_git_map,
                            gsl::not_null<DistdirGitMap*> const& distdir_git_map,
                            gsl::not_null<TreeIdGitMap*> const& tree_id_git_map,
+                           bool fetch_absent,
                            std::size_t jobs) -> ReposToSetupMap;
 
 #endif  // INCLUDED_SRC_OTHER_TOOLS_REPO_MAP_REPOS_TO_SETUP_MAP_HPP
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index c7089bfc..b2cd1dfb 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -30,8 +30,10 @@
   , "srcs": ["commit_git_map.cpp"]
   , "deps":
     [ ["src/buildtool/serve_api/remote", "serve_api"]
+    , ["src/buildtool/execution_api/common", "common"]
     , ["src/other_tools/just_mr", "utils"]
     , ["src/other_tools/ops_maps", "critical_git_op_map"]
+    , ["src/other_tools/ops_maps", "import_to_git_map"]
     , ["src/utils/cpp", "hash_combine"]
     , ["@", "json", "", "json"]
     ]
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index 64d686c1..883fe697 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -37,9 +37,13 @@ void EnsureCommit(GitRepoInfo const& repo_info,
                   std::filesystem::path const& repo_root,
                   GitCASPtr const& git_cas,
                   gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+                  gsl::not_null<ImportToGitMap*> const& import_to_git_map,
                   std::string const& git_bin,
                   std::vector<std::string> const& launcher,
                   ServeApi* serve_api,
+                  IExecutionApi* local_api,
+                  IExecutionApi* remote_api,
+                  bool fetch_absent,
                   gsl::not_null<TaskSystem*> const& ts,
                   CommitGitMap::SetterPtr const& ws_setter,
                   CommitGitMap::LoggerPtr const& logger) {
@@ -69,31 +73,146 @@ void EnsureCommit(GitRepoInfo const& repo_info,
         if (repo_info.absent) {
             if (serve_api != nullptr) {
                 if (auto tree_id = serve_api->RetrieveTreeFromCommit(
-                        repo_info.hash, repo_info.subdir)) {
-                    // set the workspace root as absent
-                    JustMRProgress::Instance().TaskTracker().Stop(
-                        repo_info.origin);
-                    (*ws_setter)(std::pair(
-                        nlohmann::json::array(
-                            {repo_info.ignore_special
-                                 ? FileRoot::kGitTreeIgnoreSpecialMarker
-                                 : FileRoot::kGitTreeMarker,
-                             *tree_id}),
-                        false));
-                    return;
+                        repo_info.hash, repo_info.subdir, fetch_absent)) {
+                    if (not fetch_absent) {
+                        // set the workspace root as absent
+                        JustMRProgress::Instance().TaskTracker().Stop(
+                            repo_info.origin);
+                        (*ws_setter)(std::pair(
+                            nlohmann::json::array(
+                                {repo_info.ignore_special
+                                     ? FileRoot::kGitTreeIgnoreSpecialMarker
+                                     : FileRoot::kGitTreeMarker,
+                                 *tree_id}),
+                            false));
+                        return;
+                    }
+                    // verify if we know the tree already locally
+                    auto wrapped_logger =
+                        std::make_shared<AsyncMapConsumerLogger>(
+                            [logger, tree = *tree_id](auto const& msg,
+                                                      bool fatal) {
+                                (*logger)(
+                                    fmt::format("While verifying presence of "
+                                                "tree {}:\n{}",
+                                                tree,
+                                                msg),
+                                    fatal);
+                            });
+                    auto tree_present =
+                        git_repo->CheckTreeExists(*tree_id, wrapped_logger);
+                    if (not tree_present) {
+                        return;
+                    }
+                    if (*tree_present) {
+                        JustMRProgress::Instance().TaskTracker().Stop(
+                            repo_info.origin);
+                        (*ws_setter)(std::pair(
+                            nlohmann::json::array(
+                                {repo_info.ignore_special
+                                     ? FileRoot::kGitTreeIgnoreSpecialMarker
+                                     : FileRoot::kGitTreeMarker,
+                                 *tree_id,
+                                 StorageConfig::GitRoot().string()}),
+                            false));
+                        return;
+                    }
+                    // try to get the tree from remote execution endpoint
+                    auto digest = ArtifactDigest{*tree_id, 0, /*is_tree=*/true};
+                    if (remote_api != nullptr and local_api != nullptr and
+                        remote_api->RetrieveToCas(
+                            {Artifact::ObjectInfo{.digest = digest,
+                                                  .type = ObjectType::Tree}},
+                            local_api)) {
+                        JustMRProgress::Instance().TaskTracker().Stop(
+                            repo_info.origin);
+                        // Move tree from CAS to local git storage
+                        auto tmp_dir = JustMR::Utils::CreateTypedTmpDir(
+                            "fetch-absent-root");
+                        if (not tmp_dir) {
+                            (*logger)(
+                                fmt::format(
+                                    "Failed to create tmp directory after "
+                                    "fetching tree {} for absent commit {}",
+                                    repo_info.hash,
+                                    *tree_id),
+                                true);
+                            return;
+                        }
+                        if (not local_api->RetrieveToPaths(
+                                {Artifact::ObjectInfo{
+                                    .digest = digest,
+                                    .type = ObjectType::Tree}},
+                                {tmp_dir->GetPath()})) {
+                            (*logger)(fmt::format("Failed to copy fetchted "
+                                                  "tree {} to {}",
+                                                  *tree_id,
+                                                  tmp_dir->GetPath().string()),
+                                      true);
+                            return;
+                        }
+                        CommitInfo c_info{tmp_dir->GetPath(), "tree", *tree_id};
+                        import_to_git_map->ConsumeAfterKeysReady(
+                            ts,
+                            {std::move(c_info)},
+                            [tmp_dir,  // keep tmp_dir alive
+                             repo_info,
+                             tree_id,
+                             logger,
+                             ws_setter](auto const& values) {
+                                if (not values[0]->second) {
+                                    (*logger)("Importing to git failed",
+                                              /*fatal=*/true);
+                                    return;
+                                }
+                                // set the workspace root as present
+                                (*ws_setter)(std::pair(
+                                    nlohmann::json::array(
+                                        {repo_info.ignore_special
+                                             ? FileRoot::
+                                                   kGitTreeIgnoreSpecialMarker
+                                             : FileRoot::kGitTreeMarker,
+                                         *tree_id,
+                                         StorageConfig::GitRoot().string()}),
+                                    false));
+                            },
+                            [logger, tmp_dir, tree_id](auto const& msg,
+                                                       bool fatal) {
+                                (*logger)(
+                                    fmt::format("While moving tree {} from {} "
+                                                "to local git:\n{}",
+                                                *tree_id,
+                                                tmp_dir->GetPath().string(),
+                                                msg),
+                                    fatal);
+                            });
+
+                        return;
+                    }
+                    // just serve should have made the tree available in the
+                    // remote CAS, so log this attempt and revert to network
+                    (*logger)(fmt::format("Tree {} marked as served, but not "
+                                          "found on remote",
+                                          *tree_id),
+                              /*fatal=*/false);
+                }
+                else {
+                    // give warning
+                    (*logger)(
+                        fmt::format("Tree at subdir {} for commit {} could "
+                                    "not be served",
+                                    repo_info.subdir,
+                                    repo_info.hash),
+                        /*fatal=*/false);
                 }
-                // give warning
-                (*logger)(fmt::format("Tree at subdir {} for commit {} could "
-                                      "not be served",
-                                      repo_info.subdir,
-                                      repo_info.hash),
-                          /*fatal=*/false);
             }
             else {
-                // give warning
-                (*logger)(
-                    "Absent root requested, but no serve endpoint provided",
-                    /*fatal=*/false);
+                if (not fetch_absent) {
+                    // give warning
+                    (*logger)(
+                        "Absent root requested, but no serve endpoint provided",
+                        /*fatal=*/false);
+                }
             }
         }
         // default to fetching it from network
@@ -151,7 +270,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
         critical_git_op_map->ConsumeAfterKeysReady(
             ts,
             {std::move(op_key)},
-            [git_cas, repo_info, repo_root, ws_setter, logger](
+            [git_cas, repo_info, repo_root, fetch_absent, ws_setter, logger](
                 auto const& values) {
                 GitOpValue op_result = *values[0];
                 // check flag
@@ -190,7 +309,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
                          ? FileRoot::kGitTreeIgnoreSpecialMarker
                          : FileRoot::kGitTreeMarker,
                      *subtree});
-                if (not repo_info.absent) {
+                if (fetch_absent or not repo_info.absent) {
                     root.emplace_back(repo_root);
                 }
                 (*ws_setter)(std::pair(std::move(root), false));
@@ -222,7 +341,7 @@ void EnsureCommit(GitRepoInfo const& repo_info,
             {repo_info.ignore_special ? FileRoot::kGitTreeIgnoreSpecialMarker
                                       : FileRoot::kGitTreeMarker,
              *subtree});
-        if (not repo_info.absent) {
+        if (fetch_absent or not repo_info.absent) {
             root.emplace_back(repo_root);
         }
         (*ws_setter)(std::pair(std::move(root), true));
@@ -234,20 +353,28 @@ void EnsureCommit(GitRepoInfo const& repo_info,
 /// \brief Create a CommitGitMap object
 auto CreateCommitGitMap(
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+    gsl::not_null<ImportToGitMap*> const& import_to_git_map,
     JustMR::PathsPtr const& just_mr_paths,
     std::string const& git_bin,
     std::vector<std::string> const& launcher,
     ServeApi* serve_api,
+    IExecutionApi* local_api,
+    IExecutionApi* remote_api,
+    bool fetch_absent,
     std::size_t jobs) -> CommitGitMap {
     auto commit_to_git = [critical_git_op_map,
+                          import_to_git_map,
                           just_mr_paths,
                           git_bin,
                           launcher,
-                          serve_api](auto ts,
-                                     auto setter,
-                                     auto logger,
-                                     auto /* unused */,
-                                     auto const& key) {
+                          serve_api,
+                          local_api,
+                          remote_api,
+                          fetch_absent](auto ts,
+                                        auto setter,
+                                        auto logger,
+                                        auto /* unused */,
+                                        auto const& key) {
         // get root for repo (making sure that if repo is a path, it is
         // absolute)
         std::string fetch_repo = key.repo_url;
@@ -275,9 +402,13 @@ auto CreateCommitGitMap(
             [key,
              repo_root,
              critical_git_op_map,
+             import_to_git_map,
              git_bin,
              launcher,
              serve_api,
+             local_api,
+             remote_api,
+             fetch_absent,
              ts,
              setter,
              logger](auto const& values) {
@@ -302,9 +433,13 @@ auto CreateCommitGitMap(
                              repo_root,
                              op_result.git_cas,
                              critical_git_op_map,
+                             import_to_git_map,
                              git_bin,
                              launcher,
                              serve_api,
+                             local_api,
+                             remote_api,
+                             fetch_absent,
                              ts,
                              setter,
                              wrapped_logger);
diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp
index 00a607cb..4f59a56f 100644
--- a/src/other_tools/root_maps/commit_git_map.hpp
+++ b/src/other_tools/root_maps/commit_git_map.hpp
@@ -19,9 +19,11 @@
 #include <utility>
 
 #include "nlohmann/json.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
 #include "src/buildtool/serve_api/remote/serve_api.hpp"
 #include "src/other_tools/just_mr/utils.hpp"
 #include "src/other_tools/ops_maps/critical_git_op_map.hpp"
+#include "src/other_tools/ops_maps/import_to_git_map.hpp"
 #include "src/utils/cpp/hash_combine.hpp"
 
 struct GitRepoInfo {
@@ -66,10 +68,14 @@ using CommitGitMap =
 
 [[nodiscard]] auto CreateCommitGitMap(
     gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
+    gsl::not_null<ImportToGitMap*> const& import_to_git_map,
     JustMR::PathsPtr const& just_mr_paths,
     std::string const& git_bin,
     std::vector<std::string> const& launcher,
     ServeApi* serve_api,
+    IExecutionApi* local_api,
+    IExecutionApi* remote_api,
+    bool fetch_absent,
     std::size_t jobs) -> CommitGitMap;
 
 #endif  // INCLUDED_SRC_OTHER_TOOLS_ROOT_MAPS_COMMIT_GIT_MAP_HPP
diff --git a/test/buildtool/serve_api/target_level_cache_client.test.cpp b/test/buildtool/serve_api/target_level_cache_client.test.cpp
index 00c3e493..972d57eb 100644
--- a/test/buildtool/serve_api/target_level_cache_client.test.cpp
+++ b/test/buildtool/serve_api/target_level_cache_client.test.cpp
@@ -35,34 +35,34 @@ TEST_CASE("Serve service client: tree-of-commit request", "[serve_api]") {
     ServeTargetLevelCacheClient tlc_client(info->host, info->port);
 
     SECTION("Commit in bare checkout") {
-        auto root_id = tlc_client.ServeCommitTree(kRootCommit, ".");
+        auto root_id = tlc_client.ServeCommitTree(kRootCommit, ".", false);
         REQUIRE(root_id);
         CHECK(root_id.value() == kRootId);
 
-        auto baz_id = tlc_client.ServeCommitTree(kRootCommit, "baz");
+        auto baz_id = tlc_client.ServeCommitTree(kRootCommit, "baz", false);
         REQUIRE(baz_id);
         CHECK(baz_id.value() == kBazId);
     }
 
     SECTION("Commit in non-bare checkout") {
-        auto root_id = tlc_client.ServeCommitTree(kRootSymCommit, ".");
+        auto root_id = tlc_client.ServeCommitTree(kRootSymCommit, ".", false);
         REQUIRE(root_id);
         CHECK(root_id.value() == kRootSymId);
 
-        auto baz_id = tlc_client.ServeCommitTree(kRootSymCommit, "baz");
+        auto baz_id = tlc_client.ServeCommitTree(kRootSymCommit, "baz", false);
         REQUIRE(baz_id);
         CHECK(baz_id.value() == kBazSymId);
     }
 
     SECTION("Subdir not found") {
         auto root_id =
-            tlc_client.ServeCommitTree(kRootCommit, "does_not_exist");
+            tlc_client.ServeCommitTree(kRootCommit, "does_not_exist", false);
         CHECK_FALSE(root_id);
     }
 
     SECTION("Commit not known") {
         auto root_id = tlc_client.ServeCommitTree(
-            "0123456789abcdef0123456789abcdef01234567", ".");
+            "0123456789abcdef0123456789abcdef01234567", ".", false);
         CHECK_FALSE(root_id);
     }
 }
diff --git a/test/end-to-end/just-mr/absent-roots.sh b/test/end-to-end/just-mr/absent-roots.sh
index ff3b738f..7f3154a6 100644
--- a/test/end-to-end/just-mr/absent-roots.sh
+++ b/test/end-to-end/just-mr/absent-roots.sh
@@ -67,6 +67,7 @@ cat repos.json
 cat > .just-servec <<EOF
 { "repositories": ["${REPO_ROOT}"]
 , "remote service": {"info file": "${INFOFILE}", "pid file": "${PIDFILE}"}
+, "local build root": "${LBR}"
 }
 EOF
 echo "Serve service configuration:"
diff --git a/test/utils/serve_service/test_runner.py b/test/utils/serve_service/test_runner.py
index 08681646..b043b363 100755
--- a/test/utils/serve_service/test_runner.py
+++ b/test/utils/serve_service/test_runner.py
@@ -62,6 +62,7 @@ if os.path.exists(TEST_SERVE_REPO_2):
 SERVE_REPOSITORIES = ";".join([TEST_SERVE_REPO_1, TEST_SERVE_REPO_2])
 
 REMOTE_SERVE_INFO = os.path.join(REMOTE_DIR, "info_serve.json")
+SERVE_LBR = os.path.join(REMOTE_DIR, "serve-build-root")
 
 if os.path.exists(REMOTE_SERVE_INFO):
     print(f"Warning: removing unexpected info file {REMOTE_SERVE_INFO}")
@@ -79,7 +80,8 @@ with open(SERVE_CONFIG_FILE, "w") as f:
             },
             "remote service": {
                 "info file": REMOTE_SERVE_INFO
-            }
+            },
+            "local build root": SERVE_LBR
         }))
 
 serve_cmd = ["./bin/just", "serve", SERVE_CONFIG_FILE]