StorageConfig: Store also the execution backend description id

...such that it will be available to the TargetCache for sharing.

Also, GC does not require remote execution information, so the
logic for this subcommand is moved earlier in main.

5 files changed, 109 insertions, 25 deletions

diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS
index 5e8e457b..c4c7f122 100644
--- a/src/buildtool/main/TARGETS
+++ b/src/buildtool/main/TARGETS
@@ -9,6 +9,7 @@
     , ["@", "json", "", "json"]
     , ["src/buildtool/common", "common"]
     , ["src/buildtool/common", "config"]
+    , ["src/buildtool/common/remote", "remote_common"]
     , ["src/buildtool/storage", "storage"]
     , ["src/buildtool/compatibility", "compatibility"]
     , ["src/buildtool/graph_traverser", "graph_traverser"]
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index 36f03345..716545a5 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -17,6 +17,7 @@
 #include <filesystem>
 #include <fstream>
 #include <iostream>
+#include <map>
 #include <mutex>
 #include <optional>
 #include <string>
@@ -24,6 +25,7 @@
 #include <unordered_set>
 #include <utility>
 #include <variant>
+#include <vector>
 
 #include "gsl/gsl"
 #include "nlohmann/json.hpp"
@@ -32,6 +34,7 @@
 #include "src/buildtool/build_engine/expression/expression.hpp"
 #include "src/buildtool/build_engine/target_map/target_map.hpp"
 #include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/common/remote/remote_common.hpp"
 #include "src/buildtool/common/repository_config.hpp"
 #include "src/buildtool/common/statistics.hpp"
 #include "src/buildtool/compatibility/compatibility.hpp"
@@ -102,14 +105,23 @@ void SetupLogging(LogArguments const& clargs) {
     }
 }
 
-[[nodiscard]] auto CreateStorageConfig(EndpointArguments const& eargs) noexcept
+[[nodiscard]] auto CreateStorageConfig(
+    EndpointArguments const& eargs,
+    std::optional<ServerAddress> const& remote_address = std::nullopt,
+    std::map<std::string, std::string> const& remote_platform_properties = {},
+    std::vector<std::pair<std::map<std::string, std::string>,
+                          ServerAddress>> const& remote_dispatch = {}) noexcept
     -> std::optional<StorageConfig> {
     StorageConfig::Builder builder;
     if (eargs.local_root.has_value()) {
         builder.SetBuildRoot(*eargs.local_root);
     }
 
-    auto config = builder.Build();
+    auto config =
+        builder
+            .SetRemoteExecutionArgs(
+                remote_address, remote_platform_properties, remote_dispatch)
+            .Build();
     if (config) {
         return *std::move(config);
     }
@@ -791,6 +803,20 @@ auto main(int argc, char* argv[]) -> int {
         SetupHashFunction();
         SetupFileChunker();
 
+        if (arguments.cmd == SubCommand::kGc) {
+            // Set up storage for GC, as we have all the config args we need.
+            auto const storage_config = CreateStorageConfig(arguments.endpoint);
+            if (not storage_config) {
+                return kExitFailure;
+            }
+
+            if (GarbageCollector::TriggerGarbageCollection(
+                    *storage_config, arguments.gc.no_rotate)) {
+                return kExitSuccess;
+            }
+            return kExitFailure;
+        }
+
         auto local_exec_config = CreateLocalExecutionConfig(arguments.build);
         if (not local_exec_config) {
             return kExitFailure;
@@ -810,24 +836,13 @@ auto main(int argc, char* argv[]) -> int {
             return kExitFailure;
         }
 
-        if (arguments.cmd == SubCommand::kGc) {
-            // Set up storage for GC, as we have all the config args we need.
-            auto const storage_config = CreateStorageConfig(arguments.endpoint);
-            if (not storage_config) {
-                return kExitFailure;
-            }
-            auto const storage = Storage::Create(&*storage_config);
-
-            if (GarbageCollector::TriggerGarbageCollection(
-                    *storage_config, arguments.gc.no_rotate)) {
-                return kExitSuccess;
-            }
-            return kExitFailure;
-        }
-
         if (arguments.cmd == SubCommand::kExecute) {
             // Set up storage for server-side operation.
-            auto const storage_config = CreateStorageConfig(arguments.endpoint);
+            auto const storage_config =
+                CreateStorageConfig(arguments.endpoint,
+                                    RemoteExecutionConfig::RemoteAddress(),
+                                    RemoteExecutionConfig::PlatformProperties(),
+                                    RemoteExecutionConfig::DispatchList());
             if (not storage_config) {
                 return kExitFailure;
             }
@@ -855,8 +870,11 @@ auto main(int argc, char* argv[]) -> int {
                                         arguments.service.pid_file);
             if (serve_server) {
                 // Set up storage for server-side operation.
-                auto const storage_config =
-                    CreateStorageConfig(arguments.endpoint);
+                auto const storage_config = CreateStorageConfig(
+                    arguments.endpoint,
+                    RemoteExecutionConfig::RemoteAddress(),
+                    RemoteExecutionConfig::PlatformProperties(),
+                    RemoteExecutionConfig::DispatchList());
                 if (not storage_config) {
                     return kExitFailure;
                 }
@@ -900,12 +918,20 @@ auto main(int argc, char* argv[]) -> int {
                         "Using '{}' as the remote execution endpoint.",
                         *arguments.serve.remote_serve_address);
         }
-#endif  // BOOTSTRAP_BUILD_TOOL
 
         // Set up storage for client-side operation. This needs to have all the
         // correct remote endpoint info in order to instantiate the
         // correctly-sharded target cache.
+        auto const storage_config =
+            CreateStorageConfig(arguments.endpoint,
+                                RemoteExecutionConfig::RemoteAddress(),
+                                RemoteExecutionConfig::PlatformProperties(),
+                                RemoteExecutionConfig::DispatchList());
+#else
+        // For bootstrapping the TargetCache sharding is not needed, so we can
+        // default all execution arguments.
         auto const storage_config = CreateStorageConfig(arguments.endpoint);
+#endif  // BOOTSTRAP_BUILD_TOOL
         if (not storage_config) {
             return kExitFailure;
         }
diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS
index 705d86c6..53982e2b 100644
--- a/src/buildtool/storage/TARGETS
+++ b/src/buildtool/storage/TARGETS
@@ -3,10 +3,13 @@
   , "name": ["config"]
   , "hdrs": ["config.hpp"]
   , "deps":
-    [ ["src/buildtool/execution_api/remote", "config"]
+    [ "backend_description"
     , ["@", "gsl", "", "gsl"]
+    , ["src/buildtool/common", "common"]
+    , ["src/buildtool/common/remote", "remote_common"]
     , ["src/buildtool/compatibility", "compatibility"]
     , ["src/buildtool/file_system", "file_system_manager"]
+    , ["src/buildtool/file_system", "object_type"]
     , ["src/buildtool/logging", "logging"]
     , ["src/buildtool/logging", "log_level"]
     , ["src/utils/cpp", "gsl"]
diff --git a/src/buildtool/storage/config.hpp b/src/buildtool/storage/config.hpp
index 7c4be29d..39bb74ac 100644
--- a/src/buildtool/storage/config.hpp
+++ b/src/buildtool/storage/config.hpp
@@ -17,13 +17,21 @@
 
 #include <cstddef>
 #include <filesystem>
+#include <map>
+#include <optional>
 #include <string>
+#include <utility>
+#include <vector>
 
 #include "gsl/gsl"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/remote/remote_common.hpp"
 #include "src/buildtool/compatibility/compatibility.hpp"
 #include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
 #include "src/buildtool/logging/log_level.hpp"
 #include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/storage/backend_description.hpp"
 #include "src/utils/cpp/expected.hpp"
 #include "src/utils/cpp/gsl.hpp"
 #include "src/utils/cpp/tmp_dir.hpp"
@@ -56,6 +64,9 @@ struct StorageConfig final {
     // Number of total storage generations (default: two generations).
     std::size_t const num_generations = 2;
 
+    // Hash of the execution backend description
+    std::string const backend_description_id = DefaultBackendDescriptionId();
+
     /// \brief Root directory of all storage generations.
     [[nodiscard]] auto CacheRoot() const noexcept -> std::filesystem::path {
         return build_root / "protocol-dependent";
@@ -121,6 +132,17 @@ struct StorageConfig final {
         bool is_compatible) -> std::filesystem::path {
         return dir / (is_compatible ? "compatible-sha256" : "git-sha1");
     };
+
+    [[nodiscard]] static auto DefaultBackendDescriptionId() noexcept
+        -> std::string {
+        try {
+            return ArtifactDigest::Create<ObjectType::File>(
+                       DescribeBackend(std::nullopt, {}, {}).value())
+                .hash();
+        } catch (...) {
+            return std::string();
+        }
+    }
 };
 
 class StorageConfig::Builder final {
@@ -136,6 +158,17 @@ class StorageConfig::Builder final {
         return *this;
     }
 
+    auto SetRemoteExecutionArgs(
+        std::optional<ServerAddress> address,
+        std::map<std::string, std::string> properties,
+        std::vector<std::pair<std::map<std::string, std::string>,
+                              ServerAddress>> dispatch) noexcept -> Builder& {
+        remote_address_ = std::move(address);
+        remote_platform_properties_ = std::move(properties);
+        remote_dispatch_ = std::move(dispatch);
+        return *this;
+    }
+
     [[nodiscard]] auto Build() const noexcept
         -> expected<StorageConfig, std::string> {
         // To not duplicate default arguments of StorageConfig in builder,
@@ -161,13 +194,33 @@ class StorageConfig::Builder final {
             }
         }
 
-        return StorageConfig{.build_root = std::move(build_root),
-                             .num_generations = num_generations};
+        // Hash the execution backend description
+        auto backend_description_id = default_config.backend_description_id;
+        auto desc = DescribeBackend(
+            remote_address_, remote_platform_properties_, remote_dispatch_);
+        if (desc) {
+            backend_description_id =
+                ArtifactDigest::Create<ObjectType::File>(*desc).hash();
+        }
+        else {
+            return unexpected{desc.error()};
+        }
+
+        return StorageConfig{
+            .build_root = std::move(build_root),
+            .num_generations = num_generations,
+            .backend_description_id = std::move(backend_description_id)};
     }
 
   private:
     std::optional<std::filesystem::path> build_root_;
     std::optional<std::size_t> num_generations_;
+
+    // Fields for computing remote execution backend description
+    std::optional<ServerAddress> remote_address_;
+    std::map<std::string, std::string> remote_platform_properties_;
+    std::vector<std::pair<std::map<std::string, std::string>, ServerAddress>>
+        remote_dispatch_;
 };
 
 #endif  // INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index 92f07944..871b86aa 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -205,7 +205,8 @@ void SetupLogging(MultiRepoLogArguments const& clargs) {
     if (args.just_mr_paths->root.has_value()) {
         builder.SetBuildRoot(*args.just_mr_paths->root);
     }
-
+    // As just-mr does not require the TargetCache, we do not need to set any of
+    // the remote execution fields for the backend description.
     auto config = builder.Build();
     if (config) {
         return *std::move(config);