summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildtool/main/TARGETS1
-rw-r--r--src/buildtool/main/main.cpp68
-rw-r--r--src/buildtool/storage/TARGETS5
-rw-r--r--src/buildtool/storage/config.hpp57
-rw-r--r--src/other_tools/just_mr/main.cpp3
5 files changed, 109 insertions, 25 deletions
diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS
index 5e8e457b..c4c7f122 100644
--- a/src/buildtool/main/TARGETS
+++ b/src/buildtool/main/TARGETS
@@ -9,6 +9,7 @@
, ["@", "json", "", "json"]
, ["src/buildtool/common", "common"]
, ["src/buildtool/common", "config"]
+ , ["src/buildtool/common/remote", "remote_common"]
, ["src/buildtool/storage", "storage"]
, ["src/buildtool/compatibility", "compatibility"]
, ["src/buildtool/graph_traverser", "graph_traverser"]
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index 36f03345..716545a5 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -17,6 +17,7 @@
#include <filesystem>
#include <fstream>
#include <iostream>
+#include <map>
#include <mutex>
#include <optional>
#include <string>
@@ -24,6 +25,7 @@
#include <unordered_set>
#include <utility>
#include <variant>
+#include <vector>
#include "gsl/gsl"
#include "nlohmann/json.hpp"
@@ -32,6 +34,7 @@
#include "src/buildtool/build_engine/expression/expression.hpp"
#include "src/buildtool/build_engine/target_map/target_map.hpp"
#include "src/buildtool/common/artifact_description.hpp"
+#include "src/buildtool/common/remote/remote_common.hpp"
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/common/statistics.hpp"
#include "src/buildtool/compatibility/compatibility.hpp"
@@ -102,14 +105,23 @@ void SetupLogging(LogArguments const& clargs) {
}
}
-[[nodiscard]] auto CreateStorageConfig(EndpointArguments const& eargs) noexcept
+[[nodiscard]] auto CreateStorageConfig(
+ EndpointArguments const& eargs,
+ std::optional<ServerAddress> const& remote_address = std::nullopt,
+ std::map<std::string, std::string> const& remote_platform_properties = {},
+ std::vector<std::pair<std::map<std::string, std::string>,
+ ServerAddress>> const& remote_dispatch = {}) noexcept
-> std::optional<StorageConfig> {
StorageConfig::Builder builder;
if (eargs.local_root.has_value()) {
builder.SetBuildRoot(*eargs.local_root);
}
- auto config = builder.Build();
+ auto config =
+ builder
+ .SetRemoteExecutionArgs(
+ remote_address, remote_platform_properties, remote_dispatch)
+ .Build();
if (config) {
return *std::move(config);
}
@@ -791,6 +803,20 @@ auto main(int argc, char* argv[]) -> int {
SetupHashFunction();
SetupFileChunker();
+ if (arguments.cmd == SubCommand::kGc) {
+ // Set up storage for GC, as we have all the config args we need.
+ auto const storage_config = CreateStorageConfig(arguments.endpoint);
+ if (not storage_config) {
+ return kExitFailure;
+ }
+
+ if (GarbageCollector::TriggerGarbageCollection(
+ *storage_config, arguments.gc.no_rotate)) {
+ return kExitSuccess;
+ }
+ return kExitFailure;
+ }
+
auto local_exec_config = CreateLocalExecutionConfig(arguments.build);
if (not local_exec_config) {
return kExitFailure;
@@ -810,24 +836,13 @@ auto main(int argc, char* argv[]) -> int {
return kExitFailure;
}
- if (arguments.cmd == SubCommand::kGc) {
- // Set up storage for GC, as we have all the config args we need.
- auto const storage_config = CreateStorageConfig(arguments.endpoint);
- if (not storage_config) {
- return kExitFailure;
- }
- auto const storage = Storage::Create(&*storage_config);
-
- if (GarbageCollector::TriggerGarbageCollection(
- *storage_config, arguments.gc.no_rotate)) {
- return kExitSuccess;
- }
- return kExitFailure;
- }
-
if (arguments.cmd == SubCommand::kExecute) {
// Set up storage for server-side operation.
- auto const storage_config = CreateStorageConfig(arguments.endpoint);
+ auto const storage_config =
+ CreateStorageConfig(arguments.endpoint,
+ RemoteExecutionConfig::RemoteAddress(),
+ RemoteExecutionConfig::PlatformProperties(),
+ RemoteExecutionConfig::DispatchList());
if (not storage_config) {
return kExitFailure;
}
@@ -855,8 +870,11 @@ auto main(int argc, char* argv[]) -> int {
arguments.service.pid_file);
if (serve_server) {
// Set up storage for server-side operation.
- auto const storage_config =
- CreateStorageConfig(arguments.endpoint);
+ auto const storage_config = CreateStorageConfig(
+ arguments.endpoint,
+ RemoteExecutionConfig::RemoteAddress(),
+ RemoteExecutionConfig::PlatformProperties(),
+ RemoteExecutionConfig::DispatchList());
if (not storage_config) {
return kExitFailure;
}
@@ -900,12 +918,20 @@ auto main(int argc, char* argv[]) -> int {
"Using '{}' as the remote execution endpoint.",
*arguments.serve.remote_serve_address);
}
-#endif // BOOTSTRAP_BUILD_TOOL
// Set up storage for client-side operation. This needs to have all the
// correct remote endpoint info in order to instantiate the
// correctly-sharded target cache.
+ auto const storage_config =
+ CreateStorageConfig(arguments.endpoint,
+ RemoteExecutionConfig::RemoteAddress(),
+ RemoteExecutionConfig::PlatformProperties(),
+ RemoteExecutionConfig::DispatchList());
+#else
+ // For bootstrapping the TargetCache sharding is not needed, so we can
+ // default all execution arguments.
auto const storage_config = CreateStorageConfig(arguments.endpoint);
+#endif // BOOTSTRAP_BUILD_TOOL
if (not storage_config) {
return kExitFailure;
}
diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS
index 705d86c6..53982e2b 100644
--- a/src/buildtool/storage/TARGETS
+++ b/src/buildtool/storage/TARGETS
@@ -3,10 +3,13 @@
, "name": ["config"]
, "hdrs": ["config.hpp"]
, "deps":
- [ ["src/buildtool/execution_api/remote", "config"]
+ [ "backend_description"
, ["@", "gsl", "", "gsl"]
+ , ["src/buildtool/common", "common"]
+ , ["src/buildtool/common/remote", "remote_common"]
, ["src/buildtool/compatibility", "compatibility"]
, ["src/buildtool/file_system", "file_system_manager"]
+ , ["src/buildtool/file_system", "object_type"]
, ["src/buildtool/logging", "logging"]
, ["src/buildtool/logging", "log_level"]
, ["src/utils/cpp", "gsl"]
diff --git a/src/buildtool/storage/config.hpp b/src/buildtool/storage/config.hpp
index 7c4be29d..39bb74ac 100644
--- a/src/buildtool/storage/config.hpp
+++ b/src/buildtool/storage/config.hpp
@@ -17,13 +17,21 @@
#include <cstddef>
#include <filesystem>
+#include <map>
+#include <optional>
#include <string>
+#include <utility>
+#include <vector>
#include "gsl/gsl"
+#include "src/buildtool/common/artifact_digest.hpp"
+#include "src/buildtool/common/remote/remote_common.hpp"
#include "src/buildtool/compatibility/compatibility.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
+#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/storage/backend_description.hpp"
#include "src/utils/cpp/expected.hpp"
#include "src/utils/cpp/gsl.hpp"
#include "src/utils/cpp/tmp_dir.hpp"
@@ -56,6 +64,9 @@ struct StorageConfig final {
// Number of total storage generations (default: two generations).
std::size_t const num_generations = 2;
+ // Hash of the execution backend description
+ std::string const backend_description_id = DefaultBackendDescriptionId();
+
/// \brief Root directory of all storage generations.
[[nodiscard]] auto CacheRoot() const noexcept -> std::filesystem::path {
return build_root / "protocol-dependent";
@@ -121,6 +132,17 @@ struct StorageConfig final {
bool is_compatible) -> std::filesystem::path {
return dir / (is_compatible ? "compatible-sha256" : "git-sha1");
};
+
+ [[nodiscard]] static auto DefaultBackendDescriptionId() noexcept
+ -> std::string {
+ try {
+ return ArtifactDigest::Create<ObjectType::File>(
+ DescribeBackend(std::nullopt, {}, {}).value())
+ .hash();
+ } catch (...) {
+ return std::string();
+ }
+ }
};
class StorageConfig::Builder final {
@@ -136,6 +158,17 @@ class StorageConfig::Builder final {
return *this;
}
+ auto SetRemoteExecutionArgs(
+ std::optional<ServerAddress> address,
+ std::map<std::string, std::string> properties,
+ std::vector<std::pair<std::map<std::string, std::string>,
+ ServerAddress>> dispatch) noexcept -> Builder& {
+ remote_address_ = std::move(address);
+ remote_platform_properties_ = std::move(properties);
+ remote_dispatch_ = std::move(dispatch);
+ return *this;
+ }
+
[[nodiscard]] auto Build() const noexcept
-> expected<StorageConfig, std::string> {
// To not duplicate default arguments of StorageConfig in builder,
@@ -161,13 +194,33 @@ class StorageConfig::Builder final {
}
}
- return StorageConfig{.build_root = std::move(build_root),
- .num_generations = num_generations};
+ // Hash the execution backend description
+ auto backend_description_id = default_config.backend_description_id;
+ auto desc = DescribeBackend(
+ remote_address_, remote_platform_properties_, remote_dispatch_);
+ if (desc) {
+ backend_description_id =
+ ArtifactDigest::Create<ObjectType::File>(*desc).hash();
+ }
+ else {
+ return unexpected{desc.error()};
+ }
+
+ return StorageConfig{
+ .build_root = std::move(build_root),
+ .num_generations = num_generations,
+ .backend_description_id = std::move(backend_description_id)};
}
private:
std::optional<std::filesystem::path> build_root_;
std::optional<std::size_t> num_generations_;
+
+ // Fields for computing remote execution backend description
+ std::optional<ServerAddress> remote_address_;
+ std::map<std::string, std::string> remote_platform_properties_;
+ std::vector<std::pair<std::map<std::string, std::string>, ServerAddress>>
+ remote_dispatch_;
};
#endif // INCLUDED_SRC_BUILDTOOL_STORAGE_CONFIG_HPP
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index 92f07944..871b86aa 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -205,7 +205,8 @@ void SetupLogging(MultiRepoLogArguments const& clargs) {
if (args.just_mr_paths->root.has_value()) {
builder.SetBuildRoot(*args.just_mr_paths->root);
}
-
+ // As just-mr does not require the TargetCache, we do not need to set any of
+ // the remote execution fields for the backend description.
auto config = builder.Build();
if (config) {
return *std::move(config);
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 18:20:23 +0000