summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/other_tools/just_mr/TARGETS10
-rw-r--r--src/other_tools/just_mr/fetch.cpp20
-rw-r--r--src/other_tools/just_mr/fetch.hpp3
-rw-r--r--src/other_tools/just_mr/launch.cpp1
-rw-r--r--src/other_tools/just_mr/main.cpp8
-rw-r--r--src/other_tools/just_mr/setup.cpp17
-rw-r--r--src/other_tools/just_mr/setup.hpp1
-rw-r--r--src/other_tools/just_mr/setup_utils.cpp67
-rw-r--r--src/other_tools/just_mr/setup_utils.hpp7
-rw-r--r--src/other_tools/ops_maps/TARGETS1
-rw-r--r--src/other_tools/ops_maps/content_cas_map.cpp23
-rw-r--r--src/other_tools/ops_maps/content_cas_map.hpp3
12 files changed, 146 insertions, 15 deletions
diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index 894d9259..e0a05163 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -84,6 +84,8 @@
, "deps":
[ ["src/buildtool/build_engine/expression", "expression_ptr_interface"]
, ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/buildtool/execution_api/common", "common"]
+ , "cli"
]
, "stage": ["src", "other_tools", "just_mr"]
, "private-deps":
@@ -91,6 +93,9 @@
, ["src/buildtool/file_system", "file_system_manager"]
, ["src/buildtool/logging", "logging"]
, "exit_codes"
+ , ["src/buildtool/auth", "auth"]
+ , ["src/buildtool/execution_api/bazel_msg", "bazel_msg"]
+ , ["src/buildtool/execution_api/remote", "bazel"]
]
}
, "fetch":
@@ -107,10 +112,11 @@
, "exit_codes"
, ["src/other_tools/just_mr/progress_reporting", "progress"]
, ["src/other_tools/just_mr/progress_reporting", "progress_reporter"]
- , "utils"
, ["src/other_tools/ops_maps", "content_cas_map"]
, ["src/other_tools/ops_maps", "repo_fetch_map"]
, "setup_utils"
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/execution_api/local", "local"]
]
}
, "update":
@@ -156,6 +162,8 @@
, ["src/other_tools/root_maps", "tree_id_git_map"]
, ["src/other_tools/symlinks_map", "resolve_symlinks_map"]
, "setup_utils"
+ , ["src/buildtool/execution_api/common", "common"]
+ , ["src/buildtool/execution_api/local", "local"]
]
}
, "launch":
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index 078586b2..49cca32e 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -17,6 +17,8 @@
#include <filesystem>
#include "nlohmann/json.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/local/local_api.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/logging/logger.hpp"
#include "src/buildtool/multithreading/task_system.hpp"
@@ -24,14 +26,14 @@
#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
#include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp"
#include "src/other_tools/just_mr/setup_utils.hpp"
-#include "src/other_tools/just_mr/utils.hpp"
#include "src/other_tools/ops_maps/content_cas_map.hpp"
#include "src/other_tools/ops_maps/repo_fetch_map.hpp"
auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
MultiRepoCommonArguments const& common_args,
MultiRepoSetupArguments const& setup_args,
- MultiRepoFetchArguments const& fetch_args) -> int {
+ MultiRepoFetchArguments const& fetch_args,
+ MultiRepoRemoteAuthArguments const& auth_args) -> int {
// provide report
Logger::Log(LogLevel::Info, "Performing repositories fetch");
@@ -249,9 +251,19 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
nr,
nr == 1 ? "archive" : "archives");
+ // setup the APIs for archive fetches
+ auto remote_api = JustMR::Utils::SetupRemoteApi(
+ common_args.remote_execution_address, auth_args);
+ IExecutionApi::Ptr local_api{remote_api ? std::make_unique<LocalApi>()
+ : nullptr};
+
// create async maps
- auto content_cas_map = CreateContentCASMap(
- common_args.just_mr_paths, common_args.ca_info, common_args.jobs);
+ auto content_cas_map =
+ CreateContentCASMap(common_args.just_mr_paths,
+ common_args.ca_info,
+ local_api ? &(*local_api) : nullptr,
+ remote_api ? &(*remote_api) : nullptr,
+ common_args.jobs);
auto repo_fetch_map =
CreateRepoFetchMap(&content_cas_map, *fetch_dir, common_args.jobs);
diff --git a/src/other_tools/just_mr/fetch.hpp b/src/other_tools/just_mr/fetch.hpp
index be96630b..e9f07acf 100644
--- a/src/other_tools/just_mr/fetch.hpp
+++ b/src/other_tools/just_mr/fetch.hpp
@@ -22,7 +22,8 @@
[[nodiscard]] auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
MultiRepoCommonArguments const& common_args,
MultiRepoSetupArguments const& setup_args,
- MultiRepoFetchArguments const& fetch_args)
+ MultiRepoFetchArguments const& fetch_args,
+ MultiRepoRemoteAuthArguments const& auth_args)
-> int;
#endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_FETCH_HPP
diff --git a/src/other_tools/just_mr/launch.cpp b/src/other_tools/just_mr/launch.cpp
index bc70bb7f..c843d3c1 100644
--- a/src/other_tools/just_mr/launch.cpp
+++ b/src/other_tools/just_mr/launch.cpp
@@ -67,6 +67,7 @@ auto CallJust(std::optional<std::filesystem::path> const& config_file,
common_args,
setup_args,
just_cmd_args,
+ auth_args,
/*interactive=*/false);
if (not mr_config_path) {
Logger::Log(LogLevel::Error,
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index e56c7064..21cba2f7 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -729,6 +729,7 @@ auto main(int argc, char* argv[]) -> int {
arguments.common,
arguments.setup,
arguments.just_cmd,
+ arguments.auth,
/*interactive=*/(arguments.cmd == SubCommand::kSetupEnv));
// dump resulting config to stdout
if (not mr_config_path) {
@@ -748,8 +749,11 @@ auto main(int argc, char* argv[]) -> int {
// Run subcommand `fetch`
if (arguments.cmd == SubCommand::kFetch) {
- return MultiRepoFetch(
- config, arguments.common, arguments.setup, arguments.fetch);
+ return MultiRepoFetch(config,
+ arguments.common,
+ arguments.setup,
+ arguments.fetch,
+ arguments.auth);
}
// Unknown subcommand should fail
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 2532ce3e..7b606fd3 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -17,6 +17,8 @@
#include <filesystem>
#include "nlohmann/json.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/buildtool/execution_api/local/local_api.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/logging/logger.hpp"
#include "src/buildtool/multithreading/task_system.hpp"
@@ -38,6 +40,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
MultiRepoCommonArguments const& common_args,
MultiRepoSetupArguments const& setup_args,
MultiRepoJustSubCmdsArguments const& just_cmd_args,
+ MultiRepoRemoteAuthArguments const& auth_args,
bool interactive) -> std::optional<std::filesystem::path> {
// provide report
Logger::Log(LogLevel::Info, "Performing repositories setup");
@@ -85,11 +88,21 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
JustMR::Utils::ReachableRepositories(repos, *main, setup_repos);
}
+ // setup the APIs for archive fetches
+ auto remote_api = JustMR::Utils::SetupRemoteApi(
+ common_args.remote_execution_address, auth_args);
+ IExecutionApi::Ptr local_api{remote_api ? std::make_unique<LocalApi>()
+ : nullptr};
+
// setup the required async maps
auto crit_git_op_ptr = std::make_shared<CriticalGitOpGuard>();
auto critical_git_op_map = CreateCriticalGitOpMap(crit_git_op_ptr);
- auto content_cas_map = CreateContentCASMap(
- common_args.just_mr_paths, common_args.ca_info, common_args.jobs);
+ auto content_cas_map =
+ CreateContentCASMap(common_args.just_mr_paths,
+ common_args.ca_info,
+ local_api ? &(*local_api) : nullptr,
+ remote_api ? &(*remote_api) : nullptr,
+ common_args.jobs);
auto import_to_git_map =
CreateImportToGitMap(&critical_git_op_map,
common_args.git_path->string(),
diff --git a/src/other_tools/just_mr/setup.hpp b/src/other_tools/just_mr/setup.hpp
index da452fcc..b11c6497 100644
--- a/src/other_tools/just_mr/setup.hpp
+++ b/src/other_tools/just_mr/setup.hpp
@@ -26,6 +26,7 @@
MultiRepoCommonArguments const& common_args,
MultiRepoSetupArguments const& setup_args,
MultiRepoJustSubCmdsArguments const& just_cmd_args,
+ MultiRepoRemoteAuthArguments const& auth_args,
bool interactive) -> std::optional<std::filesystem::path>;
#endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_SETUP_HPP
diff --git a/src/other_tools/just_mr/setup_utils.cpp b/src/other_tools/just_mr/setup_utils.cpp
index 9aad1894..28355e89 100644
--- a/src/other_tools/just_mr/setup_utils.cpp
+++ b/src/other_tools/just_mr/setup_utils.cpp
@@ -18,11 +18,55 @@
#include <unordered_set>
#include "nlohmann/json.hpp"
+#include "src/buildtool/auth/authentication.hpp"
#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/buildtool/execution_api/bazel_msg/bazel_common.hpp"
+#include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/logging/logger.hpp"
#include "src/other_tools/just_mr/exit_codes.hpp"
+namespace {
+
+void SetupAuthConfig(MultiRepoRemoteAuthArguments const& authargs) {
+ bool use_tls{false};
+ if (authargs.tls_ca_cert) {
+ use_tls = true;
+ if (not Auth::TLS::SetCACertificate(*authargs.tls_ca_cert)) {
+ Logger::Log(LogLevel::Error,
+ "Could not read '{}' certificate.",
+ authargs.tls_ca_cert->string());
+ std::exit(kExitConfigError);
+ }
+ }
+ if (authargs.tls_client_cert) {
+ use_tls = true;
+ if (not Auth::TLS::SetClientCertificate(*authargs.tls_client_cert)) {
+ Logger::Log(LogLevel::Error,
+ "Could not read '{}' certificate.",
+ authargs.tls_client_cert->string());
+ std::exit(kExitConfigError);
+ }
+ }
+ if (authargs.tls_client_key) {
+ use_tls = true;
+ if (not Auth::TLS::SetClientKey(*authargs.tls_client_key)) {
+ Logger::Log(LogLevel::Error,
+ "Could not read '{}' key.",
+ authargs.tls_client_key->string());
+ std::exit(kExitConfigError);
+ }
+ }
+
+ if (use_tls) {
+ if (not Auth::TLS::Validate()) {
+ std::exit(kExitConfigError);
+ }
+ }
+}
+
+} // namespace
+
namespace JustMR::Utils {
void ReachableRepositories(
@@ -135,4 +179,27 @@ auto ReadConfiguration(
return config;
}
+auto SetupRemoteApi(std::optional<std::string> const& remote_exec_addr,
+ MultiRepoRemoteAuthArguments const& auth)
+ -> IExecutionApi::Ptr {
+ // we only allow remotes in native mode
+ if (remote_exec_addr and not Compatibility::IsCompatible()) {
+ // setup authentication
+ SetupAuthConfig(auth);
+ // setup remote
+ if (not RemoteExecutionConfig::SetRemoteAddress(*remote_exec_addr)) {
+ Logger::Log(LogLevel::Error,
+ "setting remote execution address '{}' failed.",
+ *remote_exec_addr);
+ std::exit(kExitConfigError);
+ }
+ auto address = RemoteExecutionConfig::RemoteAddress();
+ ExecutionConfiguration config;
+ config.skip_cache_lookup = false;
+ return std::make_unique<BazelApi>(
+ "remote-execution", address->host, address->port, config);
+ }
+ return nullptr;
+}
+
} // namespace JustMR::Utils
diff --git a/src/other_tools/just_mr/setup_utils.hpp b/src/other_tools/just_mr/setup_utils.hpp
index a8c4236e..2bc067cf 100644
--- a/src/other_tools/just_mr/setup_utils.hpp
+++ b/src/other_tools/just_mr/setup_utils.hpp
@@ -23,6 +23,8 @@
#include "src/buildtool/build_engine/expression/configuration.hpp"
#include "src/buildtool/build_engine/expression/expression_ptr.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
+#include "src/other_tools/just_mr/cli.hpp"
/* Setup-related constants and utilities for just-mr */
@@ -55,6 +57,11 @@ void DefaultReachableRepositories(
std::optional<std::filesystem::path> const& config_file_opt) noexcept
-> std::shared_ptr<Configuration>;
+/// \brief Setup of a remote API based on just-mr arguments.
+auto SetupRemoteApi(std::optional<std::string> const& remote_exec_addr,
+ MultiRepoRemoteAuthArguments const& auth)
+ -> IExecutionApi::Ptr;
+
} // namespace Utils
} // namespace JustMR
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS
index 21a63306..8f88f8b0 100644
--- a/src/other_tools/ops_maps/TARGETS
+++ b/src/other_tools/ops_maps/TARGETS
@@ -57,6 +57,7 @@
, "srcs": ["content_cas_map.cpp"]
, "deps":
[ ["src/other_tools/just_mr", "utils"]
+ , ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/multithreading", "async_map_consumer"]
, ["src/utils/cpp", "hash_combine"]
, ["@", "json", "", "json"]
diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp
index 4a25b15c..51d909b5 100644
--- a/src/other_tools/ops_maps/content_cas_map.cpp
+++ b/src/other_tools/ops_maps/content_cas_map.cpp
@@ -49,12 +49,15 @@ template <Hasher::HashType type>
auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths,
JustMR::CAInfoPtr const& ca_info,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
std::size_t jobs) -> ContentCASMap {
- auto ensure_in_cas = [just_mr_paths, ca_info](auto /*unused*/,
- auto setter,
- auto logger,
- auto /*unused*/,
- auto const& key) {
+ auto ensure_in_cas = [just_mr_paths, ca_info, local_api, remote_api](
+ auto /*unused*/,
+ auto setter,
+ auto logger,
+ auto /*unused*/,
+ auto const& key) {
// check if content already in CAS
auto const& cas = Storage::Instance().CAS();
auto digest = ArtifactDigest(key.content, 0, false);
@@ -63,6 +66,16 @@ auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths,
return;
}
JustMRProgress::Instance().TaskTracker().Start(key.origin);
+ // check if content is in remote CAS, if a remote is given
+ if (remote_api and local_api and remote_api->IsAvailable({digest}) and
+ remote_api->RetrieveToCas(
+ {Artifact::ObjectInfo{.digest = digest,
+ .type = ObjectType::File}},
+ local_api)) {
+ JustMRProgress::Instance().TaskTracker().Stop(key.origin);
+ (*setter)(true);
+ return;
+ }
// add distfile to CAS
auto repo_distfile =
(key.distfile
diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp
index 3b40010f..1dc18dcd 100644
--- a/src/other_tools/ops_maps/content_cas_map.hpp
+++ b/src/other_tools/ops_maps/content_cas_map.hpp
@@ -18,6 +18,7 @@
#include <string>
#include "nlohmann/json.hpp"
+#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/multithreading/async_map_consumer.hpp"
#include "src/other_tools/just_mr/utils.hpp"
#include "src/utils/cpp/hash_combine.hpp"
@@ -58,6 +59,8 @@ using ContentCASMap = AsyncMapConsumer<ArchiveContent, bool>;
[[nodiscard]] auto CreateContentCASMap(JustMR::PathsPtr const& just_mr_paths,
JustMR::CAInfoPtr const& ca_info,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
std::size_t jobs) -> ContentCASMap;
namespace std {
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 17:10:34 +0000