just-mr: support retry for temporary grpc failures

... using the same mechanism that just uses. As just-mr now also
does quite some grpc calls, e.g., fetching distribution files from
the remote-execution endpoint and negociating roots with the serve
endpoint, it may now also be affected by a bad network or a service
temporarily be unavailable.

5 files changed, 38 insertions, 10 deletions

diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index e71f144c..7633eaea 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -4,24 +4,26 @@
   , "name": ["just-mr"]
   , "srcs": ["main.cpp"]
   , "private-deps":
-    [ ["src/buildtool/build_engine/expression", "expression"]
+    [ ["@", "cli11", "", "cli11"]
+    , ["@", "gsl", "", "gsl"]
+    , ["@", "json", "", "json"]
+    , ["src/buildtool/build_engine/expression", "expression"]
+    , ["src/buildtool/common", "retry_cli"]
+    , ["src/buildtool/compatibility", "compatibility"]
+    , ["src/buildtool/file_system", "git_context"]
     , ["src/buildtool/logging", "logging"]
     , ["src/buildtool/logging", "log_level"]
+    , ["src/buildtool/main", "retry"]
     , ["src/buildtool/main", "version"]
+    , ["src/buildtool/storage", "storage"]
     , "cli"
     , "exit_codes"
-    , ["@", "cli11", "", "cli11"]
-    , ["@", "gsl", "", "gsl"]
-    , ["@", "json", "", "json"]
-    , ["src/buildtool/storage", "storage"]
-    , ["src/buildtool/file_system", "git_context"]
-    , ["src/buildtool/compatibility", "compatibility"]
     , "fetch"
-    , "update"
-    , "setup"
     , "launch"
-    , "setup_utils"
     , "rc"
+    , "setup"
+    , "setup_utils"
+    , "update"
     ]
   , "stage": ["src", "other_tools", "just_mr"]
   , "private-ldflags":
@@ -65,6 +67,7 @@
     , ["@", "gsl", "", "gsl"]
     , ["@", "json", "", "json"]
     , ["src/buildtool/common", "clidefaults"]
+    , ["src/buildtool/common", "retry_cli"]
     , ["src/buildtool/common", "user_structs"]
     , ["src/buildtool/execution_api/local", "config"]
     , ["src/buildtool/logging", "log_level"]
diff --git a/src/other_tools/just_mr/cli.hpp b/src/other_tools/just_mr/cli.hpp
index d76eee69..cf4b0346 100644
--- a/src/other_tools/just_mr/cli.hpp
+++ b/src/other_tools/just_mr/cli.hpp
@@ -29,6 +29,7 @@
 #include "gsl/gsl"
 #include "nlohmann/json.hpp"
 #include "src/buildtool/common/clidefaults.hpp"
+#include "src/buildtool/common/retry_cli.hpp"
 #include "src/buildtool/common/user_structs.hpp"
 #include "src/buildtool/execution_api/local/config.hpp"
 #include "src/buildtool/logging/log_level.hpp"
@@ -114,6 +115,7 @@ enum class SubCommand {
 struct CommandLineArguments {
     SubCommand cmd{SubCommand::kUnknown};
     MultiRepoCommonArguments common;
+    RetryArguments retry;
     MultiRepoLogArguments log;
     MultiRepoSetupArguments setup;
     MultiRepoFetchArguments fetch;
diff --git a/src/other_tools/just_mr/launch.cpp b/src/other_tools/just_mr/launch.cpp
index 1236d9f4..693d8808 100644
--- a/src/other_tools/just_mr/launch.cpp
+++ b/src/other_tools/just_mr/launch.cpp
@@ -35,6 +35,7 @@ auto CallJust(std::optional<std::filesystem::path> const& config_file,
               MultiRepoJustSubCmdsArguments const& just_cmd_args,
               MultiRepoLogArguments const& log_args,
               MultiRepoRemoteAuthArguments const& auth_args,
+              RetryArguments const& retry_args,
               ForwardOnlyArguments const& launch_fwd,
               bool forward_build_root,
               std::string multi_repo_tool_name) -> int {
@@ -185,6 +186,19 @@ auto CallJust(std::optional<std::filesystem::path> const& config_file,
             cmd.emplace_back("--tls-client-key");
             cmd.emplace_back(auth_args.tls_client_key->string());
         }
+        if (retry_args.max_attempts) {
+            cmd.emplace_back("--max-attempts");
+            cmd.emplace_back(std::to_string(*retry_args.max_attempts));
+        }
+        if (retry_args.initial_backoff_seconds) {
+            cmd.emplace_back("--initial-backoff-seconds");
+            cmd.emplace_back(
+                std::to_string(*retry_args.initial_backoff_seconds));
+        }
+        if (retry_args.max_backoff_seconds) {
+            cmd.emplace_back("--max-backoff-seconds");
+            cmd.emplace_back(std::to_string(*retry_args.max_backoff_seconds));
+        }
     }
     if (supports_dispatch and just_cmd_args.endpoint_configuration) {
         cmd.emplace_back("--endpoint-configuration");
diff --git a/src/other_tools/just_mr/launch.hpp b/src/other_tools/just_mr/launch.hpp
index 0560f42e..d4c7ba17 100644
--- a/src/other_tools/just_mr/launch.hpp
+++ b/src/other_tools/just_mr/launch.hpp
@@ -28,6 +28,7 @@
     MultiRepoJustSubCmdsArguments const& just_cmd_args,
     MultiRepoLogArguments const& log_args,
     MultiRepoRemoteAuthArguments const& auth_args,
+    RetryArguments const& retry_args,
     ForwardOnlyArguments const& launch_fwd,
     bool forward_build_root,
     std::string multi_repo_tool_name) -> int;
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index 442d74f6..31f63d3d 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -26,6 +26,7 @@
 #include "gsl/gsl"
 #include "nlohmann/json.hpp"
 #include "src/buildtool/build_engine/expression/configuration.hpp"
+#include "src/buildtool/common/retry_cli.hpp"
 #include "src/buildtool/compatibility/compatibility.hpp"
 #include "src/buildtool/file_system/git_context.hpp"
 #include "src/buildtool/logging/log_config.hpp"
@@ -33,6 +34,7 @@
 #include "src/buildtool/logging/log_sink_cmdline.hpp"
 #include "src/buildtool/logging/log_sink_file.hpp"
 #include "src/buildtool/logging/logger.hpp"
+#include "src/buildtool/main/retry.hpp"
 #include "src/buildtool/main/version.hpp"
 #include "src/buildtool/storage/garbage_collector.hpp"
 #include "src/other_tools/just_mr/cli.hpp"
@@ -53,6 +55,7 @@ void SetupCommonCommandArguments(
     SetupMultiRepoCommonArguments(app, &clargs->common);
     SetupMultiRepoLogArguments(app, &clargs->log);
     SetupMultiRepoRemoteAuthArguments(app, &clargs->auth);
+    SetupRetryArguments(app, &clargs->retry);
 }
 
 /// \brief Setup arguments for subcommand "just-mr fetch".
@@ -311,6 +314,10 @@ auto main(int argc, char* argv[]) -> int {
             return kExitGenericFailure;
         }
 
+        if (not SetupRetryConfig(arguments.retry)) {
+            return kExitGenericFailure;
+        }
+
         /**
          * The current implementation of libgit2 uses pthread_key_t incorrectly
          * on POSIX systems to handle thread-specific data, which requires us to
@@ -336,6 +343,7 @@ auto main(int argc, char* argv[]) -> int {
                             arguments.just_cmd,
                             arguments.log,
                             arguments.auth,
+                            arguments.retry,
                             arguments.launch_fwd,
                             forward_build_root,
                             my_name);