just-mrrc: support retry options

2 files changed, 58 insertions, 0 deletions

diff --git a/share/man/just-mrrc.5.md b/share/man/just-mrrc.5.md
index c9e4a073..8ebaff18 100644
--- a/share/man/just-mrrc.5.md
+++ b/share/man/just-mrrc.5.md
@@ -127,6 +127,21 @@ The just-mrrc is given by a JSON object.
    as `--remote-execution-property` to the invocation of the build
    tool, if **`just-mr`** is used as a launcher.
 
+ - The value for the key *`"max attempts"`*, if provided, has
+   to be a number. If a remote procedure call (rpc) returns
+   `grpc::StatusCode::UNAVAILABLE`, that rpc is retried at most
+   this number of times.
+
+ - The value for the key *`"initial backoff seconds"`*, if provided,
+   has to be a number. Before retrying an rpc the second time, the
+   client will wait the given amount of seconds plus a jitter, to
+   better distribute the workload.
+
+ - The value for the key *`"max backoff seconds"`*, if provided,
+   has to be a number. Normally, on subsequent retries, the backoff
+   time is doubled; this number specifies the maximal time between
+   attempts of an rpc, not counting the jitter.
+
  - The value for the key *`"just files"`* is a JSON object. The keys correspond
    to options that some **`just`** subcommands accept and require a file as
    argument. For each key, the value is a list of location objects. When
diff --git a/src/other_tools/just_mr/rc.cpp b/src/other_tools/just_mr/rc.cpp
index c5de5ad5..8a81a686 100644
--- a/src/other_tools/just_mr/rc.cpp
+++ b/src/other_tools/just_mr/rc.cpp
@@ -385,6 +385,49 @@ namespace {
                 entry->String());
         }
     }
+    // read the defaults for the retry parameters
+    if (not clargs->retry.max_attempts) {
+        auto max_attempts = rc_config["max attempts"];
+        if (max_attempts.IsNotNull()) {
+            if (not max_attempts->IsNumber()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-file provided \"max attempts\" has "
+                            "to be a number, but found {}",
+                            max_attempts->ToString());
+                std::exit(kExitConfigError);
+            }
+            clargs->retry.max_attempts =
+                static_cast<unsigned int>(std::lround(max_attempts->Number()));
+        }
+    }
+    if (not clargs->retry.initial_backoff_seconds) {
+        auto initial_backoff_seconds = rc_config["initial backoff seconds"];
+        if (initial_backoff_seconds.IsNotNull()) {
+            if (not initial_backoff_seconds->IsNumber()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-file provided \"initial backoff "
+                            "seconds\" has to be a number, but found {}",
+                            initial_backoff_seconds->ToString());
+                std::exit(kExitConfigError);
+            }
+            clargs->retry.initial_backoff_seconds = static_cast<unsigned int>(
+                std::lround(initial_backoff_seconds->Number()));
+        }
+    }
+    if (not clargs->retry.max_backoff_seconds) {
+        auto max_backoff_seconds = rc_config["max backoff seconds"];
+        if (max_backoff_seconds.IsNotNull()) {
+            if (not max_backoff_seconds->IsNumber()) {
+                Logger::Log(LogLevel::Error,
+                            "Configuration-file provided \"max backoff "
+                            "seconds\" has to be a number, but found {}",
+                            max_backoff_seconds->ToString());
+                std::exit(kExitConfigError);
+            }
+            clargs->retry.max_backoff_seconds = static_cast<unsigned int>(
+                std::lround(max_backoff_seconds->Number()));
+        }
+    }
     // read default for local launcher
     if (not clargs->common.local_launcher) {
         auto launcher = rc_config["local launcher"];