WithRetry: Support reduced log level for failures

In some situations, we use a retry strategy, but in case of complete
failure have another way of attempting that task. In this case, we
should not emmit an error message right away. Add support for this.

3 files changed, 14 insertions, 8 deletions

diff --git a/src/buildtool/common/remote/TARGETS b/src/buildtool/common/remote/TARGETS
index 8ca9daa8..e494bd47 100644
--- a/src/buildtool/common/remote/TARGETS
+++ b/src/buildtool/common/remote/TARGETS
@@ -58,9 +58,9 @@
   , "deps":
     [ "retry_config"
     , ["@", "grpc", "", "grpc++"]
+    , ["src/buildtool/logging", "log_level"]
     , ["src/buildtool/logging", "logging"]
     ]
   , "stage": ["src", "buildtool", "common", "remote"]
-  , "private-deps": [["src/buildtool/logging", "log_level"]]
   }
 }
diff --git a/src/buildtool/common/remote/retry.cpp b/src/buildtool/common/remote/retry.cpp
index 00d275cd..8c97bf38 100644
--- a/src/buildtool/common/remote/retry.cpp
+++ b/src/buildtool/common/remote/retry.cpp
@@ -15,6 +15,7 @@
 #include "src/buildtool/common/remote/retry.hpp"
 #ifndef BOOTSTRAP_BUILD_TOOL
 
+#include <algorithm>
 #include <chrono>
 #include <thread>
 
@@ -22,7 +23,8 @@
 
 auto WithRetry(CallableReturningRetryResponse const& f,
                RetryConfig const& retry_config,
-               Logger const& logger) noexcept -> bool {
+               Logger const& logger,
+               LogLevel fatal_log_level) noexcept -> bool {
     try {
         auto const& attempts = retry_config.GetMaxAttempts();
         for (auto attempt = 1U; attempt <= attempts; ++attempt) {
@@ -32,7 +34,7 @@ auto WithRetry(CallableReturningRetryResponse const& f,
             }
             if (fatal) {
                 if (error_msg) {
-                    logger.Emit(LogLevel::Error, "{}", *error_msg);
+                    logger.Emit(fatal_log_level, "{}", *error_msg);
                 }
                 return false;
             }
@@ -51,7 +53,7 @@ auto WithRetry(CallableReturningRetryResponse const& f,
             }
             else {
                 if (error_msg) {
-                    logger.Emit(LogLevel::Error,
+                    logger.Emit(fatal_log_level,
                                 "After {} attempts: {}",
                                 attempt,
                                 *error_msg);
@@ -59,7 +61,8 @@ auto WithRetry(CallableReturningRetryResponse const& f,
             }
         }
     } catch (...) {
-        logger.Emit(LogLevel::Error, "WithRetry: caught unknown exception");
+        logger.Emit(std::min(fatal_log_level, LogLevel::Warning),
+                    "WithRetry: caught unknown exception");
     }
     return false;
 }
diff --git a/src/buildtool/common/remote/retry.hpp b/src/buildtool/common/remote/retry.hpp
index 6b8278a4..9387ef2a 100644
--- a/src/buildtool/common/remote/retry.hpp
+++ b/src/buildtool/common/remote/retry.hpp
@@ -24,6 +24,7 @@
 
 #include "grpcpp/grpcpp.h"
 #include "src/buildtool/common/remote/retry_config.hpp"
+#include "src/buildtool/logging/log_level.hpp"
 #include "src/buildtool/logging/logger.hpp"
 
 // Utility class to help detecting when exit the retry loop. This class can be
@@ -51,9 +52,11 @@ using CallableReturningRetryResponse = std::function<RetryResponse(void)>;
 /// \brief Calls a function with a retry strategy using a backoff algorithm.
 /// Retry loop interrupts when one of the two members of the function's returned
 /// RetryResponse object is set to true.
-[[nodiscard]] auto WithRetry(CallableReturningRetryResponse const& f,
-                             RetryConfig const& retry_config,
-                             Logger const& logger) noexcept -> bool;
+[[nodiscard]] auto WithRetry(
+    CallableReturningRetryResponse const& f,
+    RetryConfig const& retry_config,
+    Logger const& logger,
+    LogLevel fatal_log_level = LogLevel::Error) noexcept -> bool;
 
 using CallableReturningGrpcStatus = std::function<grpc::Status(void)>;