1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <optional>
#include <thread>
#include <utility> // std::move
#include "grpcpp/grpcpp.h"
#include "src/buildtool/common/remote/retry_parameters.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/logging/logger.hpp"
// Utility class to help detecting when exit the retry loop. This class can be
// used when the failure cannot be immediately detected by the return value of
// the function. E.g., when using a grpc stream.
//
// Please note that it is user's responsibility to do not set both to true.
//
// Design note: even though only one bool could be sufficient (e.g., exit), this
// would require to check two times if we exited because of a success or a
// failure: the first time, inside the retry loop; the second time, by the
// caller.
struct RetryResponse {
// When set to true, it means the function successfully run
bool ok{false};
// When set to true, it means that it is not worthy to retry.
bool exit_retry_loop{false};
// error message logged when exit_retry_loop was set to true or when the
// last retry attempt failed
std::optional<std::string> error_msg{std::nullopt};
};
template <typename F>
concept CallableReturningRetryResponse = requires(F const& f) {
{RetryResponse{f()}};
};
template <CallableReturningRetryResponse F>
// \p f is the callable invoked with a back off algorithm. The retry loop is
// interrupted when one of the two member of the returned RetryResponse object
// is set to true.
[[nodiscard]] auto WithRetry(F const& f, Logger const& logger) noexcept
-> bool {
try {
auto const& attempts = Retry::GetMaxAttempts();
for (auto attempt = 1U; attempt <= attempts; ++attempt) {
auto [ok, fatal, error_msg] = f();
if (ok) {
return true;
}
if (fatal) {
if (error_msg) {
logger.Emit(LogLevel::Error, *error_msg);
}
return false;
}
// don't wait if it was the last attempt
if (attempt < attempts) {
auto const sleep_for_seconds =
Retry::GetSleepTimeSeconds(attempt);
logger.Emit(kRetryLogLevel,
"Attempt {}/{} failed{} Retrying in {} seconds.",
attempt,
attempts,
error_msg ? fmt::format(": {}", *error_msg) : ".",
sleep_for_seconds);
std::this_thread::sleep_for(
std::chrono::seconds(sleep_for_seconds));
}
else {
if (error_msg) {
logger.Emit(LogLevel::Error,
"After {} attempts: {}",
attempt,
*error_msg);
}
}
}
} catch (...) {
logger.Emit(LogLevel::Error, "WithRetry: caught unknown exception");
}
return false;
}
template <typename F>
concept CallableReturningGrpcStatus = requires(F const& f) {
{grpc::Status{f()}};
};
template <CallableReturningGrpcStatus F>
// F is the function to be invoked with a back off algorithm
[[nodiscard]] auto WithRetry(F const& f, Logger const& logger) noexcept
-> std::pair<bool, grpc::Status> {
grpc::Status status{};
try {
auto attempts = Retry::GetMaxAttempts();
for (auto attempt = 1U; attempt <= attempts; ++attempt) {
status = f();
if (status.ok() or
status.error_code() != grpc::StatusCode::UNAVAILABLE) {
return {status.ok(), std::move(status)};
}
// don't wait if it was the last attempt
if (attempt < attempts) {
auto const sleep_for_seconds =
Retry::GetSleepTimeSeconds(attempt);
logger.Emit(
kRetryLogLevel,
"Attempt {}/{} failed: {}: {}: Retrying in {} seconds.",
attempt,
attempts,
static_cast<int>(status.error_code()),
status.error_message(),
sleep_for_seconds);
std::this_thread::sleep_for(
std::chrono::seconds(sleep_for_seconds));
}
else {
// The caller performs a second check on the
// status.error_code(), and, eventually, emits to Error level
// there.
//
// To avoid duplication of similar errors, we emit to Debug
// level.
logger.Emit(LogLevel::Debug,
"After {} attempts: {}: {}",
attempt,
static_cast<int>(status.error_code()),
status.error_message());
}
}
} catch (...) {
logger.Emit(LogLevel::Error, "WithRetry: caught unknown exception");
}
return {false, std::move(status)};
}
|
