1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "src/buildtool/execution_api/common/common_api.hpp"
#include <cstddef>
#include <exception>
#include "fmt/core.h"
auto CommonRetrieveToFds(
std::vector<Artifact::ObjectInfo> const& artifacts_info,
std::vector<int> const& fds,
std::function<bool(Artifact::ObjectInfo const&,
gsl::not_null<FILE*> const&)> const& dump_to_stream,
std::optional<std::function<bool(Artifact::ObjectInfo const&, int)>> const&
fallback) noexcept -> bool {
if (artifacts_info.size() != fds.size()) {
Logger::Log(LogLevel::Error,
"different number of digests and file descriptors.");
return false;
}
for (std::size_t i{}; i < artifacts_info.size(); ++i) {
auto fd = fds[i];
auto const& info = artifacts_info[i];
if (gsl::owner<FILE*> out = fdopen(dup(fd), "wb")) { // NOLINT
bool success{false};
try {
success = dump_to_stream(info, out);
} catch (std::exception const& ex) {
std::fclose(out); // close file
Logger::Log(LogLevel::Error,
"dumping {} to stream failed with:\n{}",
info.ToString(),
ex.what());
return false;
}
std::fclose(out);
if (not success) {
Logger::Log(
LogLevel::Debug,
"dumping {} {} from CAS to file descriptor {} failed.",
IsTreeObject(info.type) ? "tree" : "blob",
info.ToString(),
fd);
// locally we might be able to fallback to Git in native mode
try {
if (fallback and not(*fallback)(info, fd)) {
return false;
}
} catch (std::exception const& ex) {
Logger::Log(LogLevel::Error,
"fallback dumping {} to file descriptor {} "
"failed with:\n{}",
info.ToString(),
fd,
ex.what());
return false;
}
}
}
else {
Logger::Log(
LogLevel::Error, "dumping to file descriptor {} failed.", fd);
return false;
}
}
return true;
}
/// NOLINTNEXTLINE(misc-no-recursion)
auto CommonUploadBlobTree(BlobTreePtr const& blob_tree,
IExecutionApi const& api) noexcept -> bool {
// Create digest list from blobs for batch availability check.
auto missing_blobs_info = GetMissingArtifactsInfo<BlobTreePtr>(
api, blob_tree->begin(), blob_tree->end(), [](BlobTreePtr const& node) {
return node->Blob().digest;
});
if (not missing_blobs_info) {
Logger::Log(LogLevel::Error,
"Failed to retrieve the missing tree blobs for upload");
return false;
}
// Process missing blobs.
ArtifactBlobContainer container;
for (auto const& digest : missing_blobs_info->digests) {
if (auto it = missing_blobs_info->back_map.find(digest);
it != missing_blobs_info->back_map.end()) {
auto const& node = it->second;
// Process trees.
if (node->IsTree()) {
if (not CommonUploadBlobTree(node, api)) {
return false;
}
}
// Optimize store & upload by taking into account the maximum
// transfer size.
if (not UpdateContainerAndUpload<ArtifactDigest>(
&container,
std::move(node->Blob()),
/*exception_is_fatal=*/false,
[&api](ArtifactBlobContainer&& blobs) -> bool {
return api.Upload(std::move(blobs),
/*skip_find_missing=*/true);
})) {
return false;
}
}
}
// Transfer any remaining blobs.
return api.Upload(std::move(container), /*skip_find_missing=*/true);
}
auto CommonUploadTreeCompatible(
IExecutionApi const& api,
DirectoryTreePtr const& build_root,
BazelMsgFactory::LinkDigestResolveFunc const& resolve_links) noexcept
-> std::optional<ArtifactDigest> {
ArtifactBlobContainer blobs{};
// Store and upload blobs, taking into account the maximum transfer size.
auto digest = BazelMsgFactory::CreateDirectoryDigestFromTree(
build_root, resolve_links, [&blobs, &api](ArtifactBlob&& blob) {
return UpdateContainerAndUpload<ArtifactDigest>(
&blobs,
std::move(blob),
/*exception_is_fatal=*/false,
[&api](ArtifactBlobContainer&& container) -> bool {
return api.Upload(std::move(container),
/*skip_find_missing=*/false);
});
});
if (not digest) {
Logger::Log(LogLevel::Debug, "failed to create digest for build root.");
return std::nullopt;
}
Logger::Log(LogLevel::Trace, [&digest]() {
std::ostringstream oss{};
oss << "upload root directory" << std::endl;
oss << fmt::format(" - root digest: {}", digest->hash()) << std::endl;
return oss.str();
});
// Upload remaining blobs.
if (not api.Upload(std::move(blobs), /*skip_find_missing=*/false)) {
Logger::Log(LogLevel::Debug, "failed to upload blobs for build root.");
return std::nullopt;
}
return digest;
}
auto CommonUploadTreeNative(IExecutionApi const& api,
DirectoryTreePtr const& build_root) noexcept
-> std::optional<ArtifactDigest> {
auto blob_tree = BlobTree::FromDirectoryTree(build_root);
if (not blob_tree) {
Logger::Log(LogLevel::Debug,
"failed to create blob tree for build root.");
return std::nullopt;
}
auto tree_blob = (*blob_tree)->Blob();
// Upload blob tree if tree is not available at the remote side (content
// first).
if (not api.IsAvailable(tree_blob.digest)) {
if (not CommonUploadBlobTree(*blob_tree, api)) {
Logger::Log(LogLevel::Debug,
"failed to upload blob tree for build root.");
return std::nullopt;
}
if (not api.Upload(ArtifactBlobContainer{{tree_blob}},
/*skip_find_missing=*/true)) {
Logger::Log(LogLevel::Debug,
"failed to upload tree blob for build root.");
return std::nullopt;
}
}
return tree_blob.digest;
}
|
