1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
#include "src/buildtool/build_engine/target_map/target_cache.hpp"
#include <gsl-lite/gsl-lite.hpp>
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/execution_api/local/config.hpp"
#include "src/buildtool/execution_api/local/file_storage.hpp"
#include "src/buildtool/execution_api/local/local_cas.hpp"
#include "src/buildtool/execution_api/remote/config.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/logging/logger.hpp"
auto TargetCache::Key::Create(BuildMaps::Base::EntityName const& target,
Configuration const& effective_config) noexcept
-> std::optional<TargetCache::Key> {
static auto const& repos = RepositoryConfig::Instance();
try {
if (auto repo_key =
repos.RepositoryKey(target.GetNamedTarget().repository)) {
// target's repository is content-fixed, we can compute a cache key
auto const& name = target.GetNamedTarget();
auto target_desc = nlohmann::json{
{{"repo_key", *repo_key},
{"target_name", nlohmann::json{name.module, name.name}.dump()},
{"effective_config", effective_config.ToString()}}};
static auto const& cas = LocalCAS<ObjectType::File>::Instance();
if (auto target_key = cas.StoreBlobFromBytes(target_desc.dump(2))) {
return Key{{ArtifactDigest{*target_key}, ObjectType::File}};
}
}
} catch (std::exception const& ex) {
Logger::Log(LogLevel::Error,
"Creating target cache key failed with:\n{}",
ex.what());
}
return std::nullopt;
}
auto TargetCache::Entry::FromTarget(
AnalysedTargetPtr const& target,
std::unordered_map<ArtifactDescription, Artifact::ObjectInfo> const&
replacements) noexcept -> std::optional<TargetCache::Entry> {
auto result = TargetResult{
target->Artifacts(), target->Provides(), target->RunFiles()};
if (auto desc = result.ReplaceNonKnownAndToJson(replacements)) {
return Entry{*desc};
}
return std::nullopt;
}
auto TargetCache::Entry::ToResult() const -> std::optional<TargetResult> {
return TargetResult::FromJson(desc_);
}
auto TargetCache::Store(Key const& key, Entry const& value) const noexcept
-> bool {
// Before a target-cache entry is stored in local CAS, make sure any created
// artifact for this target is downloaded from the remote CAS to the local
// CAS.
if (not DownloadKnownArtifacts(value)) {
return false;
}
if (auto digest = CAS().StoreBlobFromBytes(value.ToJson().dump(2))) {
auto data =
Artifact::ObjectInfo{ArtifactDigest{*digest}, ObjectType::File}
.ToString();
logger_.Emit(LogLevel::Debug,
"Adding entry for key {} as {}",
key.Id().ToString(),
data);
return file_store_.AddFromBytes(key.Id().digest.hash(), data);
}
return false;
}
auto TargetCache::Read(Key const& key) const noexcept
-> std::optional<std::pair<Entry, Artifact::ObjectInfo>> {
auto entry_path = file_store_.GetPath(key.Id().digest.hash());
auto const entry =
FileSystemManager::ReadFile(entry_path, ObjectType::File);
if (not entry.has_value()) {
logger_.Emit(LogLevel::Debug,
"Cache miss, entry not found {}",
entry_path.string());
return std::nullopt;
}
if (auto info = Artifact::ObjectInfo::FromString(*entry)) {
if (auto path = CAS().BlobPath(info->digest)) {
if (auto value = FileSystemManager::ReadFile(*path)) {
try {
return std::make_pair(Entry{nlohmann::json::parse(*value)},
std::move(*info));
} catch (std::exception const& ex) {
logger_.Emit(LogLevel::Warning,
"Parsing entry for key {} failed with:\n{}",
key.Id().ToString(),
ex.what());
}
}
}
}
logger_.Emit(LogLevel::Warning,
"Reading entry for key {} failed",
key.Id().ToString());
return std::nullopt;
}
auto TargetCache::DownloadKnownArtifactsFromMap(
Expression::map_t const& expr_map) const noexcept -> bool {
// Get object infos of KNOWN artifacts from map.
std::vector<Artifact::ObjectInfo> infos;
infos.reserve(expr_map.size());
for (auto const& item : expr_map) {
try {
auto const& desc = item.second->Artifact();
// The assumption is that all artifacts mentioned in a target cache
// entry are KNOWN to the remote side. So they can be fetched to the
// local CAS.
gsl_ExpectsAudit(desc.IsKnown());
infos.push_back(*desc.ToArtifact().Info());
} catch (...) {
return false;
}
}
#ifndef BOOTSTRAP_BUILD_TOOL
// Sync KNOWN artifacts from remote to local CAS.
return remote_api_->RetrieveToCas(infos, local_api_);
#else
return true;
#endif
}
auto TargetCache::DownloadKnownArtifacts(Entry const& value) const noexcept
-> bool {
auto const& result = value.ToResult();
if (not DownloadKnownArtifactsFromMap(result->artifact_stage->Map())) {
return false;
}
if (not DownloadKnownArtifactsFromMap(result->runfiles->Map())) {
return false;
}
return true;
}
auto TargetCache::ComputeCacheDir() -> std::filesystem::path {
return LocalExecutionConfig::TargetCacheDir() / ExecutionBackendId();
}
auto TargetCache::ExecutionBackendId() -> std::string {
auto address = RemoteExecutionConfig::RemoteAddress();
auto properties = RemoteExecutionConfig::PlatformProperties();
auto backend_desc = nlohmann::json{
{"remote_address",
address ? nlohmann::json{fmt::format(
"{}:{}", address->host, address->port)}
: nlohmann::json{}},
{"platform_properties",
properties}}.dump(2);
return NativeSupport::Unprefix(
CAS().StoreBlobFromBytes(backend_desc).value().hash());
}
|
