summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorPaul Cristian Sarbu <paul.cristian.sarbu@huawei.com>2023-08-24 14:02:37 +0200
committerPaul Cristian Sarbu <paul.cristian.sarbu@huawei.com>2023-08-25 18:11:23 +0200
commit089733c819066b801e28d6441cbff887e60aef51 (patch)
treeb5d1528c0ce3e25f042c70684812711fa33fb9b8 /src
parent833974c986eb46f88c87b3a95b4b840a333b24da (diff)
downloadjustbuild-089733c819066b801e28d6441cbff887e60aef51.tar.gz
just-mr fetch: Allow to back up fetched archives to a given remote CAS
Diffstat (limited to 'src')
-rw-r--r--src/other_tools/just_mr/cli.hpp5
-rw-r--r--src/other_tools/just_mr/fetch.cpp8
-rw-r--r--src/other_tools/just_mr/main.cpp9
-rw-r--r--src/other_tools/ops_maps/TARGETS3
-rw-r--r--src/other_tools/ops_maps/repo_fetch_map.cpp131
-rw-r--r--src/other_tools/ops_maps/repo_fetch_map.hpp3
6 files changed, 103 insertions, 56 deletions
diff --git a/src/other_tools/just_mr/cli.hpp b/src/other_tools/just_mr/cli.hpp
index 60c8f745..484665a5 100644
--- a/src/other_tools/just_mr/cli.hpp
+++ b/src/other_tools/just_mr/cli.hpp
@@ -63,6 +63,7 @@ struct MultiRepoSetupArguments {
struct MultiRepoFetchArguments {
std::optional<std::filesystem::path> fetch_dir{std::nullopt};
+ bool backup_to_remote{false};
};
struct MultiRepoUpdateArguments {
@@ -243,6 +244,10 @@ static inline void SetupMultiRepoFetchArguments(
},
"Directory to write distfiles when fetching.")
->type_name("PATH");
+ app->add_flag("--backup-to-remote",
+ clargs->backup_to_remote,
+ "Backup fetched archives to a remote CAS, if a "
+ "remote-execution service is provided.");
}
static inline void SetupMultiRepoUpdateArguments(
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index 49cca32e..4519a2cc 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -264,8 +264,12 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
local_api ? &(*local_api) : nullptr,
remote_api ? &(*remote_api) : nullptr,
common_args.jobs);
- auto repo_fetch_map =
- CreateRepoFetchMap(&content_cas_map, *fetch_dir, common_args.jobs);
+ auto repo_fetch_map = CreateRepoFetchMap(
+ &content_cas_map,
+ *fetch_dir,
+ (fetch_args.backup_to_remote and local_api) ? &(*local_api) : nullptr,
+ (fetch_args.backup_to_remote and remote_api) ? &(*remote_api) : nullptr,
+ common_args.jobs);
// set up progress observer
JustMRProgress::Instance().SetTotal(repos_to_fetch.size());
diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp
index 21cba2f7..29e53048 100644
--- a/src/other_tools/just_mr/main.cpp
+++ b/src/other_tools/just_mr/main.cpp
@@ -749,6 +749,15 @@ auto main(int argc, char* argv[]) -> int {
// Run subcommand `fetch`
if (arguments.cmd == SubCommand::kFetch) {
+ // check fetch configuration arguments for validity
+ if (arguments.common.remote_execution_address and
+ arguments.fetch.backup_to_remote and
+ Compatibility::IsCompatible()) {
+ Logger::Log(LogLevel::Error,
+ "Remote backup for fetched archives only available "
+ "in native mode!");
+ return kExitConfigError;
+ }
return MultiRepoFetch(config,
arguments.common,
arguments.setup,
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS
index 8f88f8b0..a2121e33 100644
--- a/src/other_tools/ops_maps/TARGETS
+++ b/src/other_tools/ops_maps/TARGETS
@@ -77,7 +77,8 @@
, "name": ["repo_fetch_map"]
, "hdrs": ["repo_fetch_map.hpp"]
, "srcs": ["repo_fetch_map.cpp"]
- , "deps": ["content_cas_map"]
+ , "deps":
+ ["content_cas_map", ["src/buildtool/execution_api/common", "common"]]
, "stage": ["src", "other_tools", "ops_maps"]
, "private-deps":
[ ["src/other_tools/just_mr", "utils"]
diff --git a/src/other_tools/ops_maps/repo_fetch_map.cpp b/src/other_tools/ops_maps/repo_fetch_map.cpp
index 1cc663d7..f8d28d8d 100644
--- a/src/other_tools/ops_maps/repo_fetch_map.cpp
+++ b/src/other_tools/ops_maps/repo_fetch_map.cpp
@@ -20,61 +20,96 @@
#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
#include "src/other_tools/just_mr/utils.hpp"
+namespace {
+
+void ProcessContent(std::filesystem::path const& content_path,
+ std::filesystem::path const& target_name,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
+ std::string const& content,
+ ArtifactDigest const& digest,
+ RepoFetchMap::SetterPtr const& setter,
+ RepoFetchMap::LoggerPtr const& logger) {
+ // try to back up to remote CAS
+ if (local_api != nullptr and remote_api != nullptr) {
+ if (not local_api->RetrieveToCas(
+ {Artifact::ObjectInfo{.digest = digest,
+ .type = ObjectType::File}},
+ remote_api)) {
+ // give a warning
+ (*logger)(fmt::format("Failed to back up content {} from local CAS "
+ "to remote",
+ content),
+ /*fatal=*/false);
+ }
+ }
+ // then, copy content into fetch_dir
+ if (FileSystemManager::Exists(target_name)) {
+ std::filesystem::permissions(target_name,
+ std::filesystem::perms::owner_write,
+ std::filesystem::perm_options::add);
+ }
+ if (not FileSystemManager::CopyFile(content_path, target_name)) {
+ (*logger)(fmt::format("Failed to copy content {} from CAS to {}",
+ content,
+ target_name.string()),
+ /*fatal=*/true);
+ return;
+ }
+ // success
+ JustMRStatistics::Instance().IncrementExecutedCounter();
+ (*setter)(true);
+}
+
+} // namespace
+
auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map,
std::filesystem::path const& fetch_dir,
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
std::size_t jobs) -> RepoFetchMap {
- auto fetch_repo = [content_cas_map, fetch_dir](auto ts,
- auto setter,
- auto logger,
- auto /* unused */,
- auto const& key) {
+ auto fetch_repo = [content_cas_map, fetch_dir, local_api, remote_api](
+ auto ts,
+ auto setter,
+ auto logger,
+ auto /* unused */,
+ auto const& key) {
// get corresponding distfile
auto distfile =
(key.archive.distfile ? key.archive.distfile.value()
: std::filesystem::path(key.archive.fetch_url)
.filename()
.string());
+ auto target_name = fetch_dir / distfile;
// check if content not already in CAS
+ auto digest = ArtifactDigest(key.archive.content, 0, false);
auto const& cas = Storage::Instance().CAS();
- auto content_path =
- cas.BlobPath(ArtifactDigest(key.archive.content, 0, false),
- /*is_executable=*/false);
+ auto content_path = cas.BlobPath(digest,
+ /*is_executable=*/false);
if (not content_path) {
// make sure content is in CAS
content_cas_map->ConsumeAfterKeysReady(
ts,
{key.archive},
- [fetch_dir,
+ [target_name,
+ local_api,
+ remote_api,
content = key.archive.content,
- distfile,
+ digest = std::move(digest),
setter,
logger]([[maybe_unused]] auto const& values) {
- // content is now in CAS, so copy content into fetch_dir
auto const& cas = Storage::Instance().CAS();
- auto content_path =
- cas.BlobPath(ArtifactDigest(content, 0, false),
- /*is_executable=*/false)
- .value();
- auto target_name = fetch_dir / distfile;
- if (FileSystemManager::Exists(target_name)) {
- std::filesystem::permissions(
- target_name,
- std::filesystem::perms::owner_write,
- std::filesystem::perm_options::add);
- }
- if (not FileSystemManager::CopyFile(content_path,
- target_name)) {
- (*logger)(
- fmt::format("Failed to copy content {} from CAS "
- "to {}",
- content,
- target_name.string()),
- /*fatal=*/true);
- return;
- }
- // success
- JustMRStatistics::Instance().IncrementExecutedCounter();
- (*setter)(true);
+ auto content_path = cas.BlobPath(digest,
+ /*is_executable=*/false)
+ .value();
+ ProcessContent(content_path,
+ target_name,
+ local_api,
+ remote_api,
+ content,
+ digest,
+ setter,
+ logger);
},
[logger, content = key.archive.content](auto const& msg,
bool fatal) {
@@ -86,24 +121,14 @@ auto CreateRepoFetchMap(gsl::not_null<ContentCASMap*> const& content_cas_map,
});
}
else {
- auto target_name = fetch_dir / distfile;
- if (FileSystemManager::Exists(target_name)) {
- std::filesystem::permissions(
- target_name,
- std::filesystem::perms::owner_write,
- std::filesystem::perm_options::add);
- }
- if (not FileSystemManager::CopyFile(*content_path, target_name)) {
- (*logger)(fmt::format("Failed to copy content {} from CAS "
- "to {}",
- key.archive.content,
- target_name.string()),
- /*fatal=*/true);
- return;
- }
- // success
- JustMRStatistics::Instance().IncrementCacheHitsCounter();
- (*setter)(true);
+ ProcessContent(*content_path,
+ target_name,
+ local_api,
+ remote_api,
+ key.archive.content,
+ digest,
+ setter,
+ logger);
}
};
return AsyncMapConsumer<ArchiveRepoInfo, bool>(fetch_repo, jobs);
diff --git a/src/other_tools/ops_maps/repo_fetch_map.hpp b/src/other_tools/ops_maps/repo_fetch_map.hpp
index 9952bce5..424c9122 100644
--- a/src/other_tools/ops_maps/repo_fetch_map.hpp
+++ b/src/other_tools/ops_maps/repo_fetch_map.hpp
@@ -15,6 +15,7 @@
#ifndef INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP
#define INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP
+#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/other_tools/ops_maps/content_cas_map.hpp"
/// \brief Maps an archive content hash to a status flag.
@@ -23,6 +24,8 @@ using RepoFetchMap = AsyncMapConsumer<ArchiveRepoInfo, bool>;
[[nodiscard]] auto CreateRepoFetchMap(
gsl::not_null<ContentCASMap*> const& content_cas_map,
std::filesystem::path const& fetch_dir, // should exist!
+ IExecutionApi* local_api,
+ IExecutionApi* remote_api,
std::size_t jobs) -> RepoFetchMap;
#endif // INCLUDED_SRC_OTHER_TOOLS_OPS_MAPS_REPO_FETCH_MAP_HPP \ No newline at end of file