summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/other_tools/just_mr/setup.cpp1
-rw-r--r--src/other_tools/root_maps/TARGETS2
-rw-r--r--src/other_tools/root_maps/commit_git_map.cpp156
-rw-r--r--src/other_tools/root_maps/commit_git_map.hpp2
4 files changed, 134 insertions, 27 deletions
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 1d698e3a..6d568bc8 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -119,6 +119,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
CreateCommitGitMap(&critical_git_op_map,
&import_to_git_map,
common_args.just_mr_paths,
+ common_args.alternative_mirrors,
common_args.git_path->string(),
*common_args.local_launcher,
serve_api ? &(*serve_api) : nullptr,
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index 6d3cd6e4..fff48505 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -33,6 +33,7 @@
[ ["src/buildtool/serve_api/remote", "serve_api"]
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/common", "user_structs"]
+ , ["src/other_tools/just_mr", "mirrors"]
, ["src/other_tools/ops_maps", "critical_git_op_map"]
, ["src/other_tools/ops_maps", "import_to_git_map"]
, ["src/utils/cpp", "hash_combine"]
@@ -47,6 +48,7 @@
, ["src/buildtool/file_system", "file_root"]
, ["src/buildtool/storage", "config"]
, ["src/buildtool/storage", "fs_utils"]
+ , ["src/other_tools/utils", "curl_url_handle"]
]
}
, "fpath_git_map":
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp
index 6959dfe6..ad269b5e 100644
--- a/src/other_tools/root_maps/commit_git_map.cpp
+++ b/src/other_tools/root_maps/commit_git_map.cpp
@@ -22,6 +22,7 @@
#include "src/other_tools/git_operations/git_repo_remote.hpp"
#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
+#include "src/other_tools/utils/curl_url_handle.hpp"
#include "src/utils/cpp/tmp_dir.hpp"
namespace {
@@ -87,6 +88,7 @@ void WriteIdFileAndSetWSRoot(std::string const& root_tree_id,
void EnsureCommit(GitRepoInfo const& repo_info,
std::filesystem::path const& repo_root,
std::string const& fetch_repo,
+ MirrorsPtr const& additional_mirrors,
GitCASPtr const& git_cas,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
@@ -356,42 +358,138 @@ void EnsureCommit(GitRepoInfo const& repo_info,
return;
}
// store failed attempts for subsequent logging
- std::string err_messages{};
- auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
- [fetch_repo, &err_messages](auto const& msg, bool /*fatal*/) {
- err_messages +=
- fmt::format("\nWhile attempting fetch from remote {}:\n{}",
- fetch_repo,
- msg);
- });
- // try the main fetch URL
bool fetched{false};
- if (not git_repo->FetchViaTmpRepo(tmp_dir->GetPath(),
- fetch_repo,
+ std::string err_messages{};
+ // try local mirrors first
+ auto local_mirrors =
+ MirrorsUtils::GetLocalMirrors(additional_mirrors, fetch_repo);
+ for (auto mirror : local_mirrors) {
+ if (GitURLIsPath(mirror)) {
+ mirror = std::filesystem::absolute(mirror).string();
+ }
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [mirror, &err_messages](auto const& msg, bool /*fatal*/) {
+ err_messages += fmt::format(
+ "\nWhile attempting fetch from local mirror "
+ "{}:\n{}",
+ mirror,
+ msg);
+ });
+ if (git_repo->FetchViaTmpRepo(tmp_dir->GetPath(),
+ mirror,
repo_info.branch,
git_bin,
launcher,
wrapped_logger)) {
- // try to fetch from mirrors, in order, if given
- for (auto mirror : repo_info.mirrors) {
- if (GitURLIsPath(mirror)) {
- mirror = std::filesystem::absolute(mirror).string();
+ fetched = true;
+ break;
+ }
+ }
+ if (not fetched) {
+ // get preferred hostnames list
+ auto preferred_hostnames =
+ MirrorsUtils::GetPreferredHostnames(additional_mirrors);
+ // try first the main URL, but with each of the preferred hostnames,
+ // if URL is not a path
+ if (not GitURLIsPath(fetch_repo)) {
+ for (auto const& hostname : preferred_hostnames) {
+ if (auto preferred_url = CurlURLHandle::ReplaceHostname(
+ fetch_repo, hostname)) {
+ auto wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [preferred_url, &err_messages](auto const& msg,
+ bool /*fatal*/) {
+ err_messages += fmt::format(
+ "\nWhile attempting fetch from remote "
+ "{}:\n{}",
+ *preferred_url,
+ msg);
+ });
+ if (git_repo->FetchViaTmpRepo(tmp_dir->GetPath(),
+ *preferred_url,
+ repo_info.branch,
+ git_bin,
+ launcher,
+ wrapped_logger)) {
+ fetched = true;
+ break;
+ }
+ }
}
- wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
- [mirror, &err_messages](auto const& msg, bool /*fatal*/) {
+ }
+ if (not fetched) {
+ // now try the original main fetch URL
+ auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>(
+ [fetch_repo, &err_messages](auto const& msg,
+ bool /*fatal*/) {
err_messages += fmt::format(
- "\nWhile attempting fetch from mirror {}:\n{}",
- mirror,
+ "\nWhile attempting fetch from remote {}:\n{}",
+ fetch_repo,
msg);
});
- if (git_repo->FetchViaTmpRepo(tmp_dir->GetPath(),
- mirror,
- repo_info.branch,
- git_bin,
- launcher,
- wrapped_logger)) {
- fetched = true;
- break;
+ if (not git_repo->FetchViaTmpRepo(tmp_dir->GetPath(),
+ fetch_repo,
+ repo_info.branch,
+ git_bin,
+ launcher,
+ wrapped_logger)) {
+ // now try to fetch from mirrors, in order, if given
+ for (auto mirror : repo_info.mirrors) {
+ if (GitURLIsPath(mirror)) {
+ mirror = std::filesystem::absolute(mirror).string();
+ }
+ else {
+ // if non-path, try each of the preferred hostnames
+ for (auto const& hostname : preferred_hostnames) {
+ if (auto preferred_mirror =
+ CurlURLHandle::ReplaceHostname(
+ mirror, hostname)) {
+ wrapped_logger = std::make_shared<
+ AsyncMapConsumerLogger>(
+ [preferred_mirror, &err_messages](
+ auto const& msg, bool /*fatal*/) {
+ err_messages += fmt::format(
+ "\nWhile attempting fetch from "
+ "mirror {}:\n{}",
+ *preferred_mirror,
+ msg);
+ });
+ if (git_repo->FetchViaTmpRepo(
+ tmp_dir->GetPath(),
+ *preferred_mirror,
+ repo_info.branch,
+ git_bin,
+ launcher,
+ wrapped_logger)) {
+ fetched = true;
+ break;
+ }
+ }
+ }
+ }
+ if (fetched) {
+ break;
+ }
+ wrapped_logger =
+ std::make_shared<AsyncMapConsumerLogger>(
+ [mirror, &err_messages](auto const& msg,
+ bool /*fatal*/) {
+ err_messages += fmt::format(
+ "\nWhile attempting fetch from mirror "
+ "{}:\n{}",
+ mirror,
+ msg);
+ });
+ if (git_repo->FetchViaTmpRepo(tmp_dir->GetPath(),
+ mirror,
+ repo_info.branch,
+ git_bin,
+ launcher,
+ wrapped_logger)) {
+ fetched = true;
+ break;
+ }
+ }
}
}
}
@@ -523,6 +621,7 @@ auto CreateCommitGitMap(
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
LocalPathsPtr const& just_mr_paths,
+ MirrorsPtr const& additional_mirrors,
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi* serve_api,
@@ -533,6 +632,7 @@ auto CreateCommitGitMap(
auto commit_to_git = [critical_git_op_map,
import_to_git_map,
just_mr_paths,
+ additional_mirrors,
git_bin,
launcher,
serve_api,
@@ -570,6 +670,7 @@ auto CreateCommitGitMap(
[key,
repo_root,
fetch_repo,
+ additional_mirrors,
critical_git_op_map,
import_to_git_map,
git_bin,
@@ -601,6 +702,7 @@ auto CreateCommitGitMap(
EnsureCommit(key,
repo_root,
fetch_repo,
+ additional_mirrors,
op_result.git_cas,
critical_git_op_map,
import_to_git_map,
diff --git a/src/other_tools/root_maps/commit_git_map.hpp b/src/other_tools/root_maps/commit_git_map.hpp
index a85f04e1..70623598 100644
--- a/src/other_tools/root_maps/commit_git_map.hpp
+++ b/src/other_tools/root_maps/commit_git_map.hpp
@@ -23,6 +23,7 @@
#include "src/buildtool/common/user_structs.hpp"
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/serve_api/remote/serve_api.hpp"
+#include "src/other_tools/just_mr/mirrors.hpp"
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
#include "src/other_tools/ops_maps/import_to_git_map.hpp"
#include "src/utils/cpp/hash_combine.hpp"
@@ -72,6 +73,7 @@ using CommitGitMap =
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
LocalPathsPtr const& just_mr_paths,
+ MirrorsPtr const& additional_mirrors,
std::string const& git_bin,
std::vector<std::string> const& launcher,
ServeApi* serve_api,
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 21:29:16 +0000