diff options
| -rw-r--r-- | src/other_tools/root_maps/commit_git_map.cpp | 159 | ||||
| -rw-r--r-- | src/other_tools/utils/content.hpp | 82 | ||||
| -rw-r--r-- | test/end-to-end/just-mr/just-mr-mirrors.test.sh | 4 |
3 files changed, 42 insertions, 203 deletions
diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp index e5910b9b..8bd18cc9 100644 --- a/src/other_tools/root_maps/commit_git_map.cpp +++ b/src/other_tools/root_maps/commit_git_map.cpp @@ -226,10 +226,27 @@ void NetworkFetchAndSetPresentRoot( std::string err_messages{}; // keep all remotes checked to report them in case fetch fails std::string remotes_buffer{}; - // try local mirrors first + + // try repo url + auto all_mirrors = std::vector<std::string>({fetch_repo}); + // try repo mirrors afterwards + all_mirrors.insert( + all_mirrors.end(), repo_info.mirrors.begin(), repo_info.mirrors.end()); + + if (auto preferred_hostnames = + MirrorsUtils::GetPreferredHostnames(additional_mirrors); + not preferred_hostnames.empty()) { + all_mirrors = + MirrorsUtils::SortByHostname(all_mirrors, preferred_hostnames); + } + + // always try local mirrors first auto local_mirrors = MirrorsUtils::GetLocalMirrors(additional_mirrors, fetch_repo); - for (auto mirror : local_mirrors) { + all_mirrors.insert( + all_mirrors.begin(), local_mirrors.begin(), local_mirrors.end()); + + for (auto mirror : all_mirrors) { auto mirror_path = GitURLIsPath(mirror); if (mirror_path) { mirror = std::filesystem::absolute(*mirror_path).string(); @@ -237,9 +254,7 @@ void NetworkFetchAndSetPresentRoot( auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>( [mirror, &err_messages](auto const& msg, bool /*fatal*/) { err_messages += fmt::format( - "While attempting fetch from local mirror {}:\n{}\n", - mirror, - msg); + "While attempting fetch from URL {}:\n{}\n", mirror, msg); }); if (git_repo->FetchViaTmpRepo(mirror, repo_info.branch, @@ -254,140 +269,6 @@ void NetworkFetchAndSetPresentRoot( remotes_buffer.append(fmt::format("\n> {}", mirror)); } if (not fetched) { - // get preferred hostnames list - auto preferred_hostnames = - MirrorsUtils::GetPreferredHostnames(additional_mirrors); - // try first the main URL, but with each of the preferred hostnames, if - // URL is not a path - if (not GitURLIsPath(fetch_repo)) { - for (auto const& hostname : preferred_hostnames) { - if (auto preferred_url = - CurlURLHandle::ReplaceHostname(fetch_repo, hostname)) { - auto wrapped_logger = - std::make_shared<AsyncMapConsumerLogger>( - [preferred_url, &err_messages](auto const& msg, - bool /*fatal*/) { - err_messages += fmt::format( - "While attempting fetch from remote " - "{}:\n{}\n", - *preferred_url, - msg); - }); - if (git_repo->FetchViaTmpRepo(*preferred_url, - repo_info.branch, - repo_info.inherit_env, - git_bin, - launcher, - wrapped_logger)) { - fetched = true; - break; - } - // add preferred to buffer - remotes_buffer.append( - fmt::format("\n> {}", *preferred_url)); - } - else { - // report failed hostname - remotes_buffer.append( - fmt::format("\n> {} (failed hostname replace: {})", - fetch_repo, - hostname)); - } - } - } - if (not fetched) { - // now try the original main fetch URL - auto wrapped_logger = std::make_shared<AsyncMapConsumerLogger>( - [fetch_repo, &err_messages](auto const& msg, bool /*fatal*/) { - err_messages += fmt::format( - "While attempting fetch from remote {}:\n{}\n", - fetch_repo, - msg); - }); - if (git_repo->FetchViaTmpRepo(fetch_repo, - repo_info.branch, - repo_info.inherit_env, - git_bin, - launcher, - wrapped_logger)) { - fetched = true; - } - else { - // add main fetch URL to buffer - remotes_buffer.append(fmt::format("\n> {}", fetch_repo)); - // now try to fetch from mirrors, in order, if given - for (auto mirror : repo_info.mirrors) { - auto mirror_path = GitURLIsPath(mirror); - if (mirror_path) { - mirror = - std::filesystem::absolute(*mirror_path).string(); - } - else { - // if non-path, try each of the preferred hostnames - for (auto const& hostname : preferred_hostnames) { - if (auto preferred_mirror = - CurlURLHandle::ReplaceHostname(mirror, - hostname)) { - wrapped_logger = - std::make_shared<AsyncMapConsumerLogger>( - [preferred_mirror, &err_messages]( - auto const& msg, bool /*fatal*/) { - err_messages += fmt::format( - "While attempting fetch from " - "mirror {}:\n{}\n", - *preferred_mirror, - msg); - }); - if (git_repo->FetchViaTmpRepo( - *preferred_mirror, - repo_info.branch, - repo_info.inherit_env, - git_bin, - launcher, - wrapped_logger)) { - fetched = true; - break; - } - // add preferred mirror to buffer - remotes_buffer.append( - fmt::format("\n> {}", *preferred_mirror)); - } - else { - // report failed hostname - remotes_buffer.append(fmt::format( - "\n> {} (failed hostname replace: {})", - mirror, - hostname)); - } - } - } - if (fetched) { - break; - } - wrapped_logger = std::make_shared<AsyncMapConsumerLogger>( - [mirror, &err_messages](auto const& msg, - bool /*fatal*/) { - err_messages += fmt::format( - "While attempting fetch from mirror {}:\n{}\n", - mirror, - msg); - }); - if (git_repo->FetchViaTmpRepo(mirror, - repo_info.branch, - repo_info.inherit_env, - git_bin, - launcher, - wrapped_logger)) { - fetched = true; - break; - } - // add mirror to buffer - remotes_buffer.append(fmt::format("\n> {}", mirror)); - } - } - } - } - if (not fetched) { // log fetch failure and list the remotes tried (*logger)( fmt::format("While trying to fetch from provided remotes:{}Fetch " diff --git a/src/other_tools/utils/content.hpp b/src/other_tools/utils/content.hpp index 5f593225..b1308a97 100644 --- a/src/other_tools/utils/content.hpp +++ b/src/other_tools/utils/content.hpp @@ -53,79 +53,33 @@ -> std::variant<std::string, std::string> { // keep all remotes tried, to report in case fetch fails std::string remotes_buffer{}; - // first, try the local mirrors std::optional<std::string> data{std::nullopt}; + + // try repo url + auto all_mirrors = std::vector<std::string>({fetch_url}); + // try repo mirrors afterwards + all_mirrors.insert(all_mirrors.end(), mirrors.begin(), mirrors.end()); + + if (auto preferred_hostnames = + MirrorsUtils::GetPreferredHostnames(additional_mirrors); + not preferred_hostnames.empty()) { + all_mirrors = + MirrorsUtils::SortByHostname(all_mirrors, preferred_hostnames); + } + + // always try local mirrors first auto local_mirrors = MirrorsUtils::GetLocalMirrors(additional_mirrors, fetch_url); - for (auto const& mirror : local_mirrors) { + all_mirrors.insert( + all_mirrors.begin(), local_mirrors.begin(), local_mirrors.end()); + + for (auto const& mirror : all_mirrors) { if (data = NetworkFetch(mirror, ca_info); data) { break; } // add local mirror to buffer remotes_buffer.append(fmt::format("\n> {}", mirror)); } - if (not data) { - // get preferred hostnames list - auto preferred_hostnames = - MirrorsUtils::GetPreferredHostnames(additional_mirrors); - // try the main fetch URL, but with each of the preferred hostnames - for (auto const& hostname : preferred_hostnames) { - if (auto preferred_url = - CurlURLHandle::ReplaceHostname(fetch_url, hostname)) { - if (data = NetworkFetch(*preferred_url, ca_info); data) { - break; - } - // add preferred URL to buffer - remotes_buffer.append(fmt::format("\n> {}", *preferred_url)); - } - else { - // report failed hostname - remotes_buffer.append( - fmt::format("\n> {} (failed hostname replace: {})", - fetch_url, - hostname)); - } - } - if (not data) { - // now try the main fetch URL - if (data = NetworkFetch(fetch_url, ca_info); not data) { - // add main fetch URL to buffer - remotes_buffer.append(fmt::format("\n> {}", fetch_url)); - // try the mirrors, in order, if given - for (auto const& mirror : mirrors) { - // first use with preferred hostnames... - for (auto const& hostname : preferred_hostnames) { - if (auto preferred_mirror = - CurlURLHandle::ReplaceHostname(mirror, - hostname)) { - if (data = NetworkFetch(*preferred_mirror, ca_info); - data) { - break; - } - // add preferred mirror to buffer - remotes_buffer.append( - fmt::format("\n> {}", *preferred_mirror)); - } - else { - // report failed hostname - remotes_buffer.append(fmt::format( - "\n> {} (failed hostname replace: {})", - mirror, - hostname)); - } - } - // ...then the original mirror - if (not data) { - if (data = NetworkFetch(mirror, ca_info); data) { - break; - } - // add mirror to buffer - remotes_buffer.append(fmt::format("\n> {}", mirror)); - } - } - } - } - } return data ? std::variant<std::string, std::string>(std::in_place_index<1>, *data) : std::variant<std::string, std::string>(std::in_place_index<0>, diff --git a/test/end-to-end/just-mr/just-mr-mirrors.test.sh b/test/end-to-end/just-mr/just-mr-mirrors.test.sh index beb8f73c..2e8e7d2e 100644 --- a/test/end-to-end/just-mr/just-mr-mirrors.test.sh +++ b/test/end-to-end/just-mr/just-mr-mirrors.test.sh @@ -234,6 +234,10 @@ cat > just-local.json <<EOF [ "http://non-existent.example.org/zip_repo.zip" , "http://127.0.0.1:${port_num}/zip_repo.zip" ] + , "http://non-existent.example.org:${port_num}/tgz_repo.tar.gz": + [ "http://non-existent.example.org/tgz_repo.tar.gz" + , "http://127.0.0.1:${port_num}/tgz_repo.tar.gz" + ] } , "preferred hostnames": ["non-existent.example.org", "127.0.0.1"] |