summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/other_tools/just_mr/fetch.cpp1
-rw-r--r--src/other_tools/just_mr/setup.cpp2
-rw-r--r--src/other_tools/ops_maps/TARGETS1
-rw-r--r--src/other_tools/ops_maps/content_cas_map.cpp31
-rw-r--r--src/other_tools/ops_maps/content_cas_map.hpp2
-rw-r--r--src/other_tools/root_maps/TARGETS1
-rw-r--r--src/other_tools/root_maps/content_git_map.cpp23
-rw-r--r--src/other_tools/root_maps/content_git_map.hpp2
-rw-r--r--src/other_tools/utils/TARGETS2
-rw-r--r--src/other_tools/utils/content.hpp64
10 files changed, 97 insertions, 32 deletions
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index bf9cf796..70322d98 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -277,6 +277,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
// create async maps
auto content_cas_map =
CreateContentCASMap(common_args.just_mr_paths,
+ common_args.alternative_mirrors,
common_args.ca_info,
local_api ? &(*local_api) : nullptr,
remote_api ? &(*remote_api) : nullptr,
diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp
index 6d568bc8..5aab7095 100644
--- a/src/other_tools/just_mr/setup.cpp
+++ b/src/other_tools/just_mr/setup.cpp
@@ -104,6 +104,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
auto critical_git_op_map = CreateCriticalGitOpMap(crit_git_op_ptr);
auto content_cas_map =
CreateContentCASMap(common_args.just_mr_paths,
+ common_args.alternative_mirrors,
common_args.ca_info,
local_api ? &(*local_api) : nullptr,
remote_api ? &(*remote_api) : nullptr,
@@ -131,6 +132,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config,
CreateContentGitMap(&content_cas_map,
&import_to_git_map,
common_args.just_mr_paths,
+ common_args.alternative_mirrors,
common_args.ca_info,
&resolve_symlinks_map,
&critical_git_op_map,
diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS
index 1baf11e5..f30cdf22 100644
--- a/src/other_tools/ops_maps/TARGETS
+++ b/src/other_tools/ops_maps/TARGETS
@@ -61,6 +61,7 @@
, ["src/buildtool/file_system/symlinks_map", "pragma_special"]
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/multithreading", "async_map_consumer"]
+ , ["src/other_tools/just_mr", "mirrors"]
, ["src/utils/cpp", "hash_combine"]
, ["@", "json", "", "json"]
]
diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp
index b6b8712e..2deb8060 100644
--- a/src/other_tools/ops_maps/content_cas_map.cpp
+++ b/src/other_tools/ops_maps/content_cas_map.cpp
@@ -20,18 +20,23 @@
#include "src/other_tools/just_mr/progress_reporting/progress.hpp"
#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
#include "src/other_tools/utils/content.hpp"
+#include "src/other_tools/utils/curl_url_handle.hpp"
auto CreateContentCASMap(LocalPathsPtr const& just_mr_paths,
+ MirrorsPtr const& additional_mirrors,
CAInfoPtr const& ca_info,
IExecutionApi* local_api,
IExecutionApi* remote_api,
std::size_t jobs) -> ContentCASMap {
- auto ensure_in_cas = [just_mr_paths, ca_info, local_api, remote_api](
- auto /*unused*/,
- auto setter,
- auto logger,
- auto /*unused*/,
- auto const& key) {
+ auto ensure_in_cas = [just_mr_paths,
+ additional_mirrors,
+ ca_info,
+ local_api,
+ remote_api](auto /*unused*/,
+ auto setter,
+ auto logger,
+ auto /*unused*/,
+ auto const& key) {
// check if content already in CAS
auto const& cas = Storage::Instance().CAS();
auto digest = ArtifactDigest(key.content, 0, false);
@@ -69,17 +74,9 @@ auto CreateContentCASMap(LocalPathsPtr const& just_mr_paths,
/*fatal=*/true);
return;
}
- // now do the actual fetch; first, try the main fetch URL
- auto data = NetworkFetch(key.fetch_url, ca_info);
- if (not data) {
- // try the mirrors, in order, if given
- for (auto const& mirror : key.mirrors) {
- data = NetworkFetch(mirror, ca_info);
- if (data) {
- break;
- }
- }
- }
+ // now do the actual fetch
+ auto data = NetworkFetchWithMirrors(
+ key.fetch_url, key.mirrors, ca_info, additional_mirrors);
if (not data) {
(*logger)(fmt::format("Failed to fetch a file with id {} from "
"provided remotes",
diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp
index 16b790d0..810727cd 100644
--- a/src/other_tools/ops_maps/content_cas_map.hpp
+++ b/src/other_tools/ops_maps/content_cas_map.hpp
@@ -24,6 +24,7 @@
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/file_system/symlinks_map/pragma_special.hpp"
#include "src/buildtool/multithreading/async_map_consumer.hpp"
+#include "src/other_tools/just_mr/mirrors.hpp"
#include "src/utils/cpp/hash_combine.hpp"
struct ArchiveContent {
@@ -65,6 +66,7 @@ struct ArchiveRepoInfo {
using ContentCASMap = AsyncMapConsumer<ArchiveContent, bool>;
[[nodiscard]] auto CreateContentCASMap(LocalPathsPtr const& just_mr_paths,
+ MirrorsPtr const& additional_mirrors,
CAInfoPtr const& ca_info,
IExecutionApi* local_api,
IExecutionApi* remote_api,
diff --git a/src/other_tools/root_maps/TARGETS b/src/other_tools/root_maps/TARGETS
index fff48505..9340cd14 100644
--- a/src/other_tools/root_maps/TARGETS
+++ b/src/other_tools/root_maps/TARGETS
@@ -86,6 +86,7 @@
[ ["src/buildtool/serve_api/remote", "serve_api"]
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/common", "user_structs"]
+ , ["src/other_tools/just_mr", "mirrors"]
, ["src/other_tools/ops_maps", "content_cas_map"]
, ["src/other_tools/ops_maps", "import_to_git_map"]
, ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"]
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp
index a39b6cde..79500f5d 100644
--- a/src/other_tools/root_maps/content_git_map.cpp
+++ b/src/other_tools/root_maps/content_git_map.cpp
@@ -302,6 +302,7 @@ auto CreateContentGitMap(
gsl::not_null<ContentCASMap*> const& content_cas_map,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
LocalPathsPtr const& just_mr_paths,
+ MirrorsPtr const& additional_mirrors,
CAInfoPtr const& ca_info,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
@@ -315,6 +316,7 @@ auto CreateContentGitMap(
resolve_symlinks_map,
critical_git_op_map,
just_mr_paths,
+ additional_mirrors,
ca_info,
serve_api,
local_api,
@@ -505,6 +507,7 @@ auto CreateContentGitMap(
sha256 = key.archive.sha256,
sha512 = key.archive.sha512,
pragma_special = key.pragma_special,
+ additional_mirrors,
ca_info,
origin = key.archive.origin,
absent = key.absent,
@@ -708,20 +711,12 @@ auto CreateContentGitMap(
/*fatal=*/true);
return;
}
- // now do the actual fetch; first, try the
- // main fetch URL
- auto data =
- NetworkFetch(fetch_url, ca_info);
- if (not data) {
- // try the mirrors, in order, if given
- for (auto const& mirror : mirrors) {
- data =
- NetworkFetch(mirror, ca_info);
- if (data) {
- break;
- }
- }
- }
+ // now do the actual fetch
+ auto data = NetworkFetchWithMirrors(
+ fetch_url,
+ mirrors,
+ ca_info,
+ additional_mirrors);
if (not data) {
(*logger)(
fmt::format("Failed to fetch a "
diff --git a/src/other_tools/root_maps/content_git_map.hpp b/src/other_tools/root_maps/content_git_map.hpp
index e0a564f5..986a0531 100644
--- a/src/other_tools/root_maps/content_git_map.hpp
+++ b/src/other_tools/root_maps/content_git_map.hpp
@@ -21,6 +21,7 @@
#include "src/buildtool/execution_api/common/execution_api.hpp"
#include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp"
#include "src/buildtool/serve_api/remote/serve_api.hpp"
+#include "src/other_tools/just_mr/mirrors.hpp"
#include "src/other_tools/ops_maps/content_cas_map.hpp"
#include "src/other_tools/ops_maps/import_to_git_map.hpp"
@@ -33,6 +34,7 @@ using ContentGitMap =
gsl::not_null<ContentCASMap*> const& content_cas_map,
gsl::not_null<ImportToGitMap*> const& import_to_git_map,
LocalPathsPtr const& just_mr_paths,
+ MirrorsPtr const& additional_mirrors,
CAInfoPtr const& ca_info,
gsl::not_null<ResolveSymlinksMap*> const& resolve_symlinks_map,
gsl::not_null<CriticalGitOpMap*> const& critical_git_op_map,
diff --git a/src/other_tools/utils/TARGETS b/src/other_tools/utils/TARGETS
index 2bbf1b9f..aad0fec6 100644
--- a/src/other_tools/utils/TARGETS
+++ b/src/other_tools/utils/TARGETS
@@ -38,9 +38,11 @@
, "hdrs": ["content.hpp"]
, "deps":
[ "curl_easy_handle"
+ , "curl_url_handle"
, ["src/buildtool/common", "user_structs"]
, ["src/buildtool/crypto", "hasher"]
, ["src/buildtool/logging", "log_level"]
+ , ["src/other_tools/just_mr", "mirrors"]
]
, "stage": ["src", "other_tools", "utils"]
}
diff --git a/src/other_tools/utils/content.hpp b/src/other_tools/utils/content.hpp
index 907af852..19877263 100644
--- a/src/other_tools/utils/content.hpp
+++ b/src/other_tools/utils/content.hpp
@@ -21,12 +21,14 @@
#include "src/buildtool/common/user_structs.hpp"
#include "src/buildtool/crypto/hasher.hpp"
#include "src/buildtool/logging/log_level.hpp"
+#include "src/other_tools/just_mr/mirrors.hpp"
#include "src/other_tools/utils/curl_easy_handle.hpp"
+#include "src/other_tools/utils/curl_url_handle.hpp"
// Utilities related to the content of an archive
/// \brief Fetches a file from the internet and stores its content in memory.
-/// Returns the content.
+/// \returns the content.
[[nodiscard]] static auto NetworkFetch(std::string const& fetch_url,
CAInfoPtr const& ca_info) noexcept
-> std::optional<std::string> {
@@ -38,6 +40,66 @@
return curl_handle->DownloadToString(fetch_url);
}
+/// \brief Fetches a file from the internet and stores its content in memory.
+/// Tries not only a given remote, but also all associated remote locations.
+/// \returns the content.
+[[nodiscard]] static auto NetworkFetchWithMirrors(
+ std::string const& fetch_url,
+ std::vector<std::string> const& mirrors,
+ CAInfoPtr const& ca_info,
+ MirrorsPtr const& additional_mirrors) noexcept
+ -> std::optional<std::string> {
+ // first, try the local mirrors
+ std::optional<std::string> data{std::nullopt};
+ auto local_mirrors =
+ MirrorsUtils::GetLocalMirrors(additional_mirrors, fetch_url);
+ for (auto const& mirror : local_mirrors) {
+ if (data = NetworkFetch(mirror, ca_info); data) {
+ break;
+ }
+ }
+ if (not data) {
+ // get preferred hostnames list
+ auto preferred_hostnames =
+ MirrorsUtils::GetPreferredHostnames(additional_mirrors);
+ // try the main fetch URL, but with each of the preferred hostnames
+ for (auto const& hostname : preferred_hostnames) {
+ if (auto preferred_url =
+ CurlURLHandle::ReplaceHostname(fetch_url, hostname)) {
+ if (data = NetworkFetch(*preferred_url, ca_info); data) {
+ break;
+ }
+ }
+ }
+ if (not data) {
+ // now try the main fetch URL
+ if (data = NetworkFetch(fetch_url, ca_info); not data) {
+ // try the mirrors, in order, if given
+ for (auto const& mirror : mirrors) {
+ // first use with preferred hostnames...
+ for (auto const& hostname : preferred_hostnames) {
+ if (auto preferred_mirror =
+ CurlURLHandle::ReplaceHostname(mirror,
+ hostname)) {
+ if (data = NetworkFetch(*preferred_mirror, ca_info);
+ data) {
+ break;
+ }
+ }
+ }
+ // ...then the original mirror
+ if (not data) {
+ if (data = NetworkFetch(mirror, ca_info); data) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ return data;
+}
+
template <Hasher::HashType type>
[[nodiscard]] static auto GetContentHash(std::string const& data) noexcept
-> std::string {
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 07:29:45 +0000