just-mr: Add 'mirrors' field to archive-like repositories

Also extends 'distdir' repositories logic accordingly.

6 files changed, 126 insertions, 45 deletions

diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp
index 04b3d036..b6b8712e 100644
--- a/src/other_tools/ops_maps/content_cas_map.cpp
+++ b/src/other_tools/ops_maps/content_cas_map.cpp
@@ -69,12 +69,21 @@ auto CreateContentCASMap(LocalPathsPtr const& just_mr_paths,
                       /*fatal=*/true);
             return;
         }
-        // now do the actual fetch
+        // now do the actual fetch; first, try the main fetch URL
         auto data = NetworkFetch(key.fetch_url, ca_info);
-        if (data == std::nullopt) {
-            (*logger)(fmt::format("Failed to fetch a file with id {} from {}",
-                                  key.content,
-                                  key.fetch_url),
+        if (not data) {
+            // try the mirrors, in order, if given
+            for (auto const& mirror : key.mirrors) {
+                data = NetworkFetch(mirror, ca_info);
+                if (data) {
+                    break;
+                }
+            }
+        }
+        if (not data) {
+            (*logger)(fmt::format("Failed to fetch a file with id {} from "
+                                  "provided remotes",
+                                  key.content),
                       /*fatal=*/true);
             return;
         }
diff --git a/src/other_tools/ops_maps/content_cas_map.hpp b/src/other_tools/ops_maps/content_cas_map.hpp
index 846b2762..16b790d0 100644
--- a/src/other_tools/ops_maps/content_cas_map.hpp
+++ b/src/other_tools/ops_maps/content_cas_map.hpp
@@ -17,6 +17,7 @@
 
 #include <optional>
 #include <string>
+#include <vector>
 
 #include "nlohmann/json.hpp"
 #include "src/buildtool/common/user_structs.hpp"
@@ -26,15 +27,16 @@
 #include "src/utils/cpp/hash_combine.hpp"
 
 struct ArchiveContent {
-    std::string content; /* key */
-    std::optional<std::string> distfile;
-    std::string fetch_url;
-    std::optional<std::string> sha256;
-    std::optional<std::string> sha512;
+    std::string content{}; /* key */
+    std::optional<std::string> distfile{std::nullopt};
+    std::string fetch_url{};
+    std::vector<std::string> mirrors{};
+    std::optional<std::string> sha256{std::nullopt};
+    std::optional<std::string> sha512{std::nullopt};
     // name of repository for which work is done; used in progress reporting
-    std::string origin;
+    std::string origin{};
     // flag deciding whether progress reporting is needed for key
-    bool origin_from_distdir;
+    bool origin_from_distdir{};
 
     [[nodiscard]] auto operator==(const ArchiveContent& other) const -> bool {
         return content == other.content;
@@ -43,9 +45,9 @@ struct ArchiveContent {
 
 // Used in callers of ContentCASMap which need extra fields
 struct ArchiveRepoInfo {
-    ArchiveContent archive; /* key */
-    std::string repo_type;  /* key */
-    std::string subdir;     /* key */
+    ArchiveContent archive{}; /* key */
+    std::string repo_type{};  /* key */
+    std::string subdir{};     /* key */
     // create root based on "special" pragma value
     std::optional<PragmaSpecial> pragma_special{std::nullopt}; /* key */
     // create an absent root
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index 4c1b80ad..9644d615 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -213,6 +213,29 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
     auto repo_desc_distfile = repo_desc->Get("distfile", Expression::none_t{});
     auto repo_desc_sha256 = repo_desc->Get("sha256", Expression::none_t{});
     auto repo_desc_sha512 = repo_desc->Get("sha512", Expression::none_t{});
+    // check optional mirrors
+    auto repo_desc_mirrors = repo_desc->Get("mirrors", Expression::list_t{});
+    std::vector<std::string> mirrors{};
+    if (repo_desc_mirrors->IsList()) {
+        mirrors.reserve(repo_desc_mirrors->List().size());
+        for (auto const& elem : repo_desc_mirrors->List()) {
+            if (not elem->IsString()) {
+                (*logger)(fmt::format("ArchiveCheckout: Unsupported list entry "
+                                      "{} in optional field \"mirrors\"",
+                                      elem->ToString()),
+                          /*fatal=*/true);
+                return;
+            }
+            mirrors.emplace_back(elem->String());
+        }
+    }
+    else {
+        (*logger)(fmt::format("ArchiveCheckout: Optional field \"mirrors\" "
+                              "should be a list of strings, but found: {}",
+                              repo_desc_mirrors->ToString()),
+                  /*fatal=*/true);
+        return;
+    }
     // check "special" pragma
     auto repo_desc_pragma = repo_desc->At("pragma");
     auto pragma_special = repo_desc_pragma
@@ -238,6 +261,7 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
                              ? std::make_optional(repo_desc_distfile->String())
                              : std::nullopt,
              .fetch_url = repo_desc_fetch->get()->String(),
+             .mirrors = std::move(mirrors),
              .sha256 = repo_desc_sha256->IsString()
                            ? std::make_optional(repo_desc_sha256->String())
                            : std::nullopt,
@@ -511,6 +535,36 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
             auto repo_desc_sha512 =
                 (*resolved_repo_desc)->Get("sha512", Expression::none_t{});
 
+            // check optional mirrors
+            auto repo_desc_mirrors =
+                (*resolved_repo_desc)->Get("mirrors", Expression::list_t{});
+            std::vector<std::string> mirrors{};
+            if (repo_desc_mirrors->IsList()) {
+                mirrors.reserve(repo_desc_mirrors->List().size());
+                for (auto const& elem : repo_desc_mirrors->List()) {
+                    if (not elem->IsString()) {
+                        (*logger)(fmt::format(
+                                      "DistdirCheckout: Unsupported list entry "
+                                      "{} in optional field \"mirrors\" for "
+                                      "repository {}",
+                                      elem->ToString(),
+                                      nlohmann::json(dist_repo_name).dump()),
+                                  /*fatal=*/true);
+                        return;
+                    }
+                    mirrors.emplace_back(elem->String());
+                }
+            }
+            else {
+                (*logger)(fmt::format("DistdirCheckout: Optional field "
+                                      "\"mirrors\" for repository {} should be "
+                                      "a list of strings, but found: {}",
+                                      nlohmann::json(dist_repo_name).dump(),
+                                      repo_desc_mirrors->ToString()),
+                          /*fatal=*/true);
+                return;
+            }
+
             ArchiveContent archive = {
                 .content = repo_desc_content->get()->String(),
                 .distfile =
@@ -518,6 +572,7 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
                         ? std::make_optional(repo_desc_distfile->String())
                         : std::nullopt,
                 .fetch_url = repo_desc_fetch->get()->String(),
+                .mirrors = std::move(mirrors),
                 .sha256 = repo_desc_sha256->IsString()
                               ? std::make_optional(repo_desc_sha256->String())
                               : std::nullopt,
diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp
index bc9f55a2..a39b6cde 100644
--- a/src/other_tools/root_maps/content_git_map.cpp
+++ b/src/other_tools/root_maps/content_git_map.cpp
@@ -501,6 +501,7 @@ auto CreateContentGitMap(
                                  archive_tree_id_file,
                                  content = key.archive.content,
                                  fetch_url = key.archive.fetch_url,
+                                 mirrors = key.archive.mirrors,
                                  sha256 = key.archive.sha256,
                                  sha512 = key.archive.sha512,
                                  pragma_special = key.pragma_special,
@@ -707,16 +708,27 @@ auto CreateContentGitMap(
                                             /*fatal=*/true);
                                         return;
                                     }
-                                    // now do the actual fetch
+                                    // now do the actual fetch; first, try the
+                                    // main fetch URL
                                     auto data =
                                         NetworkFetch(fetch_url, ca_info);
-                                    if (data == std::nullopt) {
-                                        (*logger)(fmt::format(
-                                                      "Failed to fetch a file "
-                                                      "with id {} from {}",
-                                                      content,
-                                                      fetch_url),
-                                                  /*fatal=*/true);
+                                    if (not data) {
+                                        // try the mirrors, in order, if given
+                                        for (auto const& mirror : mirrors) {
+                                            data =
+                                                NetworkFetch(mirror, ca_info);
+                                            if (data) {
+                                                break;
+                                            }
+                                        }
+                                    }
+                                    if (not data) {
+                                        (*logger)(
+                                            fmt::format("Failed to fetch a "
+                                                        "file with content {} "
+                                                        "from provided remotes",
+                                                        content),
+                                            /*fatal=*/true);
                                         return;
                                     }
                                     // check content wrt checksums
@@ -800,8 +812,8 @@ auto CreateContentGitMap(
                                               key.archive.content),
                                   /*fatal=*/false);
                     }
-                    // if not fetching absent, request the resolved subdir tree
-                    // directly
+                    // if not fetching absent, request the resolved subdir
+                    // tree directly
                     else {
                         if (auto tree_id = serve_api->RetrieveTreeFromArchive(
                                 key.archive.content,
@@ -837,8 +849,8 @@ auto CreateContentGitMap(
                     }
                 }
                 // revert to network fetch and import_to_git
-                // before any network fetching, check that mandatory fields are
-                // provided
+                // before any network fetching, check that mandatory fields
+                // are provided
                 if (key.archive.fetch_url.empty()) {
                     (*logger)("Failed to provide archive fetch url!",
                               /*fatal=*/true);
@@ -847,11 +859,11 @@ auto CreateContentGitMap(
                 // now do the actual fetch
                 auto data = NetworkFetch(key.archive.fetch_url, ca_info);
                 if (data == std::nullopt) {
-                    (*logger)(
-                        fmt::format("Failed to fetch a file with id {} from {}",
-                                    key.archive.content,
-                                    key.archive.fetch_url),
-                        /*fatal=*/true);
+                    (*logger)(fmt::format("Failed to fetch a file with id "
+                                          "{} from {}",
+                                          key.archive.content,
+                                          key.archive.fetch_url),
+                              /*fatal=*/true);
                     return;
                 }
                 // check content wrt checksums
@@ -885,11 +897,11 @@ auto CreateContentGitMap(
                 auto path = StorageUtils::AddToCAS(*data);
                 // check one last time if content is in CAS now
                 if (not path) {
-                    (*logger)(
-                        fmt::format("Failed to fetch a file with id {} from {}",
-                                    key.archive.content,
-                                    key.archive.fetch_url),
-                        /*fatal=*/true);
+                    (*logger)(fmt::format("Failed to fetch a file with id "
+                                          "{} from {}",
+                                          key.archive.content,
+                                          key.archive.fetch_url),
+                              /*fatal=*/true);
                     return;
                 }
                 JustMRProgress::Instance().TaskTracker().Stop(
@@ -910,7 +922,8 @@ auto CreateContentGitMap(
                 // done
                 return;
             }
-            // if not marked absent, do regular fetch to CAS and import_to_git
+            // if not marked absent, do regular fetch to CAS and
+            // import_to_git
             content_cas_map->ConsumeAfterKeysReady(
                 ts,
                 {key.archive},
@@ -948,11 +961,11 @@ auto CreateContentGitMap(
                 },
                 [logger, content = key.archive.content](auto const& msg,
                                                         bool fatal) {
-                    (*logger)(
-                        fmt::format("While ensuring content {} is in CAS:\n{}",
-                                    content,
-                                    msg),
-                        fatal);
+                    (*logger)(fmt::format("While ensuring content {} is in "
+                                          "CAS:\n{}",
+                                          content,
+                                          msg),
+                              fatal);
                 });
         }
     };
diff --git a/src/other_tools/utils/TARGETS b/src/other_tools/utils/TARGETS
index 65772ae8..2bbf1b9f 100644
--- a/src/other_tools/utils/TARGETS
+++ b/src/other_tools/utils/TARGETS
@@ -40,6 +40,7 @@
     [ "curl_easy_handle"
     , ["src/buildtool/common", "user_structs"]
     , ["src/buildtool/crypto", "hasher"]
+    , ["src/buildtool/logging", "log_level"]
     ]
   , "stage": ["src", "other_tools", "utils"]
   }
diff --git a/src/other_tools/utils/content.hpp b/src/other_tools/utils/content.hpp
index e7cb7501..907af852 100644
--- a/src/other_tools/utils/content.hpp
+++ b/src/other_tools/utils/content.hpp
@@ -20,6 +20,7 @@
 
 #include "src/buildtool/common/user_structs.hpp"
 #include "src/buildtool/crypto/hasher.hpp"
+#include "src/buildtool/logging/log_level.hpp"
 #include "src/other_tools/utils/curl_easy_handle.hpp"
 
 // Utilities related to the content of an archive
@@ -29,8 +30,8 @@
 [[nodiscard]] static auto NetworkFetch(std::string const& fetch_url,
                                        CAInfoPtr const& ca_info) noexcept
     -> std::optional<std::string> {
-    auto curl_handle =
-        CurlEasyHandle::Create(ca_info->no_ssl_verify, ca_info->ca_bundle);
+    auto curl_handle = CurlEasyHandle::Create(
+        ca_info->no_ssl_verify, ca_info->ca_bundle, LogLevel::Debug);
     if (not curl_handle) {
         return std::nullopt;
     }