summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/other_tools/just_mr/TARGETS1
-rw-r--r--src/other_tools/just_mr/fetch.cpp131
-rw-r--r--src/other_tools/repo_map/TARGETS1
-rw-r--r--src/other_tools/repo_map/repos_to_setup_map.cpp100
-rw-r--r--src/other_tools/utils/TARGETS11
-rw-r--r--src/other_tools/utils/parse_archive.cpp133
-rw-r--r--src/other_tools/utils/parse_archive.hpp33
7 files changed, 204 insertions, 206 deletions
diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS
index 4b9cf071..b31efa8d 100644
--- a/src/other_tools/just_mr/TARGETS
+++ b/src/other_tools/just_mr/TARGETS
@@ -113,6 +113,7 @@
, ["src/other_tools/ops_maps", "critical_git_op_map"]
, ["src/other_tools/ops_maps", "git_tree_fetch_map"]
, ["src/other_tools/ops_maps", "import_to_git_map"]
+ , ["src/other_tools/utils", "parse_archive"]
, "setup_utils"
, ["src/buildtool/execution_api/common", "common"]
, ["src/buildtool/execution_api/local", "local"]
diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp
index 81709f05..8779a7bd 100644
--- a/src/other_tools/just_mr/fetch.cpp
+++ b/src/other_tools/just_mr/fetch.cpp
@@ -32,6 +32,7 @@
#include "src/other_tools/ops_maps/critical_git_op_map.hpp"
#include "src/other_tools/ops_maps/git_tree_fetch_map.hpp"
#include "src/other_tools/ops_maps/import_to_git_map.hpp"
+#include "src/other_tools/utils/parse_archive.hpp"
auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
MultiRepoCommonArguments const& common_args,
@@ -196,119 +197,27 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config,
// only do work if repo is archive or git tree type
switch (kCheckoutTypeMap.at(repo_type_str)) {
case CheckoutType::Archive: {
- // check "absent" pragma
- auto repo_desc_pragma = (*resolved_repo_desc)->At("pragma");
- auto pragma_absent =
- (repo_desc_pragma and repo_desc_pragma->get()->IsMap())
- ? repo_desc_pragma->get()->At("absent")
- : std::nullopt;
- auto pragma_absent_value =
- pragma_absent and pragma_absent->get()->IsBool() and
- pragma_absent->get()->Bool();
- // only fetch if either archive is not marked absent, or if
- // explicitly told to fetch absent archives
- if (not pragma_absent_value or common_args.fetch_absent) {
- // check mandatory fields
- auto repo_desc_content =
- (*resolved_repo_desc)->At("content");
- if (not repo_desc_content) {
- Logger::Log(LogLevel::Error,
- "Config: Mandatory field \"content\" "
- "is missing");
- return kExitFetchError;
- }
- if (not repo_desc_content->get()->IsString()) {
- Logger::Log(LogLevel::Error,
- "Config: Unsupported value {} for "
- "mandatory field \"content\"",
- repo_desc_content->get()->ToString());
- return kExitFetchError;
- }
- auto repo_desc_fetch =
- (*resolved_repo_desc)->At("fetch");
- if (not repo_desc_fetch) {
+ auto logger = std::make_shared<AsyncMapConsumerLogger>(
+ [&repo_name](std::string const& msg, bool fatal) {
Logger::Log(
- LogLevel::Error,
- "Config: Mandatory field \"fetch\" is missing");
- return kExitFetchError;
- }
- if (not repo_desc_fetch->get()->IsString()) {
- Logger::Log(LogLevel::Error,
- "Config: Unsupported value {} for "
- "mandatory field \"fetch\"",
- repo_desc_fetch->get()->ToString());
- return kExitFetchError;
- }
- auto repo_desc_subdir =
- (*resolved_repo_desc)
- ->Get("subdir", Expression::none_t{});
- auto subdir = std::filesystem::path(
- repo_desc_subdir->IsString()
- ? repo_desc_subdir->String()
- : "")
- .lexically_normal();
- auto repo_desc_distfile =
- (*resolved_repo_desc)
- ->Get("distfile", Expression::none_t{});
- auto repo_desc_sha256 =
- (*resolved_repo_desc)
- ->Get("sha256", Expression::none_t{});
- auto repo_desc_sha512 =
- (*resolved_repo_desc)
- ->Get("sha512", Expression::none_t{});
- auto repo_desc_mirrors =
- (*resolved_repo_desc)
- ->Get("mirrors", Expression::list_t{});
- std::vector<std::string> mirrors{};
- if (repo_desc_mirrors->IsList()) {
- mirrors.reserve(repo_desc_mirrors->List().size());
- for (auto const& elem : repo_desc_mirrors->List()) {
- if (not elem->IsString()) {
- Logger::Log(
- LogLevel::Error,
- "Config: Unsupported list entry {} in "
- "optional field \"mirrors\"",
- elem->ToString());
- return kExitFetchError;
- }
- mirrors.emplace_back(elem->String());
- }
- }
- else {
- Logger::Log(LogLevel::Error,
- "Config: Optional field \"mirrors\" "
- "should be a list of strings, but "
- "found: {}",
- repo_desc_mirrors->ToString());
- return kExitFetchError;
- }
+ fatal ? LogLevel::Error : LogLevel::Warning,
+ "While parsing description of repository "
+ "{}:\n{}",
+ nlohmann::json(repo_name).dump(),
+ msg);
+ });
- ArchiveRepoInfo archive_info = {
- .archive =
- {.content = repo_desc_content->get()->String(),
- .distfile =
- repo_desc_distfile->IsString()
- ? std::make_optional(
- repo_desc_distfile->String())
- : std::nullopt,
- .fetch_url = repo_desc_fetch->get()->String(),
- .mirrors = std::move(mirrors),
- .sha256 = repo_desc_sha256->IsString()
- ? std::make_optional(
- repo_desc_sha256->String())
- : std::nullopt,
- .sha512 = repo_desc_sha512->IsString()
- ? std::make_optional(
- repo_desc_sha512->String())
- : std::nullopt,
- .origin = repo_name},
- .repo_type = repo_type_str,
- .subdir = subdir.empty() ? "." : subdir.string(),
- .pragma_special = std::nullopt, // not used
- .absent = false // not used
- };
- // add to list
- archives_to_fetch.emplace_back(std::move(archive_info));
+ auto archive_repo_info = ParseArchiveDescription(
+ *resolved_repo_desc, repo_type_str, repo_name, logger);
+ if (not archive_repo_info) {
+ return kExitFetchError;
+ }
+ // only fetch if either archive is not marked absent, or if
+ // explicitly told to fetch absent archives
+ if (not archive_repo_info->absent or
+ common_args.fetch_absent) {
+ archives_to_fetch.emplace_back(
+ std::move(*archive_repo_info));
}
} break;
case CheckoutType::GitTree: {
diff --git a/src/other_tools/repo_map/TARGETS b/src/other_tools/repo_map/TARGETS
index 5872c802..94a6094a 100644
--- a/src/other_tools/repo_map/TARGETS
+++ b/src/other_tools/repo_map/TARGETS
@@ -22,6 +22,7 @@
, ["src/buildtool/multithreading", "task_system"]
, ["src/other_tools/ops_maps", "content_cas_map"]
, ["src/other_tools/ops_maps", "git_tree_fetch_map"]
+ , ["src/other_tools/utils", "parse_archive"]
]
}
}
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index 7d0762c8..f2b28685 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -22,6 +22,7 @@
#include "src/other_tools/just_mr/progress_reporting/statistics.hpp"
#include "src/other_tools/ops_maps/content_cas_map.hpp"
#include "src/other_tools/ops_maps/git_tree_fetch_map.hpp"
+#include "src/other_tools/utils/parse_archive.hpp"
namespace {
@@ -206,106 +207,15 @@ void ArchiveCheckout(ExpressionPtr const& repo_desc,
gsl::not_null<TaskSystem*> const& ts,
ReposToSetupMap::SetterPtr const& setter,
ReposToSetupMap::LoggerPtr const& logger) {
- // enforce mandatory fields
- auto repo_desc_content = repo_desc->At("content");
- if (not repo_desc_content) {
- (*logger)("ArchiveCheckout: Mandatory field \"content\" is missing",
- /*fatal=*/true);
- return;
- }
- if (not repo_desc_content->get()->IsString()) {
- (*logger)(fmt::format("ArchiveCheckout: Unsupported value {} for "
- "mandatory field \"content\"",
- repo_desc_content->get()->ToString()),
- /*fatal=*/true);
- return;
- }
- auto repo_desc_fetch = repo_desc->At("fetch");
- if (not repo_desc_fetch) {
- (*logger)("ArchiveCheckout: Mandatory field \"fetch\" is missing",
- /*fatal=*/true);
- return;
- }
- if (not repo_desc_fetch->get()->IsString()) {
- (*logger)(fmt::format("ArchiveCheckout: Unsupported value {} for "
- "mandatory field \"fetch\"",
- repo_desc_fetch->get()->ToString()),
- /*fatal=*/true);
+ auto archive_repo_info =
+ ParseArchiveDescription(repo_desc, repo_type, repo_name, logger);
+ if (not archive_repo_info) {
return;
}
- auto repo_desc_subdir = repo_desc->Get("subdir", Expression::none_t{});
- auto subdir = std::filesystem::path(repo_desc_subdir->IsString()
- ? repo_desc_subdir->String()
- : "")
- .lexically_normal();
- auto repo_desc_distfile = repo_desc->Get("distfile", Expression::none_t{});
- auto repo_desc_sha256 = repo_desc->Get("sha256", Expression::none_t{});
- auto repo_desc_sha512 = repo_desc->Get("sha512", Expression::none_t{});
- // check optional mirrors
- auto repo_desc_mirrors = repo_desc->Get("mirrors", Expression::list_t{});
- std::vector<std::string> mirrors{};
- if (repo_desc_mirrors->IsList()) {
- mirrors.reserve(repo_desc_mirrors->List().size());
- for (auto const& elem : repo_desc_mirrors->List()) {
- if (not elem->IsString()) {
- (*logger)(fmt::format("ArchiveCheckout: Unsupported list entry "
- "{} in optional field \"mirrors\"",
- elem->ToString()),
- /*fatal=*/true);
- return;
- }
- mirrors.emplace_back(elem->String());
- }
- }
- else {
- (*logger)(fmt::format("ArchiveCheckout: Optional field \"mirrors\" "
- "should be a list of strings, but found: {}",
- repo_desc_mirrors->ToString()),
- /*fatal=*/true);
- return;
- }
- // check "special" pragma
- auto repo_desc_pragma = repo_desc->At("pragma");
- bool const& pragma_is_map =
- repo_desc_pragma and repo_desc_pragma->get()->IsMap();
- auto pragma_special =
- pragma_is_map ? repo_desc_pragma->get()->At("special") : std::nullopt;
- auto pragma_special_value =
- pragma_special and pragma_special->get()->IsString() and
- kPragmaSpecialMap.contains(pragma_special->get()->String())
- ? std::make_optional(
- kPragmaSpecialMap.at(pragma_special->get()->String()))
- : std::nullopt;
- // check "absent" pragma
- auto pragma_absent =
- pragma_is_map ? repo_desc_pragma->get()->At("absent") : std::nullopt;
- auto pragma_absent_value = pragma_absent and
- pragma_absent->get()->IsBool() and
- pragma_absent->get()->Bool();
- // populate struct
- ArchiveRepoInfo archive_repo_info = {
- .archive =
- {.content = repo_desc_content->get()->String(),
- .distfile = repo_desc_distfile->IsString()
- ? std::make_optional(repo_desc_distfile->String())
- : std::nullopt,
- .fetch_url = repo_desc_fetch->get()->String(),
- .mirrors = std::move(mirrors),
- .sha256 = repo_desc_sha256->IsString()
- ? std::make_optional(repo_desc_sha256->String())
- : std::nullopt,
- .sha512 = repo_desc_sha512->IsString()
- ? std::make_optional(repo_desc_sha512->String())
- : std::nullopt,
- .origin = repo_name},
- .repo_type = repo_type,
- .subdir = subdir.empty() ? "." : subdir.string(),
- .pragma_special = pragma_special_value,
- .absent = pragma_absent_value};
// get the WS root as git tree
content_git_map->ConsumeAfterKeysReady(
ts,
- {std::move(archive_repo_info)},
+ {std::move(*archive_repo_info)},
[repos = std::move(repos), repo_name, setter](auto const& values) {
auto ws_root = values[0]->first;
nlohmann::json cfg({});
diff --git a/src/other_tools/utils/TARGETS b/src/other_tools/utils/TARGETS
index aad0fec6..27c4a6af 100644
--- a/src/other_tools/utils/TARGETS
+++ b/src/other_tools/utils/TARGETS
@@ -46,4 +46,15 @@
]
, "stage": ["src", "other_tools", "utils"]
}
+, "parse_archive":
+ { "type": ["@", "rules", "CC", "library"]
+ , "name": ["parse_archive"]
+ , "hdrs": ["parse_archive.hpp"]
+ , "srcs": ["parse_archive.cpp"]
+ , "deps":
+ [ ["src/buildtool/build_engine/expression", "expression"]
+ , ["src/other_tools/ops_maps", "content_cas_map"]
+ ]
+ , "stage": ["src", "other_tools", "utils"]
+ }
}
diff --git a/src/other_tools/utils/parse_archive.cpp b/src/other_tools/utils/parse_archive.cpp
new file mode 100644
index 00000000..ff5d20bf
--- /dev/null
+++ b/src/other_tools/utils/parse_archive.cpp
@@ -0,0 +1,133 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/other_tools/utils/parse_archive.hpp"
+
+namespace {
+auto ParseArchiveContent(ExpressionPtr const& repo_desc,
+ std::string const& origin,
+ const AsyncMapConsumerLoggerPtr& logger)
+ -> std::optional<ArchiveContent> {
+
+ // enforce mandatory fields
+ auto repo_desc_content = repo_desc->At("content");
+ if (not repo_desc_content) {
+ (*logger)("ArchiveCheckout: Mandatory field \"content\" is missing",
+ /*fatal=*/true);
+ return std::nullopt;
+ }
+ if (not repo_desc_content->get()->IsString()) {
+ (*logger)(fmt::format("ArchiveCheckout: Unsupported value {} for "
+ "mandatory field \"content\"",
+ repo_desc_content->get()->ToString()),
+ /*fatal=*/true);
+ return std::nullopt;
+ }
+ auto repo_desc_fetch = repo_desc->At("fetch");
+ if (not repo_desc_fetch) {
+ (*logger)("ArchiveCheckout: Mandatory field \"fetch\" is missing",
+ /*fatal=*/true);
+ return std::nullopt;
+ }
+ if (not repo_desc_fetch->get()->IsString()) {
+ (*logger)(fmt::format("ArchiveCheckout: Unsupported value {} for "
+ "mandatory field \"fetch\"",
+ repo_desc_fetch->get()->ToString()),
+ /*fatal=*/true);
+ return std::nullopt;
+ }
+ auto repo_desc_distfile = repo_desc->Get("distfile", Expression::none_t{});
+ auto repo_desc_sha256 = repo_desc->Get("sha256", Expression::none_t{});
+ auto repo_desc_sha512 = repo_desc->Get("sha512", Expression::none_t{});
+ // check optional mirrors
+ auto repo_desc_mirrors = repo_desc->Get("mirrors", Expression::list_t{});
+ std::vector<std::string> mirrors{};
+ if (repo_desc_mirrors->IsList()) {
+ mirrors.reserve(repo_desc_mirrors->List().size());
+ for (auto const& elem : repo_desc_mirrors->List()) {
+ if (not elem->IsString()) {
+ (*logger)(fmt::format("ArchiveCheckout: Unsupported list entry "
+ "{} in optional field \"mirrors\"",
+ elem->ToString()),
+ /*fatal=*/true);
+ return std::nullopt;
+ }
+ mirrors.emplace_back(elem->String());
+ }
+ }
+ else {
+ (*logger)(fmt::format("ArchiveCheckout: Optional field \"mirrors\" "
+ "should be a list of strings, but found: {}",
+ repo_desc_mirrors->ToString()),
+ /*fatal=*/true);
+ return std::nullopt;
+ }
+
+ return ArchiveContent{
+ .content = repo_desc_content->get()->String(),
+ .distfile = repo_desc_distfile->IsString()
+ ? std::make_optional(repo_desc_distfile->String())
+ : std::nullopt,
+ .fetch_url = repo_desc_fetch->get()->String(),
+ .mirrors = std::move(mirrors),
+ .sha256 = repo_desc_sha256->IsString()
+ ? std::make_optional(repo_desc_sha256->String())
+ : std::nullopt,
+ .sha512 = repo_desc_sha512->IsString()
+ ? std::make_optional(repo_desc_sha512->String())
+ : std::nullopt,
+ .origin = origin};
+}
+} // namespace
+
+auto ParseArchiveDescription(ExpressionPtr const& repo_desc,
+ std::string const& repo_type,
+ std::string const& origin,
+ const AsyncMapConsumerLoggerPtr& logger)
+ -> std::optional<ArchiveRepoInfo> {
+ auto archive_content = ParseArchiveContent(repo_desc, origin, logger);
+ if (not archive_content) {
+ return std::nullopt;
+ }
+ // additional mandatory fields
+ auto repo_desc_subdir = repo_desc->Get("subdir", Expression::none_t{});
+ auto subdir = std::filesystem::path(repo_desc_subdir->IsString()
+ ? repo_desc_subdir->String()
+ : "")
+ .lexically_normal();
+
+ // check "special" pragma
+ auto repo_desc_pragma = repo_desc->At("pragma");
+ bool const& pragma_is_map =
+ repo_desc_pragma and repo_desc_pragma->get()->IsMap();
+ auto pragma_special =
+ pragma_is_map ? repo_desc_pragma->get()->At("special") : std::nullopt;
+ auto pragma_special_value =
+ pragma_special and pragma_special->get()->IsString() and
+ kPragmaSpecialMap.contains(pragma_special->get()->String())
+ ? std::make_optional(
+ kPragmaSpecialMap.at(pragma_special->get()->String()))
+ : std::nullopt;
+ // check "absent" pragma
+ auto pragma_absent =
+ pragma_is_map ? repo_desc_pragma->get()->At("absent") : std::nullopt;
+ auto pragma_absent_value = pragma_absent and
+ pragma_absent->get()->IsBool() and
+ pragma_absent->get()->Bool();
+ return ArchiveRepoInfo{.archive = *archive_content,
+ .repo_type = repo_type,
+ .subdir = subdir.empty() ? "." : subdir.string(),
+ .pragma_special = pragma_special_value,
+ .absent = pragma_absent_value};
+}
diff --git a/src/other_tools/utils/parse_archive.hpp b/src/other_tools/utils/parse_archive.hpp
new file mode 100644
index 00000000..91f3e280
--- /dev/null
+++ b/src/other_tools/utils/parse_archive.hpp
@@ -0,0 +1,33 @@
+// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_ARCHIVE_HPP
+#define INCLUDED_SRC_OTHER_TOOLS_UTILS_PARSE_ARCHIVE_HPP
+
+#include <optional>
+#include <string>
+
+#include "src/buildtool/build_engine/expression/expression.hpp"
+#include "src/other_tools/ops_maps/content_cas_map.hpp"
+
+// Parse the description of an archive repository; if an error
+// occurs, call the logger with fatal set to true and return std::nullopt
+// instead.
+auto ParseArchiveDescription(ExpressionPtr const& repo_desc,
+ std::string const& repo_type,
+ std::string const& origin,
+ const AsyncMapConsumerLoggerPtr& logger)
+ -> std::optional<ArchiveRepoInfo>;
+
+#endif
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 17:01:03 +0000