diff options
Diffstat (limited to 'src/other_tools/git_operations')
| -rw-r--r-- | src/other_tools/git_operations/TARGETS | 2 | ||||
| -rw-r--r-- | src/other_tools/git_operations/git_repo_remote.cpp | 146 | ||||
| -rw-r--r-- | src/other_tools/git_operations/git_repo_remote.hpp | 22 |
3 files changed, 120 insertions, 50 deletions
diff --git a/src/other_tools/git_operations/TARGETS b/src/other_tools/git_operations/TARGETS index 19dabe4f..8a135ecf 100644 --- a/src/other_tools/git_operations/TARGETS +++ b/src/other_tools/git_operations/TARGETS @@ -32,6 +32,8 @@ , ["src/buildtool/file_system", "git_utils"] , ["@", "fmt", "", "fmt"] , ["", "libgit2"] + , ["@", "json", "", "json"] + , ["src/buildtool/system", "system_command"] , "git_config_settings" ] } diff --git a/src/other_tools/git_operations/git_repo_remote.cpp b/src/other_tools/git_operations/git_repo_remote.cpp index 4965f121..6cff2a82 100644 --- a/src/other_tools/git_operations/git_repo_remote.cpp +++ b/src/other_tools/git_operations/git_repo_remote.cpp @@ -15,8 +15,10 @@ #include <src/other_tools/git_operations/git_repo_remote.hpp> #include "fmt/core.h" +#include "nlohmann/json.hpp" #include "src/buildtool/file_system/git_utils.hpp" #include "src/buildtool/logging/logger.hpp" +#include "src/buildtool/system/system_command.hpp" #include "src/other_tools/git_operations/git_config_settings.hpp" extern "C" { @@ -26,6 +28,21 @@ extern "C" { namespace { +/// \brief Basic check for libgit2 protocols we support. For all other cases, we +/// should default to shell out to git instead. +[[nodiscard]] auto IsSupported(std::string const& url) -> bool { + // look for explicit schemes + if (url.starts_with("git://") or url.starts_with("http://") or + url.starts_with("https://") or url.starts_with("file://")) { + return true; + } + // look for url as existing filesystem directory path + if (FileSystemManager::IsDirectory(std::filesystem::path(url))) { + return true; + } + return false; // as default +} + struct FetchIntoODBBackend { git_odb_backend parent; git_odb* target_odb; // the odb where the fetch objects will end up into @@ -414,56 +431,105 @@ auto GitRepoRemote::UpdateCommitViaTmpRepo( } } -auto GitRepoRemote::FetchViaTmpRepo(std::filesystem::path const& tmp_repo_path, +auto GitRepoRemote::FetchViaTmpRepo(std::filesystem::path const& tmp_dir, std::string const& repo_url, std::optional<std::string> const& branch, + std::vector<std::string> const& launcher, anon_logger_ptr const& logger) noexcept -> bool { try { - // preferably with a "fake" repository! - if (not IsRepoFake()) { - Logger::Log(LogLevel::Debug, - "Branch fetch called on a real repository"); - } - // create the temporary real repository - // it can be bare, as the refspecs for this fetch will be given - // explicitly. - auto tmp_repo = - GitRepoRemote::InitAndOpen(tmp_repo_path, /*is_bare=*/true); - if (tmp_repo == std::nullopt) { + // check for internally supported protocols + if (IsSupported(repo_url)) { + // preferably with a "fake" repository! + if (not IsRepoFake()) { + Logger::Log(LogLevel::Debug, + "Branch fetch called on a real repository"); + } + // create the temporary real repository + // it can be bare, as the refspecs for this fetch will be given + // explicitly. + auto tmp_repo = + GitRepoRemote::InitAndOpen(tmp_dir, /*is_bare=*/true); + if (tmp_repo == std::nullopt) { + return false; + } + // add backend, with max priority + FetchIntoODBBackend b{kFetchIntoODBParent, GetGitOdb().get()}; + if (git_odb_add_backend( + tmp_repo->GetGitOdb().get(), + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast<git_odb_backend*>(&b), + std::numeric_limits<int>::max()) == 0) { + // setup wrapped logger + auto wrapped_logger = std::make_shared<anon_logger_t>( + [logger](auto const& msg, bool fatal) { + (*logger)(fmt::format("While doing branch fetch via " + "tmp repo:\n{}", + msg), + fatal); + }); + // get the config of the correct target repo + auto cfg = GetConfigSnapshot(); + if (cfg == nullptr) { + (*logger)( + fmt::format("retrieving config object in fetch via " + "tmp repo failed with:\n{}", + GitLastError()), + true /*fatal*/); + return false; + } + return tmp_repo->FetchFromRemote( + cfg, repo_url, branch, wrapped_logger); + } return false; } - // add backend, with max priority - FetchIntoODBBackend b{kFetchIntoODBParent, GetGitOdb().get()}; - if (git_odb_add_backend( - tmp_repo->GetGitOdb().get(), - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - reinterpret_cast<git_odb_backend*>(&b), - std::numeric_limits<int>::max()) == 0) { - // setup wrapped logger - auto wrapped_logger = std::make_shared<anon_logger_t>( - [logger](auto const& msg, bool fatal) { - (*logger)( - fmt::format( - "While doing branch fetch via tmp repo:\n{}", msg), - fatal); - }); - // get the config of the correct target repo - auto cfg = GetConfigSnapshot(); - if (cfg == nullptr) { - (*logger)(fmt::format("retrieving config object in fetch via " - "tmp repo failed with:\n{}", - GitLastError()), - true /*fatal*/); - return false; + // default to shelling out to git for non-explicitly supported protocols + auto cmdline = launcher; + // Note: Because we fetch with URL, not a known remote, no refs are + // being updated by default, so git has no reason to get a lock + // file. This however does not imply automatically that fetches + // might not internally wait for each other through other means. + cmdline.insert(cmdline.end(), + {"git", + "fetch", + "--no-auto-gc", + "--no-write-fetch-head", + repo_url}); + if (branch) { + cmdline.push_back(*branch); + } + // run command + SystemCommand system{repo_url}; + auto const command_output = system.Execute(cmdline, + {}, // caller env + GetGitPath(), + tmp_dir); + if (not command_output) { + std::string out_str{}; + std::string err_str{}; + auto cmd_out = + FileSystemManager::ReadFile(command_output->stdout_file); + auto cmd_err = + FileSystemManager::ReadFile(command_output->stderr_file); + if (cmd_out) { + out_str = *cmd_out; + } + if (cmd_err) { + err_str = *cmd_err; + } + std::string output{}; + if (!out_str.empty() || !err_str.empty()) { + output = fmt::format(" with output:\n{}{}", out_str, err_str); } - return tmp_repo->FetchFromRemote( - cfg, repo_url, branch, wrapped_logger); + (*logger)(fmt::format("Fetch command {} failed{}", + nlohmann::json(cmdline).dump(), + output), + /*fatal=*/true); + return false; } - return false; + return true; // fetch successful } catch (std::exception const& ex) { - Logger::Log( - LogLevel::Error, "fetch via tmp repo failed with:\n{}", ex.what()); + Logger::Log(LogLevel::Error, "Fetch failed with:\n{}", ex.what()); return false; } } diff --git a/src/other_tools/git_operations/git_repo_remote.hpp b/src/other_tools/git_operations/git_repo_remote.hpp index 8b0021b8..64c827c4 100644 --- a/src/other_tools/git_operations/git_repo_remote.hpp +++ b/src/other_tools/git_operations/git_repo_remote.hpp @@ -88,20 +88,22 @@ class GitRepoRemote : public GitRepo { anon_logger_ptr const& logger) const noexcept -> std::optional<std::string>; - /// \brief Fetch from a remote via a temporary repository. - /// Calling it from a fake repository allows thread-safe use. - /// Creates a temporary real repository at the given location and uses a - /// custom backend to redirect the fetched objects into the desired odb. + /// \brief Fetch from a remote. If URL is SSH, shells out to system git to + /// retrieve packs in a safe manner, with the only side-effect being that + /// there can be some redundancy in the fetched packs. The tmp dir is used + /// to pipe the stdout and stderr to. + /// If URL is non-SSH, uses tmp dir to fetch asynchronously using libgit2. /// Caller needs to make sure the temporary directory exists and that the /// given path is thread- and process-safe! - /// Uses either a given branch, or fetches using base refspecs. + /// Uses either a given branch, or fetches all (with base refspecs). /// Returns a success flag. /// It guarantees the logger is called exactly once with fatal if failure. - [[nodiscard]] auto FetchViaTmpRepo( - std::filesystem::path const& tmp_repo_path, - std::string const& repo_url, - std::optional<std::string> const& branch, - anon_logger_ptr const& logger) noexcept -> bool; + [[nodiscard]] auto FetchViaTmpRepo(std::filesystem::path const& tmp_dir, + std::string const& repo_url, + std::optional<std::string> const& branch, + std::vector<std::string> const& launcher, + anon_logger_ptr const& logger) noexcept + -> bool; /// \brief Get a snapshot of the repository configuration. /// Returns nullptr on errors. |