diff options
| -rw-r--r-- | src/buildtool/file_system/git_repo.cpp | 222 | ||||
| -rw-r--r-- | src/buildtool/file_system/git_repo.hpp | 32 | ||||
| -rw-r--r-- | src/other_tools/git_operations/git_repo_remote.cpp | 8 | ||||
| -rw-r--r-- | src/other_tools/git_operations/git_repo_remote.hpp | 4 | ||||
| -rw-r--r-- | test/buildtool/file_system/git_repo.test.cpp | 89 |
5 files changed, 342 insertions, 13 deletions
diff --git a/src/buildtool/file_system/git_repo.cpp b/src/buildtool/file_system/git_repo.cpp index 1b9eaf1f..d30f6614 100644 --- a/src/buildtool/file_system/git_repo.cpp +++ b/src/buildtool/file_system/git_repo.cpp @@ -287,6 +287,56 @@ void backend_free(git_odb_backend* /*_backend*/) {} // A backend that can be used to read and create tree objects in-memory. auto const kInMemoryODBParent = CreateInMemoryODBParent(); +struct FetchIntoODBBackend { + git_odb_backend parent; + git_odb* target_odb; // the odb where the fetch objects will end up into +}; + +[[nodiscard]] auto fetch_backend_writepack(git_odb_writepack** _writepack, + git_odb_backend* _backend, + [[maybe_unused]] git_odb* odb, + git_indexer_progress_cb progress_cb, + void* progress_payload) -> int { + if (_backend != nullptr) { + auto* b = reinterpret_cast<FetchIntoODBBackend*>(_backend); // NOLINT + return git_odb_write_pack( + _writepack, b->target_odb, progress_cb, progress_payload); + } + return GIT_ERROR; +} + +[[nodiscard]] auto fetch_backend_exists(git_odb_backend* _backend, + const git_oid* oid) -> int { + if (_backend != nullptr) { + auto* b = reinterpret_cast<FetchIntoODBBackend*>(_backend); // NOLINT + return git_odb_exists(b->target_odb, oid); + } + return GIT_ERROR; +} + +void fetch_backend_free(git_odb_backend* /*_backend*/) {} + +[[nodiscard]] auto CreateFetchIntoODBParent() -> git_odb_backend { + git_odb_backend b{}; + b.version = GIT_ODB_BACKEND_VERSION; + // only populate the functions needed + b.writepack = &fetch_backend_writepack; // needed for fetch + b.exists = &fetch_backend_exists; + b.free = &fetch_backend_free; + return b; +} + +// A backend that can be used to fetch from the remote of another repository. +auto const kFetchIntoODBParent = CreateFetchIntoODBParent(); + +// callback to remote fetch without an SSL certificate check +const auto certificate_passthrough_cb = [](git_cert* /*cert*/, + int /*valid*/, + const char* /*host*/, + void* /*payload*/) -> int { + return 0; +}; + } // namespace #endif // BOOTSTRAP_BUILD_TOOL @@ -784,6 +834,101 @@ auto GitRepo::GetHeadCommit(anon_logger_ptr const& logger) noexcept #endif // BOOTSTRAP_BUILD_TOOL } +auto GitRepo::FetchFromPath(std::shared_ptr<git_config> cfg, + std::string const& repo_path, + std::optional<std::string> const& branch, + anon_logger_ptr const& logger) noexcept -> bool { +#ifdef BOOTSTRAP_BUILD_TOOL + return false; +#else + try { + // only possible for real repository! + if (IsRepoFake()) { + (*logger)("Cannot fetch commit using a fake repository!", + true /*fatal*/); + return false; + } + // create remote from repo + git_remote* remote_ptr{nullptr}; + if (git_remote_create_anonymous( + &remote_ptr, GetRepoRef()->Ptr(), repo_path.c_str()) != 0) { + (*logger)(fmt::format("Creating remote {} for local repository {} " + "failed with:\n{}", + repo_path, + GetGitPath().string(), + GitLastError()), + true /*fatal*/); + // cleanup resources + git_remote_free(remote_ptr); + return false; + } + // wrap remote object + auto remote = std::unique_ptr<git_remote, decltype(&remote_closer)>( + remote_ptr, remote_closer); + // get the canonical url + auto canonical_url = std::string(git_remote_url(remote.get())); + + // get a well-defined config file + if (not cfg) { + // get config snapshot of current repo + git_config* cfg_ptr{nullptr}; + if (git_repository_config_snapshot(&cfg_ptr, GetRepoRef()->Ptr()) != + 0) { + (*logger)(fmt::format("Retrieving config object in fetch from " + "path failed with:\n{}", + GitLastError()), + true /*fatal*/); + return false; + } + // share pointer with caller + cfg.reset(cfg_ptr, config_closer); + } + + // define default fetch options + git_fetch_options fetch_opts{}; + git_fetch_options_init(&fetch_opts, GIT_FETCH_OPTIONS_VERSION); + // no proxy + fetch_opts.proxy_opts.type = GIT_PROXY_NONE; + // no SSL verification + fetch_opts.callbacks.certificate_check = certificate_passthrough_cb; + // disable update of the FETCH_HEAD pointer + fetch_opts.update_fetchhead = 0; + + // setup fetch refspecs array + git_strarray refspecs_array_obj{}; + if (branch) { + // make sure we check for tags as well + std::string tag = fmt::format("+refs/tags/{}", *branch); + std::string head = fmt::format("+refs/heads/{}", *branch); + PopulateStrarray(&refspecs_array_obj, {tag, head}); + } + auto refspecs_array = + std::unique_ptr<git_strarray, decltype(&strarray_deleter)>( + &refspecs_array_obj, strarray_deleter); + + if (git_remote_fetch( + remote.get(), refspecs_array.get(), &fetch_opts, nullptr) != + 0) { + (*logger)(fmt::format( + "Fetching {} in local repository {} failed with:\n{}", + branch ? fmt::format("branch {}", *branch) : "all", + GetGitPath().string(), + GitLastError()), + true /*fatal*/); + return false; + } + return true; // success! + } catch (std::exception const& ex) { + Logger::Log(LogLevel::Error, + "Fetch {} from local repository {} failed with:\n{}", + branch ? fmt::format("branch {}", *branch) : "all", + repo_path, + ex.what()); + return false; + } +#endif // BOOTSTRAP_BUILD_TOOL +} + auto GitRepo::GetSubtreeFromCommit(std::string const& commit, std::string const& subdir, anon_logger_ptr const& logger) noexcept @@ -1341,6 +1486,83 @@ auto GitRepo::GetObjectByPathFromTree(std::string const& tree_id, #endif // BOOTSTRAP_BUILD_TOOL } +auto GitRepo::LocalFetchViaTmpRepo(std::filesystem::path const& tmp_dir, + std::string const& repo_path, + std::optional<std::string> const& branch, + anon_logger_ptr const& logger) noexcept + -> bool { +#ifdef BOOTSTRAP_BUILD_TOOL + return false; +#else + try { + // preferably with a "fake" repository! + if (not IsRepoFake()) { + Logger::Log(LogLevel::Debug, + "Branch local fetch called on a real repository"); + } + // create the temporary real repository + // it can be bare, as the refspecs for this fetch will be given + // explicitly. + auto tmp_repo = GitRepo::InitAndOpen(tmp_dir, /*is_bare=*/true); + if (tmp_repo == std::nullopt) { + return false; + } + // add backend, with max priority + FetchIntoODBBackend b{.parent = kFetchIntoODBParent, + .target_odb = GetGitOdb().get()}; + if (git_odb_add_backend( + tmp_repo->GetGitOdb().get(), + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + reinterpret_cast<git_odb_backend*>(&b), + std::numeric_limits<int>::max()) == 0) { + // setup wrapped logger + auto wrapped_logger = std::make_shared<anon_logger_t>( + [logger](auto const& msg, bool fatal) { + (*logger)(fmt::format("While doing branch local fetch via " + "tmp repo:\n{}", + msg), + fatal); + }); + // get the config of the correct target repo + auto cfg = GetConfigSnapshot(); + if (cfg == nullptr) { + (*logger)(fmt::format("Retrieving config object in local fetch " + "via tmp repo failed with:\n{}", + GitLastError()), + true /*fatal*/); + return false; + } + return tmp_repo->FetchFromPath( + cfg, repo_path, branch, wrapped_logger); + } + (*logger)(fmt::format( + "Adding custom backend for local fetch failed with:\n{}", + GitLastError()), + true /*fatal*/); + return false; + } catch (std::exception const& ex) { + Logger::Log( + LogLevel::Error, + "Fetch {} from local repository {} via tmp dir failed with:\n{}", + branch ? fmt::format("branch {}", *branch) : "all", + repo_path, + ex.what()); + return false; + } +#endif // BOOTSTRAP_BUILD_TOOL +} + +auto GitRepo::GetConfigSnapshot() const noexcept + -> std::shared_ptr<git_config> { +#ifndef BOOTSTRAP_BUILD_TOOL + git_config* cfg_ptr{nullptr}; + if (git_repository_config_snapshot(&cfg_ptr, GetRepoRef()->Ptr()) == 0) { + return std::shared_ptr<git_config>(cfg_ptr, config_closer); + } +#endif // BOOTSTRAP_BUILD_TOOL + return nullptr; +} + auto GitRepo::IsRepoFake() const noexcept -> bool { return is_repo_fake_; } diff --git a/src/buildtool/file_system/git_repo.hpp b/src/buildtool/file_system/git_repo.hpp index 7314289a..9c31f21b 100644 --- a/src/buildtool/file_system/git_repo.hpp +++ b/src/buildtool/file_system/git_repo.hpp @@ -22,6 +22,7 @@ extern "C" { struct git_repository; +struct git_config; } /// \brief Git repository logic. @@ -162,6 +163,19 @@ class GitRepo { [[nodiscard]] auto GetHeadCommit(anon_logger_ptr const& logger) noexcept -> std::optional<std::string>; + /// \brief Fetch from given local repository. It can either fetch a given + /// named branch, or it can fetch with base refspecs. + /// Only possible with real repository and thus non-thread-safe. + /// If non-null, use given config snapshot to interact with config entries; + /// otherwise, use a snapshot from the current repo and share pointer to it. + /// Returns a success flag. It guarantees the logger is called exactly once + /// with fatal if failure. + [[nodiscard]] auto FetchFromPath(std::shared_ptr<git_config> cfg, + std::string const& repo_path, + std::optional<std::string> const& branch, + anon_logger_ptr const& logger) noexcept + -> bool; + /// \brief Get the tree id of a subtree given the root commit /// Calling it from a fake repository allows thread-safe use. /// Returns the subtree hash, as a string, or nullopt if failure. @@ -238,6 +252,24 @@ class GitRepo { std::string const& tree_id, std::string const& rel_path) noexcept -> std::optional<TreeEntryInfo>; + /// \brief Fetch from given local repository via a temporary location. Uses + /// tmp dir to fetch asynchronously using libgit2. + /// Caller needs to make sure the temporary directory exists and that the + /// given path is thread- and process-safe! + /// Uses either a given branch, or fetches all (with base refspecs). + /// Returns a success flag. + /// It guarantees the logger is called exactly once with fatal if failure. + [[nodiscard]] auto LocalFetchViaTmpRepo( + std::filesystem::path const& tmp_dir, + std::string const& repo_path, + std::optional<std::string> const& branch, + anon_logger_ptr const& logger) noexcept -> bool; + + /// \brief Get a snapshot of the repository configuration. + /// Returns nullptr on errors. + [[nodiscard]] auto GetConfigSnapshot() const noexcept + -> std::shared_ptr<git_config>; + private: /// \brief Wrapped git_repository with guarded destructor. /// Kept privately nested to avoid misuse of its raw pointer members. diff --git a/src/other_tools/git_operations/git_repo_remote.cpp b/src/other_tools/git_operations/git_repo_remote.cpp index 580375ec..9739362a 100644 --- a/src/other_tools/git_operations/git_repo_remote.cpp +++ b/src/other_tools/git_operations/git_repo_remote.cpp @@ -619,11 +619,3 @@ auto GitRepoRemote::FetchViaTmpRepo(std::filesystem::path const& tmp_dir, return false; } } - -auto GitRepoRemote::GetConfigSnapshot() const -> std::shared_ptr<git_config> { - git_config* cfg_ptr{nullptr}; - if (git_repository_config_snapshot(&cfg_ptr, GetRepoRef()->Ptr()) != 0) { - return nullptr; - } - return std::shared_ptr<git_config>(cfg_ptr, config_closer); -} diff --git a/src/other_tools/git_operations/git_repo_remote.hpp b/src/other_tools/git_operations/git_repo_remote.hpp index cfe1736d..ac4f59aa 100644 --- a/src/other_tools/git_operations/git_repo_remote.hpp +++ b/src/other_tools/git_operations/git_repo_remote.hpp @@ -110,10 +110,6 @@ class GitRepoRemote : public GitRepo { anon_logger_ptr const& logger) noexcept -> bool; - /// \brief Get a snapshot of the repository configuration. - /// Returns nullptr on errors. - [[nodiscard]] auto GetConfigSnapshot() const -> std::shared_ptr<git_config>; - private: /// \brief Open "fake" repository wrapper for existing CAS. explicit GitRepoRemote(GitCASPtr git_cas) noexcept; diff --git a/test/buildtool/file_system/git_repo.test.cpp b/test/buildtool/file_system/git_repo.test.cpp index 4d8c5b4a..ac346ffb 100644 --- a/test/buildtool/file_system/git_repo.test.cpp +++ b/test/buildtool/file_system/git_repo.test.cpp @@ -205,6 +205,29 @@ TEST_CASE("Single-threaded real repository local operations", "[git_repo]") { REQUIRE(head_commit); CHECK(*head_commit == kRootCommit); } + + SECTION("Fetch with base refspecs from path") { + // make bare real repo to fetch into + auto path_fetch_all = TestUtils::CreateTestRepoWithCheckout(); + REQUIRE(path_fetch_all); + auto repo_fetch_all = GitRepo::Open(*path_fetch_all); + + // fetch all + CHECK(repo_fetch_all->FetchFromPath( + nullptr, *path_fetch_all, std::nullopt, logger)); + } + + SECTION("Fetch branch from path") { + // make bare real repo to fetch into + auto path_fetch_branch = TestUtils::CreateTestRepoWithCheckout(); + REQUIRE(path_fetch_branch); + auto repo_fetch_branch = GitRepo::Open(*path_fetch_branch); + REQUIRE(repo_fetch_branch); + + // fetch branch + CHECK(repo_fetch_branch->FetchFromPath( + nullptr, *path_fetch_branch, "master", logger)); + } } TEST_CASE("Single-threaded fake repository operations", "[git_repo]") { @@ -369,6 +392,52 @@ TEST_CASE("Single-threaded fake repository operations", "[git_repo]") { CHECK(*obj_info->symlink_content == "bar"); } } + + SECTION("Fetch from local repository via temporary repository") { + SECTION("Fetch all") { + // set repo to fetch into + auto path_fetch_all = TestUtils::GetRepoPath(); + auto repo_fetch_all = + GitRepo::InitAndOpen(path_fetch_all, /*is_bare=*/true); + REQUIRE(repo_fetch_all); + + // check commit is not there before fetch + CHECK_FALSE( + *repo_fetch_all->CheckCommitExists(kRootCommit, logger)); + + // create tmp dir to use for fetch + auto tmp_path_fetch_all = TestUtils::GetRepoPath(); + REQUIRE(FileSystemManager::CreateDirectory(tmp_path_fetch_all)); + // fetch all with base refspecs + REQUIRE(repo_fetch_all->LocalFetchViaTmpRepo( + tmp_path_fetch_all, *repo_path, std::nullopt, logger)); + + // check commit is there after fetch + CHECK(*repo_fetch_all->CheckCommitExists(kRootCommit, logger)); + } + + SECTION("Fetch branch") { + // set repo to fetch into + auto path_fetch_branch = TestUtils::GetRepoPath(); + auto repo_fetch_branch = + GitRepo::InitAndOpen(path_fetch_branch, /*is_bare=*/true); + REQUIRE(repo_fetch_branch); + + // check commit is not there before fetch + CHECK_FALSE( + *repo_fetch_branch->CheckCommitExists(kRootCommit, logger)); + + // create tmp dir to use for fetch + auto tmp_path_fetch_branch = TestUtils::GetRepoPath(); + REQUIRE(FileSystemManager::CreateDirectory(tmp_path_fetch_branch)); + // fetch branch + REQUIRE(repo_fetch_branch->LocalFetchViaTmpRepo( + tmp_path_fetch_branch, *repo_path, "master", logger)); + + // check commit is there after fetch + CHECK(*repo_fetch_branch->CheckCommitExists(kRootCommit, logger)); + } + } } TEST_CASE("Multi-threaded fake repository operations", "[git_repo]") { @@ -397,7 +466,7 @@ TEST_CASE("Multi-threaded fake repository operations", "[git_repo]") { threads.reserve(kNumThreads); SECTION("Lookups in the same ODB") { - constexpr int NUM_CASES = 5; + constexpr int NUM_CASES = 6; for (int id{}; id < kNumThreads; ++id) { threads.emplace_back( [&remote_cas, &remote_repo_path, &logger, &starting_signal]( @@ -457,6 +526,24 @@ TEST_CASE("Multi-threaded fake repository operations", "[git_repo]") { logger); CHECK(*result_containing); } break; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) + case 5: { + auto remote_repo = GitRepo::Open(remote_cas); + REQUIRE(remote_repo); + REQUIRE(remote_repo->IsRepoFake()); + // set up tmp dir + // create tmp dir to use for fetch + auto tmp_path_fetch_branch = + TestUtils::GetRepoPath(); + REQUIRE(FileSystemManager::CreateDirectory( + tmp_path_fetch_branch)); + // fetch all + REQUIRE(remote_repo->LocalFetchViaTmpRepo( + tmp_path_fetch_branch, + *remote_repo_path, + std::nullopt, + logger)); + } break; } }, id); |