diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/buildtool/serve_api/serve_service/source_tree.cpp | 14 | ||||
-rw-r--r-- | src/buildtool/storage/TARGETS | 4 | ||||
-rw-r--r-- | src/buildtool/storage/fs_utils.cpp | 46 | ||||
-rw-r--r-- | src/buildtool/storage/fs_utils.hpp | 24 | ||||
-rw-r--r-- | src/other_tools/just_mr/TARGETS | 1 | ||||
-rw-r--r-- | src/other_tools/just_mr/setup.cpp | 3 | ||||
-rw-r--r-- | src/other_tools/ops_maps/TARGETS | 1 | ||||
-rw-r--r-- | src/other_tools/ops_maps/content_cas_map.cpp | 5 | ||||
-rw-r--r-- | src/other_tools/root_maps/commit_git_map.cpp | 7 | ||||
-rw-r--r-- | src/other_tools/root_maps/content_git_map.cpp | 10 | ||||
-rw-r--r-- | src/other_tools/root_maps/distdir_git_map.cpp | 4 | ||||
-rw-r--r-- | src/other_tools/root_maps/foreign_file_git_map.cpp | 87 | ||||
-rw-r--r-- | src/other_tools/root_maps/fpath_git_map.cpp | 4 |
13 files changed, 119 insertions, 91 deletions
diff --git a/src/buildtool/serve_api/serve_service/source_tree.cpp b/src/buildtool/serve_api/serve_service/source_tree.cpp index 8be074fc..75dab8c2 100644 --- a/src/buildtool/serve_api/serve_service/source_tree.cpp +++ b/src/buildtool/serve_api/serve_service/source_tree.cpp @@ -374,8 +374,8 @@ auto SourceTreeService::ResolveContentTree( ServeArchiveTreeResponse* response) -> ::grpc::Status { if (resolve_special) { // get the resolved tree - auto tree_id_file = - StorageUtils::GetResolvedTreeIDFile(tree_id, *resolve_special); + auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( + StorageConfig::Instance(), tree_id, *resolve_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -711,8 +711,8 @@ auto SourceTreeService::ServeArchiveTree( SymlinksResolveToPragmaSpecial(request->resolve_symlinks()); // check for archive_tree_id_file - auto archive_tree_id_file = - StorageUtils::GetArchiveTreeIDFile(archive_type, content); + auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( + StorageConfig::Instance(), archive_type, content); if (FileSystemManager::Exists(archive_tree_id_file)) { // read archive_tree_id from file tree_id_file auto archive_tree_id = @@ -792,7 +792,8 @@ auto SourceTreeService::ServeArchiveTree( StorageConfig::Instance().GitRoot(), content, logger_); if (res) { // add to CAS - content_cas_path = StorageUtils::AddToCAS(*res); + content_cas_path = + StorageUtils::AddToCAS(Storage::Instance(), *res); } if (res.error() == GitLookupError::Fatal) { logger_->Emit( @@ -810,7 +811,8 @@ auto SourceTreeService::ServeArchiveTree( auto res = GetBlobFromRepo(path, content, logger_); if (res) { // add to CAS - content_cas_path = StorageUtils::AddToCAS(*res); + content_cas_path = + StorageUtils::AddToCAS(Storage::Instance(), *res); if (content_cas_path) { break; } diff --git a/src/buildtool/storage/TARGETS b/src/buildtool/storage/TARGETS index c6e44f0f..2e2fc549 100644 --- a/src/buildtool/storage/TARGETS +++ b/src/buildtool/storage/TARGETS @@ -80,6 +80,8 @@ , "deps": [ ["src/buildtool/common", "user_structs"] , ["src/buildtool/file_system/symlinks_map", "pragma_special"] + , "config" + , "storage" ] , "stage": ["src", "buildtool", "storage"] , "private-deps": @@ -88,8 +90,6 @@ , ["src/buildtool/logging", "log_level"] , ["src/buildtool/logging", "logging"] , ["src/utils/cpp", "path"] - , "config" - , "storage" ] } , "file_chunker": diff --git a/src/buildtool/storage/fs_utils.cpp b/src/buildtool/storage/fs_utils.cpp index 539e0956..e50e892e 100644 --- a/src/buildtool/storage/fs_utils.cpp +++ b/src/buildtool/storage/fs_utils.cpp @@ -14,6 +14,7 @@ #include "src/buildtool/storage/fs_utils.hpp" +#include <tuple> //std::ignore #include <unordered_map> #include <utility> @@ -21,13 +22,12 @@ #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" -#include "src/buildtool/storage/config.hpp" -#include "src/buildtool/storage/storage.hpp" #include "src/utils/cpp/path.hpp" namespace StorageUtils { -auto GetGitRoot(LocalPathsPtr const& just_mr_paths, +auto GetGitRoot(StorageConfig const& storage_config, + LocalPathsPtr const& just_mr_paths, std::string const& repo_url) noexcept -> std::filesystem::path { if (just_mr_paths->git_checkout_locations.contains(repo_url)) { if (just_mr_paths->git_checkout_locations[repo_url].is_string()) { @@ -48,26 +48,29 @@ auto GetGitRoot(LocalPathsPtr const& just_mr_paths, FileSystemManager::IsDirectory(repo_url_as_path)) { return repo_url_as_path; } - return StorageConfig::Instance().GitRoot(); + return storage_config.GitRoot(); } -auto GetCommitTreeIDFile(std::string const& commit) noexcept +auto GetCommitTreeIDFile(StorageConfig const& storage_config, + std::string const& commit) noexcept -> std::filesystem::path { - return StorageConfig::Instance().BuildRoot() / "commit-tree-map" / commit; + return storage_config.BuildRoot() / "commit-tree-map" / commit; } -auto GetArchiveTreeIDFile(std::string const& repo_type, +auto GetArchiveTreeIDFile(StorageConfig const& storage_config, + std::string const& repo_type, std::string const& content) noexcept -> std::filesystem::path { - return StorageConfig::Instance().BuildRoot() / "tree-map" / repo_type / - content; + return storage_config.BuildRoot() / "tree-map" / repo_type / content; } -auto GetForeignFileTreeIDFile(std::string const& content, +auto GetForeignFileTreeIDFile(StorageConfig const& storage_config, + std::string const& content, std::string const& name, bool executable) noexcept -> std::filesystem::path { return GetDistdirTreeIDFile( + storage_config, HashFunction::ComputeBlobHash( nlohmann::json( std::unordered_map<std::string, std::pair<std::string, bool>>{ @@ -76,16 +79,17 @@ auto GetForeignFileTreeIDFile(std::string const& content, .HexString()); } -auto GetDistdirTreeIDFile(std::string const& content) noexcept +auto GetDistdirTreeIDFile(StorageConfig const& storage_config, + std::string const& content) noexcept -> std::filesystem::path { - return StorageConfig::Instance().BuildRoot() / "distfiles-tree-map" / - content; + return storage_config.BuildRoot() / "distfiles-tree-map" / content; } -auto GetResolvedTreeIDFile(std::string const& tree_hash, +auto GetResolvedTreeIDFile(StorageConfig const& storage_config, + std::string const& tree_hash, PragmaSpecial const& pragma_special) noexcept -> std::filesystem::path { - return StorageConfig::Instance().BuildRoot() / "special-tree-map" / + return storage_config.BuildRoot() / "special-tree-map" / kPragmaSpecialInverseMap.at(pragma_special) / tree_hash; } @@ -107,10 +111,10 @@ auto WriteTreeIDFile(std::filesystem::path const& tree_id_file, return FileSystemManager::Rename(tmp_file.string(), tree_id_file); } -auto AddToCAS(std::string const& data) noexcept +auto AddToCAS(Storage const& storage, std::string const& data) noexcept -> std::optional<std::filesystem::path> { // get file CAS instance - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage.CAS(); // write to cas auto digest = cas.StoreBlob(data); if (digest) { @@ -119,15 +123,15 @@ auto AddToCAS(std::string const& data) noexcept return std::nullopt; } -void AddDistfileToCAS(std::filesystem::path const& distfile, +void AddDistfileToCAS(Storage const& storage, + std::filesystem::path const& distfile, LocalPathsPtr const& just_mr_paths) noexcept { - auto const& cas = Storage::Instance().CAS(); + auto const& cas = storage.CAS(); for (auto const& dirpath : just_mr_paths->distdirs) { auto candidate = dirpath / distfile; if (FileSystemManager::Exists(candidate)) { // try to add to CAS - [[maybe_unused]] auto digest = - cas.StoreBlob(candidate, /*is_executable=*/false); + std::ignore = cas.StoreBlob(candidate, /*is_executable=*/false); } } } diff --git a/src/buildtool/storage/fs_utils.hpp b/src/buildtool/storage/fs_utils.hpp index e24f50a3..bd4852a9 100644 --- a/src/buildtool/storage/fs_utils.hpp +++ b/src/buildtool/storage/fs_utils.hpp @@ -21,6 +21,8 @@ #include "src/buildtool/common/user_structs.hpp" #include "src/buildtool/file_system/symlinks_map/pragma_special.hpp" +#include "src/buildtool/storage/config.hpp" +#include "src/buildtool/storage/storage.hpp" /* Utilities related to CAS and paths therein */ @@ -28,35 +30,41 @@ namespace StorageUtils { /// \brief Get location of Git repository. Defaults to the Git cache root when /// no better location is found. -[[nodiscard]] auto GetGitRoot(LocalPathsPtr const& just_mr_paths, +[[nodiscard]] auto GetGitRoot(StorageConfig const& storage_config, + LocalPathsPtr const& just_mr_paths, std::string const& repo_url) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the tree id associated with /// a given commit. -[[nodiscard]] auto GetCommitTreeIDFile(std::string const& commit) noexcept +[[nodiscard]] auto GetCommitTreeIDFile(StorageConfig const& storage_config, + std::string const& commit) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the tree id of an archive /// content. -[[nodiscard]] auto GetArchiveTreeIDFile(std::string const& repo_type, +[[nodiscard]] auto GetArchiveTreeIDFile(StorageConfig const& storage_config, + std::string const& repo_type, std::string const& content) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the tree id of an archive /// content. -[[nodiscard]] auto GetForeignFileTreeIDFile(std::string const& content, +[[nodiscard]] auto GetForeignFileTreeIDFile(StorageConfig const& storage_config, + std::string const& content, std::string const& name, bool executable) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing the tree id of a distdir list /// content. -[[nodiscard]] auto GetDistdirTreeIDFile(std::string const& content) noexcept +[[nodiscard]] auto GetDistdirTreeIDFile(StorageConfig const& storage_config, + std::string const& content) noexcept -> std::filesystem::path; /// \brief Get the path to the file storing a resolved tree hash. [[nodiscard]] auto GetResolvedTreeIDFile( + StorageConfig const& storage_config, std::string const& tree_hash, PragmaSpecial const& pragma_special) noexcept -> std::filesystem::path; @@ -66,11 +74,13 @@ namespace StorageUtils { /// \brief Add data to file CAS. /// Returns the path to the file added to CAS, or nullopt if not added. -[[nodiscard]] auto AddToCAS(std::string const& data) noexcept +[[nodiscard]] auto AddToCAS(Storage const& storage, + std::string const& data) noexcept -> std::optional<std::filesystem::path>; /// \brief Try to add distfile to CAS. -void AddDistfileToCAS(std::filesystem::path const& distfile, +void AddDistfileToCAS(Storage const& storage, + std::filesystem::path const& distfile, LocalPathsPtr const& just_mr_paths) noexcept; } // namespace StorageUtils diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index 8d105b22..8f265405 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -181,6 +181,7 @@ , ["src/buildtool/execution_api/common", "api_bundle"] , ["src/buildtool/serve_api/remote", "config"] , ["src/buildtool/serve_api/remote", "serve_api"] + , ["src/buildtool/storage", "storage"] ] } , "launch": diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index 6866bc26..5094c3aa 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -32,6 +32,7 @@ #include "src/buildtool/serve_api/remote/config.hpp" #include "src/buildtool/serve_api/remote/serve_api.hpp" #include "src/buildtool/storage/fs_utils.hpp" +#include "src/buildtool/storage/storage.hpp" #include "src/other_tools/just_mr/exit_codes.hpp" #include "src/other_tools/just_mr/progress_reporting/progress.hpp" #include "src/other_tools/just_mr/progress_reporting/progress_reporter.hpp" @@ -380,5 +381,5 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, return std::nullopt; } // if successful, return the output config - return StorageUtils::AddToCAS(mr_config.dump(2)); + return StorageUtils::AddToCAS(Storage::Instance(), mr_config.dump(2)); } diff --git a/src/other_tools/ops_maps/TARGETS b/src/other_tools/ops_maps/TARGETS index 205e7b83..01152ed9 100644 --- a/src/other_tools/ops_maps/TARGETS +++ b/src/other_tools/ops_maps/TARGETS @@ -79,6 +79,7 @@ , ["src/other_tools/git_operations", "git_repo_remote"] , ["src/other_tools/just_mr/progress_reporting", "statistics"] , ["src/other_tools/just_mr/progress_reporting", "progress"] + , ["src/buildtool/storage", "storage"] ] } , "archive_fetch_map": diff --git a/src/other_tools/ops_maps/content_cas_map.cpp b/src/other_tools/ops_maps/content_cas_map.cpp index 8877a547..59a22327 100644 --- a/src/other_tools/ops_maps/content_cas_map.cpp +++ b/src/other_tools/ops_maps/content_cas_map.cpp @@ -75,7 +75,7 @@ void FetchFromNetwork(ArchiveContent const& key, } } // add the fetched data to CAS - auto path = StorageUtils::AddToCAS(*data); + auto path = StorageUtils::AddToCAS(Storage::Instance(), *data); // check one last time if content is in CAS now if (not path) { (*logger)(fmt::format("Failed to store fetched content from {}", @@ -206,7 +206,8 @@ auto CreateContentCASMap( : std::filesystem::path(key.fetch_url) .filename() .string()); - StorageUtils::AddDistfileToCAS(repo_distfile, just_mr_paths); + StorageUtils::AddDistfileToCAS( + Storage::Instance(), repo_distfile, just_mr_paths); // check if content is in CAS now if (cas.BlobPath(digest, /*is_executable=*/false)) { JustMRProgress::Instance().TaskTracker().Stop(key.origin); diff --git a/src/other_tools/root_maps/commit_git_map.cpp b/src/other_tools/root_maps/commit_git_map.cpp index 7d74dc43..fac4d98d 100644 --- a/src/other_tools/root_maps/commit_git_map.cpp +++ b/src/other_tools/root_maps/commit_git_map.cpp @@ -435,7 +435,8 @@ void EnsureCommit(GitRepoInfo const& repo_info, return; } if (not is_commit_present.value()) { - auto tree_id_file = StorageUtils::GetCommitTreeIDFile(repo_info.hash); + auto tree_id_file = StorageUtils::GetCommitTreeIDFile( + StorageConfig::Instance(), repo_info.hash); // Check if we have stored a file association between commit and tree; // if an association file exists, the respective tree MUST be in the // Git cache @@ -950,8 +951,8 @@ auto CreateCommitGitMap( if (fetch_repo_path) { fetch_repo = std::filesystem::absolute(*fetch_repo_path).string(); } - std::filesystem::path repo_root = - StorageUtils::GetGitRoot(just_mr_paths, fetch_repo); + std::filesystem::path repo_root = StorageUtils::GetGitRoot( + StorageConfig::Instance(), just_mr_paths, fetch_repo); // ensure git repo // define Git operation to be done GitOpKey op_key = { diff --git a/src/other_tools/root_maps/content_git_map.cpp b/src/other_tools/root_maps/content_git_map.cpp index 9c7e2cee..c73d39a5 100644 --- a/src/other_tools/root_maps/content_git_map.cpp +++ b/src/other_tools/root_maps/content_git_map.cpp @@ -182,8 +182,8 @@ void ResolveContentTree( ContentGitMap::LoggerPtr const& logger) { if (key.pragma_special) { // get the resolved tree - auto tree_id_file = - StorageUtils::GetResolvedTreeIDFile(tree_hash, *key.pragma_special); + auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( + StorageConfig::Instance(), tree_hash, *key.pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); @@ -532,7 +532,7 @@ auto CreateContentGitMap( auto /* unused */, auto const& key) { auto archive_tree_id_file = StorageUtils::GetArchiveTreeIDFile( - key.repo_type, key.archive.content); + StorageConfig::Instance(), key.repo_type, key.archive.content); if (FileSystemManager::Exists(archive_tree_id_file)) { // read archive_tree_id from file tree_id_file auto archive_tree_id = @@ -788,8 +788,8 @@ auto CreateContentGitMap( : std::filesystem::path(key.archive.fetch_url) .filename() .string()); - StorageUtils::AddDistfileToCAS(repo_distfile, - just_mr_paths); + StorageUtils::AddDistfileToCAS( + Storage::Instance(), repo_distfile, just_mr_paths); // check if content is in CAS now if (auto content_cas_path = cas.BlobPath(digest, /*is_executable=*/false)) { diff --git a/src/other_tools/root_maps/distdir_git_map.cpp b/src/other_tools/root_maps/distdir_git_map.cpp index 4bfb32ca..b5c863e1 100644 --- a/src/other_tools/root_maps/distdir_git_map.cpp +++ b/src/other_tools/root_maps/distdir_git_map.cpp @@ -147,8 +147,8 @@ auto CreateDistdirGitMap( auto logger, auto /* unused */, auto const& key) { - auto distdir_tree_id_file = - StorageUtils::GetDistdirTreeIDFile(key.content_id); + auto distdir_tree_id_file = StorageUtils::GetDistdirTreeIDFile( + StorageConfig::Instance(), key.content_id); if (FileSystemManager::Exists(distdir_tree_id_file)) { // read distdir_tree_id from file tree_id_file auto distdir_tree_id = diff --git a/src/other_tools/root_maps/foreign_file_git_map.cpp b/src/other_tools/root_maps/foreign_file_git_map.cpp index cf3690f3..2ce582ce 100644 --- a/src/other_tools/root_maps/foreign_file_git_map.cpp +++ b/src/other_tools/root_maps/foreign_file_git_map.cpp @@ -34,8 +34,11 @@ void WithRootImportedToGit(ForeignFileInfo const& key, (*logger)("Importing to git failed", /*fatal=*/true); return; } - auto tree_id_file = StorageUtils::GetForeignFileTreeIDFile( - key.archive.content, key.name, key.executable); + auto tree_id_file = + StorageUtils::GetForeignFileTreeIDFile(StorageConfig::Instance(), + key.archive.content, + key.name, + key.executable); auto cache_written = StorageUtils::WriteTreeIDFile(tree_id_file, result.first); if (not cache_written) { @@ -197,46 +200,50 @@ void HandleAbsentForeignFile(ForeignFileInfo const& key, ServeApi const* serve, bool fetch_absent, std::size_t jobs) -> ForeignFileGitMap { - auto setup_foreign_file = - [content_cas_map, import_to_git_map, fetch_absent, serve]( - auto ts, - auto setter, - auto logger, - auto /* unused */, - auto const& key) { - if (key.absent and not fetch_absent) { - HandleAbsentForeignFile(key, serve, setter, logger); - return; - } - auto tree_id_file = StorageUtils::GetForeignFileTreeIDFile( - key.archive.content, key.name, key.executable); - if (FileSystemManager::Exists(tree_id_file)) { - auto tree_id = FileSystemManager::ReadFile(tree_id_file); - if (not tree_id) { - (*logger)(fmt::format("Failed to read tree id from file {}", - tree_id_file.string()), - /*fatal=*/true); - return; - } - UseCacheHit(*tree_id, setter); + auto setup_foreign_file = [content_cas_map, + import_to_git_map, + fetch_absent, + serve](auto ts, + auto setter, + auto logger, + auto /* unused */, + auto const& key) { + if (key.absent and not fetch_absent) { + HandleAbsentForeignFile(key, serve, setter, logger); + return; + } + auto tree_id_file = + StorageUtils::GetForeignFileTreeIDFile(StorageConfig::Instance(), + key.archive.content, + key.name, + key.executable); + if (FileSystemManager::Exists(tree_id_file)) { + auto tree_id = FileSystemManager::ReadFile(tree_id_file); + if (not tree_id) { + (*logger)(fmt::format("Failed to read tree id from file {}", + tree_id_file.string()), + /*fatal=*/true); return; } - content_cas_map->ConsumeAfterKeysReady( - ts, - {key.archive}, - [key, import_to_git_map, setter, logger, ts]( - [[maybe_unused]] auto const& values) { - WithFetchedFile(key, import_to_git_map, ts, setter, logger); - }, - [logger, content = key.archive.content](auto const& msg, - bool fatal) { - (*logger)(fmt::format("While ensuring content {} is in " - "CAS:\n{}", - content, - msg), - fatal); - }); - }; + UseCacheHit(*tree_id, setter); + return; + } + content_cas_map->ConsumeAfterKeysReady( + ts, + {key.archive}, + [key, import_to_git_map, setter, logger, ts]( + [[maybe_unused]] auto const& values) { + WithFetchedFile(key, import_to_git_map, ts, setter, logger); + }, + [logger, content = key.archive.content](auto const& msg, + bool fatal) { + (*logger)(fmt::format("While ensuring content {} is in " + "CAS:\n{}", + content, + msg), + fatal); + }); + }; return AsyncMapConsumer<ForeignFileInfo, std::pair<nlohmann::json, bool>>( setup_foreign_file, jobs); } diff --git a/src/other_tools/root_maps/fpath_git_map.cpp b/src/other_tools/root_maps/fpath_git_map.cpp index 65716b5c..f15051d7 100644 --- a/src/other_tools/root_maps/fpath_git_map.cpp +++ b/src/other_tools/root_maps/fpath_git_map.cpp @@ -107,8 +107,8 @@ void ResolveFilePathTree( FilePathGitMap::LoggerPtr const& logger) { if (pragma_special) { // get the resolved tree - auto tree_id_file = - StorageUtils::GetResolvedTreeIDFile(tree_hash, *pragma_special); + auto tree_id_file = StorageUtils::GetResolvedTreeIDFile( + StorageConfig::Instance(), tree_hash, *pragma_special); if (FileSystemManager::Exists(tree_id_file)) { // read resolved tree id auto resolved_tree_id = FileSystemManager::ReadFile(tree_id_file); |