Storage config: change layout to support several storage generations

4 files changed, 60 insertions, 23 deletions

diff --git a/bin/just-mr.py b/bin/just-mr.py
index 3eade552..39d9d7c2 100755
--- a/bin/just-mr.py
+++ b/bin/just-mr.py
@@ -199,7 +199,7 @@ def git_root(*, upstream: Optional[str]) -> str:
     elif upstream and os.path.isabs(upstream) and os.path.isdir(upstream):
         return upstream
     else:
-        return os.path.join(g_ROOT, "git")
+        return os.path.join(g_ROOT, "repositories/generation-0/git")
 
 
 def is_cache_git_root(upstream: Optional[str]) -> bool:
@@ -393,7 +393,8 @@ def archive_tmp_checkout_dir(content: str, repo_type: str) -> str:
 
 
 def archive_tree_id_file(content: str, repo_type: str) -> str:
-    return os.path.join(g_ROOT, "tree-map", repo_type, content)
+    return os.path.join(g_ROOT, "repositories/generation-0/tree-map",
+                        repo_type, content)
 
 
 def get_distfile(desc: Json) -> str:
@@ -577,7 +578,8 @@ def distdir_tmp_dir(content: str) -> str:
 
 
 def distdir_tree_id_file(content: str) -> str:
-    return os.path.join(g_ROOT, "distfiles-tree-map", content)
+    return os.path.join(g_ROOT, "repositories/generation-0/distfiles-tree-map",
+                        content)
 
 
 def distdir_checkout(desc: Json, repos: Json):
diff --git a/src/buildtool/storage/config.hpp b/src/buildtool/storage/config.hpp
index 39bb74ac..4bc8911c 100644
--- a/src/buildtool/storage/config.hpp
+++ b/src/buildtool/storage/config.hpp
@@ -72,9 +72,29 @@ struct StorageConfig final {
         return build_root / "protocol-dependent";
     }
 
+    /// \brief Root directory of all repository generations.
+    [[nodiscard]] auto RepositoryRoot() const noexcept
+        -> std::filesystem::path {
+        return build_root / "repositories";
+    }
+
+    /// \brief Directory for the given generation of stored repositories
+    [[nodiscard]] auto RepositoryGenerationRoot(
+        std::size_t index) const noexcept -> std::filesystem::path {
+        ExpectsAudit(index < num_generations);
+        auto generation = std::string{"generation-"} + std::to_string(index);
+        return RepositoryRoot() / generation;
+    }
+
+    /// \brief Directory for the git repository of the given generation
+    [[nodiscard]] auto GitGenerationRoot(std::size_t index) const noexcept
+        -> std::filesystem::path {
+        return RepositoryGenerationRoot(index) / "git";
+    }
+
     /// \brief Directory for the git repository storing various roots
     [[nodiscard]] auto GitRoot() const noexcept -> std::filesystem::path {
-        return build_root / "git";
+        return GitGenerationRoot(0);
     }
 
     /// \brief Root directory of specific storage generation
diff --git a/src/buildtool/storage/fs_utils.cpp b/src/buildtool/storage/fs_utils.cpp
index 3286c4ec..03ef75c6 100644
--- a/src/buildtool/storage/fs_utils.cpp
+++ b/src/buildtool/storage/fs_utils.cpp
@@ -52,22 +52,27 @@ auto GetGitRoot(StorageConfig const& storage_config,
 }
 
 auto GetCommitTreeIDFile(StorageConfig const& storage_config,
-                         std::string const& commit) noexcept
+                         std::string const& commit,
+                         std::size_t generation) noexcept
     -> std::filesystem::path {
-    return storage_config.build_root / "commit-tree-map" / commit;
+    return storage_config.RepositoryGenerationRoot(generation) /
+           "commit-tree-map" / commit;
 }
 
 auto GetArchiveTreeIDFile(StorageConfig const& storage_config,
                           std::string const& repo_type,
-                          std::string const& content) noexcept
+                          std::string const& content,
+                          std::size_t generation) noexcept
     -> std::filesystem::path {
-    return storage_config.build_root / "tree-map" / repo_type / content;
+    return storage_config.RepositoryGenerationRoot(generation) / "tree-map" /
+           repo_type / content;
 }
 
 auto GetForeignFileTreeIDFile(StorageConfig const& storage_config,
                               std::string const& content,
                               std::string const& name,
-                              bool executable) noexcept
+                              bool executable,
+                              std::size_t generation) noexcept
     -> std::filesystem::path {
     return GetDistdirTreeIDFile(
         storage_config,
@@ -76,21 +81,26 @@ auto GetForeignFileTreeIDFile(StorageConfig const& storage_config,
                 std::unordered_map<std::string, std::pair<std::string, bool>>{
                     {name, {content, executable}}})
                 .dump())
-            .HexString());
+            .HexString(),
+        generation);
 }
 
 auto GetDistdirTreeIDFile(StorageConfig const& storage_config,
-                          std::string const& content) noexcept
+                          std::string const& content,
+                          std::size_t generation) noexcept
     -> std::filesystem::path {
-    return storage_config.build_root / "distfiles-tree-map" / content;
+    return storage_config.RepositoryGenerationRoot(generation) /
+           "distfiles-tree-map" / content;
 }
 
 auto GetResolvedTreeIDFile(StorageConfig const& storage_config,
                            std::string const& tree_hash,
-                           PragmaSpecial const& pragma_special) noexcept
+                           PragmaSpecial const& pragma_special,
+                           std::size_t generation) noexcept
     -> std::filesystem::path {
-    return storage_config.build_root / "special-tree-map" /
-           kPragmaSpecialInverseMap.at(pragma_special) / tree_hash;
+    return storage_config.RepositoryGenerationRoot(generation) /
+           "special-tree-map" / kPragmaSpecialInverseMap.at(pragma_special) /
+           tree_hash;
 }
 
 auto WriteTreeIDFile(std::filesystem::path const& tree_id_file,
diff --git a/src/buildtool/storage/fs_utils.hpp b/src/buildtool/storage/fs_utils.hpp
index bd4852a9..7482457b 100644
--- a/src/buildtool/storage/fs_utils.hpp
+++ b/src/buildtool/storage/fs_utils.hpp
@@ -38,14 +38,16 @@ namespace StorageUtils {
 /// \brief Get the path to the file storing the tree id associated with
 /// a given commit.
 [[nodiscard]] auto GetCommitTreeIDFile(StorageConfig const& storage_config,
-                                       std::string const& commit) noexcept
+                                       std::string const& commit,
+                                       std::size_t generation = 0) noexcept
     -> std::filesystem::path;
 
 /// \brief Get the path to the file storing the tree id of an archive
 /// content.
 [[nodiscard]] auto GetArchiveTreeIDFile(StorageConfig const& storage_config,
                                         std::string const& repo_type,
-                                        std::string const& content) noexcept
+                                        std::string const& content,
+                                        std::size_t generation = 0) noexcept
     -> std::filesystem::path;
 
 /// \brief Get the path to the file storing the tree id of an archive
@@ -53,20 +55,23 @@ namespace StorageUtils {
 [[nodiscard]] auto GetForeignFileTreeIDFile(StorageConfig const& storage_config,
                                             std::string const& content,
                                             std::string const& name,
-                                            bool executable) noexcept
+                                            bool executable,
+                                            std::size_t generation = 0) noexcept
     -> std::filesystem::path;
 
 /// \brief Get the path to the file storing the tree id of a distdir list
 /// content.
 [[nodiscard]] auto GetDistdirTreeIDFile(StorageConfig const& storage_config,
-                                        std::string const& content) noexcept
+                                        std::string const& content,
+                                        std::size_t generation = 0) noexcept
     -> std::filesystem::path;
 
 /// \brief Get the path to the file storing a resolved tree hash.
-[[nodiscard]] auto GetResolvedTreeIDFile(
-    StorageConfig const& storage_config,
-    std::string const& tree_hash,
-    PragmaSpecial const& pragma_special) noexcept -> std::filesystem::path;
+[[nodiscard]] auto GetResolvedTreeIDFile(StorageConfig const& storage_config,
+                                         std::string const& tree_hash,
+                                         PragmaSpecial const& pragma_special,
+                                         std::size_t generation = 0) noexcept
+    -> std::filesystem::path;
 
 /// \brief Write a tree id to file. The parent folder of the file must exist!
 [[nodiscard]] auto WriteTreeIDFile(std::filesystem::path const& tree_id_file,