serve protocol: support executable distfiles

As serve and just-mr share their caching of description-tree
association, we also have to change this cache. Thanks to json
encoding before hashing, we know that the old and new hash keys
do not overlap, so the change is save. As distdirs also keep the
respective files in the git root, no new downlaoding will happen
either, hence no warning in the CHANGELOG is needed.

5 files changed, 33 insertions, 13 deletions

diff --git a/src/buildtool/serve_api/remote/source_tree_client.cpp b/src/buildtool/serve_api/remote/source_tree_client.cpp
index a998e1a1..0cb93a36 100644
--- a/src/buildtool/serve_api/remote/source_tree_client.cpp
+++ b/src/buildtool/serve_api/remote/source_tree_client.cpp
@@ -133,6 +133,7 @@ auto SourceTreeClient::ServeDistdirTree(
         auto* distfile = request.add_distfiles();
         distfile->set_name(k);
         distfile->set_content(v);
+        distfile->set_executable(false);
     }
     request.set_sync_tree(sync_tree);
 
diff --git a/src/buildtool/serve_api/serve_service/just_serve.proto b/src/buildtool/serve_api/serve_service/just_serve.proto
index f9bcf1b1..198daa9d 100644
--- a/src/buildtool/serve_api/serve_service/just_serve.proto
+++ b/src/buildtool/serve_api/serve_service/just_serve.proto
@@ -138,6 +138,10 @@ message ServeDistdirTreeRequest {
 
     // The git blob identifier of the distfile content.
     string content = 2;
+
+    // Whether the blob should occur executable in the resulting
+    // directory.
+    bool executable = 3;
   }
 
   // The list of distfiles.
diff --git a/src/buildtool/serve_api/serve_service/source_tree.cpp b/src/buildtool/serve_api/serve_service/source_tree.cpp
index 63261f4b..3e5dd01d 100644
--- a/src/buildtool/serve_api/serve_service/source_tree.cpp
+++ b/src/buildtool/serve_api/serve_service/source_tree.cpp
@@ -847,7 +847,8 @@ auto SourceTreeService::ServeArchiveTree(
 auto SourceTreeService::DistdirImportToGit(
     std::string const& distdir_tree_id,
     std::string const& content_id,
-    std::unordered_map<std::string, std::string> const& content_list,
+    std::unordered_map<std::string, std::pair<std::string, bool>> const&
+        content_list,
     bool sync_tree,
     ServeDistdirTreeResponse* response) -> ::grpc::Status {
     // create tmp directory for the distdir
@@ -866,8 +867,9 @@ auto SourceTreeService::DistdirImportToGit(
                         content_list.end(),
                         [&cas, tmp_path](auto const& kv) {
                             auto content_path = cas.BlobPath(
-                                ArtifactDigest(kv.second, 0, /*is_tree=*/false),
-                                /*is_executable=*/false);
+                                ArtifactDigest(
+                                    kv.second.first, 0, /*is_tree=*/false),
+                                kv.second.second);
                             if (content_path) {
                                 return FileSystemManager::CreateFileHardlink(
                                     *content_path,  // from: cas_path/content_id
@@ -962,7 +964,8 @@ auto SourceTreeService::ServeDistdirTree(
 
     bool blob_found{};
     auto const& cas = Storage::Instance().CAS();
-    std::unordered_map<std::string, std::string> content_list{};
+    std::unordered_map<std::string, std::pair<std::string, bool>>
+        content_list{};
     content_list.reserve(request->distfiles().size());
 
     for (auto const& kv : request->distfiles()) {
@@ -970,8 +973,8 @@ auto SourceTreeService::ServeDistdirTree(
         // check content blob is known
         auto digest = ArtifactDigest(content, 0, /*is_tree=*/false);
         // first check the local CAS itself
-        if (blob_found = static_cast<bool>(
-                cas.BlobPath(digest, /*is_executable=*/false));
+        if (blob_found =
+                static_cast<bool>(cas.BlobPath(digest, kv.executable()));
             not blob_found) {
             // check local Git cache
             auto res =
@@ -979,7 +982,7 @@ auto SourceTreeService::ServeDistdirTree(
             if (std::holds_alternative<std::string>(res)) {
                 // add content to local CAS
                 if (not cas.StoreBlob(std::get<std::string>(res),
-                                      /*is_executable=*/false)) {
+                                      kv.executable())) {
                     auto str = fmt::format(
                         "Failed to store content {} from local Git cache to "
                         "local CAS",
@@ -1010,7 +1013,7 @@ auto SourceTreeService::ServeDistdirTree(
                     if (std::holds_alternative<std::string>(res)) {
                         // add content to local CAS
                         if (not cas.StoreBlob(std::get<std::string>(res),
-                                              /*is_executable=*/false)) {
+                                              kv.executable())) {
                             auto str = fmt::format(
                                 "Failed to store content {} from known "
                                 "repository {} to local CAS",
@@ -1049,7 +1052,9 @@ auto SourceTreeService::ServeDistdirTree(
                         if (not remote_api_->RetrieveToCas(
                                 {Artifact::ObjectInfo{
                                     .digest = digest_clone,
-                                    .type = ObjectType::File}},
+                                    .type = kv.executable()
+                                                ? ObjectType::Executable
+                                                : ObjectType::File}},
                                 &(*local_api_))) {
                             auto str = fmt::format(
                                 "Failed to retrieve content {} from remote to "
@@ -1074,7 +1079,9 @@ auto SourceTreeService::ServeDistdirTree(
         }
         // store content blob to the entries list, using the expected raw id
         if (auto raw_id = FromHexString(content)) {
-            entries[*raw_id].emplace_back(kv.name(), ObjectType::File);
+            entries[*raw_id].emplace_back(
+                kv.name(),
+                kv.executable() ? ObjectType::Executable : ObjectType::File);
         }
         else {
             auto str = fmt::format(
@@ -1085,7 +1092,8 @@ auto SourceTreeService::ServeDistdirTree(
             return ::grpc::Status::OK;
         }
         // store to content_list for import-to-git hardlinking
-        content_list.insert_or_assign(kv.name(), kv.content());
+        content_list.insert_or_assign(
+            kv.name(), std::make_pair(kv.content(), kv.executable()));
     }
     // get hash of distdir content; this must match with that in just-mr
     auto content_id =
diff --git a/src/buildtool/serve_api/serve_service/source_tree.hpp b/src/buildtool/serve_api/serve_service/source_tree.hpp
index bd89bf78..ce53d750 100644
--- a/src/buildtool/serve_api/serve_service/source_tree.hpp
+++ b/src/buildtool/serve_api/serve_service/source_tree.hpp
@@ -20,6 +20,7 @@
 #include <mutex>
 #include <optional>
 #include <string>
+#include <utility>
 #include <variant>
 #include <vector>
 
@@ -207,7 +208,8 @@ class SourceTreeService final
     [[nodiscard]] auto DistdirImportToGit(
         std::string const& tree_id,
         std::string const& content_id,
-        std::unordered_map<std::string, std::string> const& content_list,
+        std::unordered_map<std::string, std::pair<std::string, bool>> const&
+            content_list,
         bool sync_tree,
         ServeDistdirTreeResponse* response) -> ::grpc::Status;
 };
diff --git a/src/other_tools/repo_map/repos_to_setup_map.cpp b/src/other_tools/repo_map/repos_to_setup_map.cpp
index f2b28685..e98cf848 100644
--- a/src/other_tools/repo_map/repos_to_setup_map.cpp
+++ b/src/other_tools/repo_map/repos_to_setup_map.cpp
@@ -367,6 +367,8 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
                                pragma_absent->get()->IsBool() and
                                pragma_absent->get()->Bool();
     // map of distfile to content
+    auto distdir_content_for_id = std::make_shared<
+        std::unordered_map<std::string, std::pair<std::string, bool>>>();
     auto distdir_content =
         std::make_shared<std::unordered_map<std::string, std::string>>();
     // get distdir list
@@ -534,6 +536,8 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
                                   : std::filesystem::path(archive.fetch_url)
                                         .filename()
                                         .string());
+            distdir_content_for_id->insert_or_assign(
+                repo_distfile, std::make_pair(archive.content, false));
             distdir_content->insert_or_assign(repo_distfile, archive.content);
             // add to fetch list
             dist_repos_to_fetch->emplace_back(std::move(archive));
@@ -541,7 +545,8 @@ void DistdirCheckout(ExpressionPtr const& repo_desc,
     }
     // get hash of distdir content
     auto distdir_content_id =
-        HashFunction::ComputeBlobHash(nlohmann::json(*distdir_content).dump())
+        HashFunction::ComputeBlobHash(
+            nlohmann::json(*distdir_content_for_id).dump())
             .HexString();
     // get the WS root as git tree
     DistdirInfo distdir_info = {