From cae2ec50abedb135bbeab2aae95fecebcfe977b3 Mon Sep 17 00:00:00 2001
From: Klaus Aehlig <klaus.aehlig@huawei.com>
Date: Mon, 4 Mar 2024 14:06:54 +0100
Subject: serve protocol: support executable distfiles

As serve and just-mr share their caching of description-tree
association, we also have to change this cache. Thanks to json
encoding before hashing, we know that the old and new hash keys
do not overlap, so the change is save. As distdirs also keep the
respective files in the git root, no new downlaoding will happen
either, hence no warning in the CHANGELOG is needed.
---
 .../serve_api/serve_service/source_tree.cpp        | 30 ++++++++++++++--------
 1 file changed, 19 insertions(+), 11 deletions(-)

(limited to 'src/buildtool/serve_api/serve_service/source_tree.cpp')

diff --git a/src/buildtool/serve_api/serve_service/source_tree.cpp b/src/buildtool/serve_api/serve_service/source_tree.cpp
index 63261f4b..3e5dd01d 100644
--- a/src/buildtool/serve_api/serve_service/source_tree.cpp
+++ b/src/buildtool/serve_api/serve_service/source_tree.cpp
@@ -847,7 +847,8 @@ auto SourceTreeService::ServeArchiveTree(
 auto SourceTreeService::DistdirImportToGit(
     std::string const& distdir_tree_id,
     std::string const& content_id,
-    std::unordered_map<std::string, std::string> const& content_list,
+    std::unordered_map<std::string, std::pair<std::string, bool>> const&
+        content_list,
     bool sync_tree,
     ServeDistdirTreeResponse* response) -> ::grpc::Status {
     // create tmp directory for the distdir
@@ -866,8 +867,9 @@ auto SourceTreeService::DistdirImportToGit(
                         content_list.end(),
                         [&cas, tmp_path](auto const& kv) {
                             auto content_path = cas.BlobPath(
-                                ArtifactDigest(kv.second, 0, /*is_tree=*/false),
-                                /*is_executable=*/false);
+                                ArtifactDigest(
+                                    kv.second.first, 0, /*is_tree=*/false),
+                                kv.second.second);
                             if (content_path) {
                                 return FileSystemManager::CreateFileHardlink(
                                     *content_path,  // from: cas_path/content_id
@@ -962,7 +964,8 @@ auto SourceTreeService::ServeDistdirTree(
 
     bool blob_found{};
     auto const& cas = Storage::Instance().CAS();
-    std::unordered_map<std::string, std::string> content_list{};
+    std::unordered_map<std::string, std::pair<std::string, bool>>
+        content_list{};
     content_list.reserve(request->distfiles().size());
 
     for (auto const& kv : request->distfiles()) {
@@ -970,8 +973,8 @@ auto SourceTreeService::ServeDistdirTree(
         // check content blob is known
         auto digest = ArtifactDigest(content, 0, /*is_tree=*/false);
         // first check the local CAS itself
-        if (blob_found = static_cast<bool>(
-                cas.BlobPath(digest, /*is_executable=*/false));
+        if (blob_found =
+                static_cast<bool>(cas.BlobPath(digest, kv.executable()));
             not blob_found) {
             // check local Git cache
             auto res =
@@ -979,7 +982,7 @@ auto SourceTreeService::ServeDistdirTree(
             if (std::holds_alternative<std::string>(res)) {
                 // add content to local CAS
                 if (not cas.StoreBlob(std::get<std::string>(res),
-                                      /*is_executable=*/false)) {
+                                      kv.executable())) {
                     auto str = fmt::format(
                         "Failed to store content {} from local Git cache to "
                         "local CAS",
@@ -1010,7 +1013,7 @@ auto SourceTreeService::ServeDistdirTree(
                     if (std::holds_alternative<std::string>(res)) {
                         // add content to local CAS
                         if (not cas.StoreBlob(std::get<std::string>(res),
-                                              /*is_executable=*/false)) {
+                                              kv.executable())) {
                             auto str = fmt::format(
                                 "Failed to store content {} from known "
                                 "repository {} to local CAS",
@@ -1049,7 +1052,9 @@ auto SourceTreeService::ServeDistdirTree(
                         if (not remote_api_->RetrieveToCas(
                                 {Artifact::ObjectInfo{
                                     .digest = digest_clone,
-                                    .type = ObjectType::File}},
+                                    .type = kv.executable()
+                                                ? ObjectType::Executable
+                                                : ObjectType::File}},
                                 &(*local_api_))) {
                             auto str = fmt::format(
                                 "Failed to retrieve content {} from remote to "
@@ -1074,7 +1079,9 @@ auto SourceTreeService::ServeDistdirTree(
         }
         // store content blob to the entries list, using the expected raw id
         if (auto raw_id = FromHexString(content)) {
-            entries[*raw_id].emplace_back(kv.name(), ObjectType::File);
+            entries[*raw_id].emplace_back(
+                kv.name(),
+                kv.executable() ? ObjectType::Executable : ObjectType::File);
         }
         else {
             auto str = fmt::format(
@@ -1085,7 +1092,8 @@ auto SourceTreeService::ServeDistdirTree(
             return ::grpc::Status::OK;
         }
         // store to content_list for import-to-git hardlinking
-        content_list.insert_or_assign(kv.name(), kv.content());
+        content_list.insert_or_assign(
+            kv.name(), std::make_pair(kv.content(), kv.executable()));
     }
     // get hash of distdir content; this must match with that in just-mr
     auto content_id =
-- 
cgit v1.2.3