path: root/src/other_tools/root_maps/foreign_file_git_map.cpp

// Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "src/other_tools/root_maps/foreign_file_git_map.hpp"

#include <filesystem>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <vector>

#include "fmt/core.h"
#include "src/buildtool/common/artifact_digest.hpp"
#include "src/buildtool/crypto/hash_info.hpp"
#include "src/buildtool/file_system/file_root.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/file_system/git_cas.hpp"
#include "src/buildtool/file_system/git_repo.hpp"
#include "src/buildtool/file_system/git_types.hpp"
#include "src/buildtool/file_system/object_type.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/multithreading/task_system.hpp"
#include "src/buildtool/storage/fs_utils.hpp"
#include "src/utils/cpp/expected.hpp"
#include "src/utils/cpp/hex_string.hpp"
#include "src/utils/cpp/tmp_dir.hpp"

namespace {

void WithRootImportedToGit(ForeignFileInfo const& key,
                           StorageConfig const& storage_config,
                           std::pair<std::string, GitCASPtr> const& result,
                           ForeignFileGitMap::SetterPtr const& setter,
                           ForeignFileGitMap::LoggerPtr const& logger) {
    if (not result.second) {
        (*logger)("Importing to git failed", /*fatal=*/true);
        return;
    }
    auto tree_id_file =
        StorageUtils::GetForeignFileTreeIDFile(storage_config,
                                               key.archive.content_hash.Hash(),
                                               key.name,
                                               key.executable);
    auto cache_written =
        StorageUtils::WriteTreeIDFile(tree_id_file, result.first);
    if (not cache_written) {
        (*logger)(
            fmt::format("Failed to write cache file {}", tree_id_file.string()),
            /*fatal=*/false);
    }
    (*setter)(
        std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker,
                                         result.first,
                                         storage_config.GitRoot().string()}),
                  /*is_cache_hit=*/false));
}

void WithFetchedFile(ForeignFileInfo const& key,
                     gsl::not_null<StorageConfig const*> const& storage_config,
                     Storage const& storage,
                     gsl::not_null<ImportToGitMap*> const& import_to_git_map,
                     gsl::not_null<TaskSystem*> const& ts,
                     ForeignFileGitMap::SetterPtr const& setter,
                     ForeignFileGitMap::LoggerPtr const& logger) {
    auto tmp_dir = storage_config->CreateTypedTmpDir("foreign-file");
    auto const& cas = storage.CAS();
    auto digest = ArtifactDigest{key.archive.content_hash, 0};
    auto content_cas_path = cas.BlobPath(digest, key.executable);
    if (not content_cas_path) {
        (*logger)(
            fmt::format("Failed to locally find {} after fetching for repo {}",
                        key.archive.content_hash.Hash(),
                        nlohmann::json(key.archive.origin).dump()),
            true);
        return;
    }
    auto did_create_hardlink = FileSystemManager::CreateFileHardlink(
        *content_cas_path, tmp_dir->GetPath() / key.name, LogLevel::Warning);
    if (not did_create_hardlink) {
        (*logger)(fmt::format(
                      "Failed to hard link {} as {} in temporary directory {}",
                      content_cas_path->string(),
                      nlohmann::json(key.name).dump(),
                      tmp_dir->GetPath().string()),
                  true);
        return;
    }
    CommitInfo c_info{
        tmp_dir->GetPath(),
        fmt::format("foreign file at {}", nlohmann::json(key.name).dump()),
        key.archive.content_hash.Hash()};
    import_to_git_map->ConsumeAfterKeysReady(
        ts,
        {std::move(c_info)},
        [tmp_dir,  // keep tmp_dir alive
         key,
         storage_config,
         setter,
         logger](auto const& values) {
            WithRootImportedToGit(
                key, *storage_config, *values[0], setter, logger);
        },
        [logger, target_path = tmp_dir->GetPath()](auto const& msg,
                                                   bool fatal) {
            (*logger)(fmt::format("While importing target {} to Git:\n{}",
                                  target_path.string(),
                                  msg),
                      fatal);
        });
}

void UseCacheHit(StorageConfig const& storage_config,
                 const std::string& tree_id,
                 ForeignFileGitMap::SetterPtr const& setter) {
    // We keep the invariant, that, whenever a cache entry is written,
    // the root is in our git root; in particular, the latter is present,
    // initialized, etc; so we can directly write the result.
    (*setter)(
        std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker,
                                         tree_id,
                                         storage_config.GitRoot().string()}),
                  /*is_cache_hit=*/true));
}

void HandleAbsentForeignFile(ForeignFileInfo const& key,
                             ServeApi const* serve,
                             ForeignFileGitMap::SetterPtr const& setter,
                             ForeignFileGitMap::LoggerPtr const& logger) {
    // Compute tree in memory
    GitRepo::tree_entries_t entries{};
    auto raw_id = FromHexString(key.archive.content_hash.Hash());
    if (not raw_id) {
        (*logger)(fmt::format("Failure converting {} to raw id.",
                              key.archive.content_hash.Hash()),
                  true);
        return;
    }
    entries[*raw_id].emplace_back(
        key.name, key.executable ? ObjectType::Executable : ObjectType::File);
    auto tree = GitRepo::CreateShallowTree(entries);
    if (not tree) {
        (*logger)(fmt::format("Failure to construct in-memory tree with entry "
                              "{} at place {}",
                              key.archive.content_hash.Hash(),
                              nlohmann::json(key.name).dump()),
                  true);
        return;
    }
    auto tree_id = ToHexString(tree->first);
    if (serve != nullptr) {
        auto const has_tree = serve->CheckRootTree(tree_id);
        if (not has_tree) {
            (*logger)(fmt::format("Checking that the serve endpoint knows tree "
                                  "{} failed.",
                                  tree_id),
                      /*fatal=*/true);
            return;
        }
        if (*has_tree) {
            (*setter)(std::pair(
                nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}),
                /*is_cache_hit=*/false));
            return;
        }
        auto const serve_result = serve->RetrieveTreeFromForeignFile(
            key.archive.content_hash.Hash(), key.name, key.executable);
        if (serve_result) {
            // if serve has set up the tree, it must match what we expect
            if (tree_id != serve_result->tree) {
                (*logger)(fmt::format("Mismatch in served root tree "
                                      "id: expected {}, but got {}",
                                      tree_id,
                                      serve_result->tree),
                          /*fatal=*/true);
                return;
            }
            // set workspace root as absent
            (*setter)(std::pair(
                nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}),
                /*is_cache_hit=*/false));
            return;
        }
        if (serve_result.error() == GitLookupError::Fatal) {
            (*logger)(fmt::format("Serve endpoint failed to set up root "
                                  "from known foreign-file content {}",
                                  key.archive.content_hash.Hash()),
                      /*fatal=*/true);
            return;
        }
    }
    else {
        (*logger)(fmt::format("Workspace root {} marked absent but no "
                              "serve endpoint provided.",
                              tree_id),
                  /*fatal=*/false);
    }
    (*setter)(
        std::pair(nlohmann::json::array({FileRoot::kGitTreeMarker, tree_id}),
                  false /*no cache hit*/));
}
}  // namespace

[[nodiscard]] auto CreateForeignFileGitMap(
    gsl::not_null<ContentCASMap*> const& content_cas_map,
    gsl::not_null<ImportToGitMap*> const& import_to_git_map,
    ServeApi const* serve,
    gsl::not_null<StorageConfig const*> const& storage_config,
    gsl::not_null<Storage const*> const& storage,
    bool fetch_absent,
    std::size_t jobs) -> ForeignFileGitMap {
    auto setup_foreign_file = [content_cas_map,
                               import_to_git_map,
                               fetch_absent,
                               serve,
                               storage,
                               storage_config](auto ts,
                                               auto setter,
                                               auto logger,
                                               auto /* unused */,
                                               auto const& key) {
        if (key.absent and not fetch_absent) {
            HandleAbsentForeignFile(key, serve, setter, logger);
            return;
        }
        auto tree_id_file = StorageUtils::GetForeignFileTreeIDFile(
            *storage_config,
            key.archive.content_hash.Hash(),
            key.name,
            key.executable);
        if (FileSystemManager::Exists(tree_id_file)) {
            auto tree_id = FileSystemManager::ReadFile(tree_id_file);
            if (not tree_id) {
                (*logger)(fmt::format("Failed to read tree id from file {}",
                                      tree_id_file.string()),
                          /*fatal=*/true);
                return;
            }
            UseCacheHit(*storage_config, *tree_id, setter);
            return;
        }
        content_cas_map->ConsumeAfterKeysReady(
            ts,
            {key.archive},
            [key,
             import_to_git_map,
             storage,
             storage_config,
             setter,
             logger,
             ts]([[maybe_unused]] auto const& values) {
                WithFetchedFile(key,
                                storage_config,
                                *storage,
                                import_to_git_map,
                                ts,
                                setter,
                                logger);
            },
            [logger, hash = key.archive.content_hash.Hash()](auto const& msg,
                                                             bool fatal) {
                (*logger)(fmt::format("While ensuring content {} is in "
                                      "CAS:\n{}",
                                      hash,
                                      msg),
                          fatal);
            });
    };
    return AsyncMapConsumer<ForeignFileInfo, std::pair<nlohmann::json, bool>>(
        setup_foreign_file, jobs);
}