path: root/src/other_tools/just_mr/setup_utils.cpp

// Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "src/other_tools/just_mr/setup_utils.hpp"

#include <algorithm>
#include <cstdlib>
#include <deque>
#include <exception>
#include <fstream>
#include <iterator>
#include <queue>
#include <unordered_set>
#include <utility>
#include <variant>

#include "nlohmann/json.hpp"
#include "src/buildtool/build_engine/expression/expression.hpp"
#include "src/buildtool/execution_api/remote/config.hpp"
#include "src/buildtool/file_system/file_system_manager.hpp"
#include "src/buildtool/file_system/precomputed_root.hpp"
#include "src/buildtool/logging/log_level.hpp"
#include "src/buildtool/logging/logger.hpp"
#include "src/other_tools/just_mr/exit_codes.hpp"
#include "src/other_tools/utils/parse_precomputed_root.hpp"
#include "src/utils/cpp/expected.hpp"

namespace {

void WarnUnknownKeys(std::string const& name, ExpressionPtr const& repo_def) {
    if (not repo_def->IsMap()) {
        return;
    }
    for (auto const& [key, value] : repo_def->Map()) {
        if (not kRepositoryExpectedFields.contains(key)) {
            Logger::Log(std::any_of(kRepositoryPossibleFieldTrunks.begin(),
                                    kRepositoryPossibleFieldTrunks.end(),
                                    [k = key](auto const& trunk) {
                                        return k.find(trunk) !=
                                               std::string::npos;
                                    })
                            ? LogLevel::Debug
                            : LogLevel::Warning,
                        "Ignoring unknown field {} in repository {}",
                        key,
                        name);
        }
    }
}

[[nodiscard]] auto GetTargetRepoIfPrecomputed(ExpressionPtr const& repos,
                                              std::string const& name)
    -> std::optional<std::string> {
    // Resolve indirections while the root's workspace root is declared
    // implicitly:
    ExpressionPtr root{name};
    while (root.IsNotNull() and root->IsString()) {
        auto const repo = repos->Get(root->String(), Expression::none_t{});
        if (not repo.IsNotNull() or not repo->IsMap()) {
            return std::nullopt;
        }
        root = repo->Get("repository", Expression::none_t{});
    }

    // Check the root is a precomputed root:
    if (auto const precomputed = ParsePrecomputedRoot(root)) {
        return precomputed->GetReferencedRepository();
    }
    return std::nullopt;
}

[[nodiscard]] auto IsAbsent(ExpressionPtr const& repo_def) -> bool {
    if (repo_def.IsNotNull() and repo_def->IsMap()) {
        if (auto repo = repo_def->Get("repository", Expression::none_t{});
            repo.IsNotNull() and repo->IsMap()) {
            if (auto pragma = repo->Get("pragma", Expression::none_t{});
                pragma.IsNotNull() and pragma->IsMap()) {
                auto absent = pragma->Get("absent", Expression::none_t{});
                return absent.IsNotNull() and absent->IsBool() and
                       absent->Bool();
            }
        }
    }
    return false;
}

[[nodiscard]] auto IsNotContentFixed(ExpressionPtr const& repo_def) -> bool {
    if (not repo_def.IsNotNull() or not repo_def->IsMap()) {
        return false;
    }
    if (auto repo = repo_def->Get("repository", Expression::none_t{});
        repo.IsNotNull() and repo->IsMap()) {
        // Check if type == "file"
        auto type = repo->Get("type", Expression::none_t{});
        if (not type.IsNotNull() or not type->IsString()) {
            return false;
        }
        if (type->String() == "file") {
            auto pragma = repo->Get("pragma", Expression::none_t{});
            if (not pragma.IsNotNull() or not pragma->IsMap()) {
                return true;  // not content-fixed if not to_git
            }
            // Check for explicit to_git == true
            if (auto to_git = pragma->Get("to_git", Expression::none_t{});
                to_git.IsNotNull() and to_git->IsBool() and to_git->Bool()) {
                return false;
            }
            // Check for implicit to_git == true
            if (auto special = pragma->Get("special", Expression::none_t{});
                special.IsNotNull() and special->IsString()) {
                auto const& special_str = special->String();
                if (special_str == "resolve-partially" or
                    special_str == "resolve-completely") {
                    return false;
                }
            }
            return true;  // not content-fixed if not to_git
        }
    }
    return false;
}

}  // namespace

namespace JustMR::Utils {

void ReachableRepositories(
    ExpressionPtr const& repos,
    std::string const& main,
    std::shared_ptr<JustMR::SetupRepos> const& setup_repos) {
    // use temporary sets to avoid duplicates
    std::unordered_set<std::string> include_repos_set;
    std::unordered_set<std::string> setup_repos_set;

    bool absent_main = IsAbsent(repos->Get(main, Expression::none_t{}));

    // traverse all bindings of main repository
    for (std::queue<std::string> to_process({main}); not to_process.empty();
         to_process.pop()) {
        auto const& repo_name = to_process.front();

        // Check the repo hasn't been processed yet
        if (not include_repos_set.insert(repo_name).second) {
            continue;
        }
        auto const repos_repo_name =
            repos->Get(repo_name, Expression::none_t{});
        if (not repos_repo_name.IsNotNull()) {
            continue;
        }
        WarnUnknownKeys(repo_name, repos_repo_name);

        // Warn if main repo is marked absent and current repo (including main)
        // is not content-fixed
        if (absent_main and IsNotContentFixed(repos_repo_name)) {
            Logger::Log(LogLevel::Warning,
                        "Found non-content-fixed repository {} as dependency "
                        "of absent main repository {}",
                        nlohmann::json(repo_name).dump(),
                        nlohmann::json(main).dump());
        }

        // If the current repo is a computed one, process its target repo
        if (auto precomputed = GetTargetRepoIfPrecomputed(repos, repo_name)) {
            to_process.push(*std::move(precomputed));
        }

        // check bindings
        auto const bindings =
            repos_repo_name->Get("bindings", Expression::none_t{});
        if (bindings.IsNotNull() and bindings->IsMap()) {
            for (auto const& bound : bindings->Map().Values()) {
                if (bound.IsNotNull() and bound->IsString()) {
                    to_process.push(bound->String());
                }
            }
        }

        for (auto const& layer : kAltDirs) {
            auto const layer_val =
                repos_repo_name->Get(layer, Expression::none_t{});
            if (layer_val.IsNotNull() and layer_val->IsString()) {
                auto const layer_repo_name = layer_val->String();
                setup_repos_set.insert(layer_repo_name);

                // If the overlay repo is a computed one, process its target
                // repo
                if (auto precomputed =
                        GetTargetRepoIfPrecomputed(repos, layer_repo_name)) {
                    to_process.push(*std::move(precomputed));
                }
            }
        }
    }
    setup_repos_set.insert(include_repos_set.begin(), include_repos_set.end());

    // copy to vectors
    setup_repos->to_setup.clear();
    setup_repos->to_setup.reserve(setup_repos_set.size());
    std::copy(
        setup_repos_set.begin(),
        setup_repos_set.end(),
        std::inserter(setup_repos->to_setup, setup_repos->to_setup.end()));
    setup_repos->to_include.clear();
    setup_repos->to_include.reserve(include_repos_set.size());
    std::copy(
        include_repos_set.begin(),
        include_repos_set.end(),
        std::inserter(setup_repos->to_include, setup_repos->to_include.end()));
}

void DefaultReachableRepositories(
    ExpressionPtr const& repos,
    std::shared_ptr<JustMR::SetupRepos> const& setup_repos) {
    setup_repos->to_setup = repos->Map().Keys();
    setup_repos->to_include = setup_repos->to_setup;
}

auto ReadConfiguration(
    std::optional<std::filesystem::path> const& config_file_opt,
    std::optional<std::filesystem::path> const& absent_file_opt)
    -> std::shared_ptr<Configuration> {
    if (not config_file_opt) {
        Logger::Log(LogLevel::Error, "Cannot find repository configuration.");
        std::exit(kExitConfigError);
    }
    auto const& config_file = *config_file_opt;

    auto config = nlohmann::json::object();
    if (not FileSystemManager::IsFile(config_file)) {
        Logger::Log(LogLevel::Error,
                    "Cannot read config file {}.",
                    config_file.string());
        std::exit(kExitConfigError);
    }
    try {
        std::ifstream fs(config_file);
        config = nlohmann::json::parse(fs);
        if (not config.is_object()) {
            Logger::Log(LogLevel::Error,
                        "Config file {} does not contain a JSON object.",
                        config_file.string());
            std::exit(kExitConfigError);
        }
    } catch (std::exception const& e) {
        Logger::Log(LogLevel::Error,
                    "Parsing config file {} failed with error:\n{}",
                    config_file.string(),
                    e.what());
        std::exit(kExitConfigError);
    }

    if (absent_file_opt) {
        if (not FileSystemManager::IsFile(*absent_file_opt)) {
            Logger::Log(LogLevel::Error,
                        "Not file specifying the absent repositories: {}",
                        absent_file_opt->string());
            std::exit(kExitConfigError);
        }
        try {
            std::ifstream fs(*absent_file_opt);
            auto absent = nlohmann::json::parse(fs);
            if (not absent.is_array()) {
                Logger::Log(LogLevel::Error,
                            "Expected {} to contain a list of repository "
                            "names, but found {}",
                            absent_file_opt->string(),
                            absent.dump());
                std::exit(kExitConfigError);
            }
            std::unordered_set<std::string> absent_set{};
            for (auto const& repo : absent) {
                if (not repo.is_string()) {
                    Logger::Log(LogLevel::Error,
                                "Repositories names have to be strings, but "
                                "found entry {} in {}",
                                repo.dump(),
                                absent_file_opt->string());
                    std::exit(kExitConfigError);
                }
                absent_set.insert(repo.get<std::string>());
            }
            auto new_repos = nlohmann::json::object();
            auto repos = config.value("repositories", nlohmann::json::object());
            for (auto const& [key, val] : repos.items()) {
                new_repos[key] = val;
                auto ws = val.value("repository", nlohmann::json::object());
                if (ws.is_object()) {
                    auto pragma = ws.value("pragma", nlohmann::json::object());
                    pragma["absent"] = absent_set.contains(key);
                    ws["pragma"] = pragma;
                    new_repos[key]["repository"] = ws;
                }
            }
            config["repositories"] = new_repos;
        } catch (std::exception const& e) {
            Logger::Log(LogLevel::Error,
                        "Parsing absent-repos file {} failed with error:\n{}",
                        absent_file_opt->string(),
                        e.what());
            std::exit(kExitConfigError);
        }
    }

    try {
        return std::make_shared<Configuration>(Expression::FromJson(config));
    } catch (std::exception const& e) {
        Logger::Log(LogLevel::Error,
                    "Parsing configuration file failed with error:\n{}",
                    e.what());
        std::exit(kExitConfigError);
    }
}

auto CreateAuthConfig(MultiRepoRemoteAuthArguments const& authargs) noexcept
    -> std::optional<Auth> {

    Auth::TLS::Builder tls_builder;
    tls_builder.SetCACertificate(authargs.tls_ca_cert)
        .SetClientCertificate(authargs.tls_client_cert)
        .SetClientKey(authargs.tls_client_key);

    // create auth config (including validation)
    auto result = tls_builder.Build();
    if (result) {
        if (*result) {
            // correctly configured TLS/SSL certification
            return *std::move(*result);
        }
        Logger::Log(LogLevel::Error, result->error());
        return std::nullopt;
    }

    // no TLS/SSL configuration was given, and we currently support no other
    // certification method, so return an empty config (no certification)
    return Auth{};
}

auto CreateLocalExecutionConfig(MultiRepoCommonArguments const& cargs) noexcept
    -> std::optional<LocalExecutionConfig> {

    LocalExecutionConfig::Builder builder;
    if (cargs.local_launcher.has_value()) {
        builder.SetLauncher(*cargs.local_launcher);
    }

    auto config = builder.Build();
    if (config) {
        return *std::move(config);
    }
    Logger::Log(LogLevel::Error, config.error());
    return std::nullopt;
}

auto CreateRemoteExecutionConfig(
    std::optional<std::string> const& remote_exec_addr,
    std::optional<std::string> const& remote_serve_addr) noexcept
    -> std::optional<RemoteExecutionConfig> {
    // if only a serve endpoint address is given, we assume it is one that acts
    // also as remote-execution
    auto remote_addr = remote_exec_addr ? remote_exec_addr : remote_serve_addr;

    RemoteExecutionConfig::Builder builder;
    auto config = builder.SetRemoteAddress(remote_addr).Build();

    if (config) {
        return *std::move(config);
    }

    Logger::Log(LogLevel::Error, config.error());
    return std::nullopt;
}

auto CreateServeConfig(
    std::optional<std::string> const& remote_serve_addr) noexcept
    -> std::optional<RemoteServeConfig> {
    RemoteServeConfig::Builder builder;
    auto config = builder.SetRemoteAddress(remote_serve_addr).Build();

    if (config) {
        return *std::move(config);
    }

    Logger::Log(LogLevel::Error, config.error());
    return std::nullopt;
}

}  // namespace JustMR::Utils