diff options
35 files changed, 383 insertions, 599 deletions
diff --git a/src/buildtool/serve_api/serve_service/TARGETS b/src/buildtool/serve_api/serve_service/TARGETS index 27e0d58c..80c79f2f 100644 --- a/src/buildtool/serve_api/serve_service/TARGETS +++ b/src/buildtool/serve_api/serve_service/TARGETS @@ -74,6 +74,8 @@ , ["src/buildtool/execution_api/execution_service", "cas_server"] , ["src/buildtool/execution_api/execution_service", "execution_server"] , ["src/buildtool/execution_api/execution_service", "operations_server"] + , ["src/buildtool/execution_api/local", "local"] + , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] diff --git a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp index f03a21e8..88fad61a 100644 --- a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp +++ b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp @@ -37,6 +37,8 @@ #include "src/buildtool/execution_api/execution_service/cas_server.hpp" #include "src/buildtool/execution_api/execution_service/execution_server.hpp" #include "src/buildtool/execution_api/execution_service/operations_server.hpp" +#include "src/buildtool/execution_api/local/local_api.hpp" +#include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/log_level.hpp" @@ -132,7 +134,9 @@ auto ServeServerImpl::Run( std::unique_ptr<StorageConfig> secondary_storage_config = nullptr; std::unique_ptr<Storage> secondary_storage = nullptr; std::unique_ptr<LocalContext> secondary_local_context = nullptr; - if (not ProtocolTraits::IsNative(hash_type)) { + IExecutionApi::Ptr secondary_local_api = nullptr; + auto const is_compat = not ProtocolTraits::IsNative(hash_type); + if (is_compat) { auto config = StorageConfig::Builder{} .SetBuildRoot(local_context->storage_config->build_root) @@ -150,16 +154,28 @@ auto ServeServerImpl::Run( LocalContext{.exec_config = local_context->exec_config, .storage_config = &*secondary_storage_config, .storage = &*secondary_storage}); + secondary_local_api = + std::make_shared<LocalApi>(&*secondary_local_context); } - SourceTreeService sts{&serve_config, - &apis, - /*native_context=*/secondary_local_context != nullptr - ? &*secondary_local_context - : local_context, - /*compat_context=*/secondary_local_context != nullptr - ? &*local_context - : nullptr}; + // setup the overall local api, aware of compatibility + IExecutionApi::Ptr mr_local_api = std::make_shared<MRLocalApi>( + is_compat ? &*secondary_local_context : local_context, + is_compat ? &*secondary_local_api : &*apis.local, + is_compat ? &*local_context : nullptr, + is_compat ? &*apis.local : nullptr); + // setup the apis to pass to SourceTreeService + auto const mr_apis = ApiBundle{.hash_function = apis.hash_function, + .local = mr_local_api, + .remote = apis.remote}; + + SourceTreeService sts{ + &serve_config, + &mr_apis, + is_compat ? &*secondary_local_context + : local_context, // native_context + is_compat ? &*local_context : nullptr // compat_context + }; // set up the server grpc::ServerBuilder builder; diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index a1180e56..002e8fda 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -131,6 +131,7 @@ , ["src/buildtool/execution_api/remote", "bazel"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] + , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "retry"] , ["src/buildtool/multithreading", "async_map_utils"] @@ -206,6 +207,7 @@ , ["src/buildtool/execution_api/remote", "bazel"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] + , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "retry"] diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index fdea8812..99f21eb1 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -32,6 +32,7 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" +#include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/retry.hpp" @@ -341,6 +342,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, std::unique_ptr<Storage> compat_storage = nullptr; std::unique_ptr<LocalContext> compat_local_context = nullptr; std::optional<LockFile> compat_lock = std::nullopt; + IExecutionApi::Ptr compat_local_api = nullptr; if (common_args.compatible) { auto config = StorageConfig::Builder{} .SetBuildRoot(native_storage_config.build_root) @@ -366,8 +368,16 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, "Failed to acquire compatible storage gc lock"); return kExitConfigError; } + compat_local_api = std::make_shared<LocalApi>(&*compat_local_context); } + // setup the overall local api, aware of compatibility + IExecutionApi::Ptr mr_local_api = std::make_shared<MRLocalApi>( + &native_local_context, + &*native_local_api, + common_args.compatible ? &*compat_local_context : nullptr, + common_args.compatible ? &*compat_local_api : nullptr); + // setup authentication config auto const auth_config = JustMR::Utils::CreateAuthConfig(auth_args); if (not auth_config) { @@ -404,8 +414,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, config, &hash_fct); } - bool const has_remote_api = - remote_api != nullptr and not common_args.compatible; + bool const has_remote_api = remote_api != nullptr; // pack the remote context RemoteContext const remote_context{.auth = &*auth_config, @@ -420,8 +429,8 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, } auto const apis = ApiBundle{.hash_function = hash_fct, - .local = native_local_api, - .remote = has_remote_api ? remote_api : native_local_api}; + .local = mr_local_api, + .remote = has_remote_api ? remote_api : mr_local_api}; auto serve = ServeApi::Create( *serve_config, compat_local_context != nullptr diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index bcc42bac..e4eaa18c 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -365,13 +365,6 @@ auto main(int argc, char* argv[]) -> int { // Run subcommands known to just and `do` if (arguments.cmd == SubCommand::kJustDo or arguments.cmd == SubCommand::kJustSubCmd) { - // check setup configuration arguments for validity - if (arguments.common.compatible and arguments.common.fetch_absent) { - Logger::Log(LogLevel::Error, - "Fetching absent repositories only available in " - "native mode!"); - return kExitConfigError; - } return CallJust(config_file, arguments.common, arguments.setup, @@ -402,13 +395,6 @@ auto main(int argc, char* argv[]) -> int { // Run subcommand `setup` or `setup-env` if (arguments.cmd == SubCommand::kSetup or arguments.cmd == SubCommand::kSetupEnv) { - // check setup configuration arguments for validity - if (arguments.common.compatible and arguments.common.fetch_absent) { - Logger::Log(LogLevel::Error, - "Fetching absent repositories only available in " - "native mode!"); - return kExitConfigError; - } auto mr_config_path = MultiRepoSetup( config, arguments.common, @@ -442,23 +428,6 @@ auto main(int argc, char* argv[]) -> int { // Run subcommand `fetch` if (arguments.cmd == SubCommand::kFetch) { - // check fetch configuration arguments for validity - if (arguments.common.compatible) { - if (arguments.common.remote_execution_address and - arguments.fetch.backup_to_remote) { - Logger::Log( - LogLevel::Error, - "Remote backup for fetched archives only available " - "in native mode!"); - return kExitConfigError; - } - if (arguments.common.fetch_absent) { - Logger::Log(LogLevel::Error, - "Fetching absent repositories only available " - "in native mode!"); - return kExitConfigError; - } - } return MultiRepoFetch(config, arguments.common, arguments.setup, diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index f6ada636..4ab032de 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -34,6 +34,7 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" +#include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -153,6 +154,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, std::unique_ptr<Storage> compat_storage = nullptr; std::unique_ptr<LocalContext> compat_local_context = nullptr; std::optional<LockFile> compat_lock = std::nullopt; + IExecutionApi::Ptr compat_local_api = nullptr; if (common_args.compatible) { auto config = StorageConfig::Builder{} .SetBuildRoot(native_storage_config.build_root) @@ -178,8 +180,16 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, "Failed to acquire compatible storage gc lock"); return std::nullopt; } + compat_local_api = std::make_shared<LocalApi>(&*compat_local_context); } + // setup the overall local api, aware of compatibility + IExecutionApi::Ptr mr_local_api = std::make_shared<MRLocalApi>( + &native_local_context, + &*native_local_api, + common_args.compatible ? &*compat_local_context : nullptr, + common_args.compatible ? &*compat_local_api : nullptr); + // setup authentication config auto const auth_config = JustMR::Utils::CreateAuthConfig(auth_args); if (not auth_config) { @@ -216,8 +226,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, config, &hash_fct); } - bool const has_remote_api = - remote_api != nullptr and not common_args.compatible; + bool const has_remote_api = remote_api != nullptr; // pack the remote context RemoteContext const remote_context{.auth = &*auth_config, @@ -232,8 +241,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, } auto const apis = ApiBundle{.hash_function = hash_fct, - .local = native_local_api, - .remote = has_remote_api ? remote_api : native_local_api}; + .local = mr_local_api, + .remote = has_remote_api ? remote_api : mr_local_api}; auto serve = ServeApi::Create( *serve_config, compat_local_context != nullptr diff --git a/test/buildtool/serve_api/TARGETS b/test/buildtool/serve_api/TARGETS index 69455203..89e2379c 100644 --- a/test/buildtool/serve_api/TARGETS +++ b/test/buildtool/serve_api/TARGETS @@ -21,13 +21,7 @@ } , "TESTS": { "type": ["@", "rules", "test", "suite"] - , "arguments_config": ["TEST_COMPATIBLE_REMOTE"] , "stage": ["serve_api"] - , "deps": - { "type": "if" - , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} - , "then": [] - , "else": ["source_tree_client"] - } + , "deps": ["source_tree_client"] } } diff --git a/test/end-to-end/just-mr/TARGETS b/test/end-to-end/just-mr/TARGETS index 5a47b7a4..636e5383 100644 --- a/test/end-to-end/just-mr/TARGETS +++ b/test/end-to-end/just-mr/TARGETS @@ -243,18 +243,15 @@ , { "type": "if" , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} , "then": [] - , "else": - [ "fetch-remote" - , "fetch-remote-git-tree" - , "fetch-absent" - , "fetch-absent-git-tree" - , "absent-config" - , "fetch-absent-archives" - , "fetch-absent-archives-symlinks" - , "fetch-absent-distdir-archive" - , "stay-local" - ] + , "else": ["fetch-remote", "fetch-remote-git-tree", "stay-local"] } + , [ "fetch-absent" + , "fetch-absent-git-tree" + , "absent-config" + , "fetch-absent-archives" + , "fetch-absent-archives-symlinks" + , "fetch-absent-distdir-archive" + ] ] } } diff --git a/test/end-to-end/just-mr/absent-config.sh b/test/end-to-end/just-mr/absent-config.sh index 236325c8..9bfd41a6 100644 --- a/test/end-to-end/just-mr/absent-config.sh +++ b/test/end-to-end/just-mr/absent-config.sh @@ -26,6 +26,11 @@ readonly RCFILE="${TEST_TMPDIR}/mrrc.json" readonly OUT="${TEST_TMPDIR}/out" readonly LOCAL_REPO="${TEST_TMPDIR}/local-repository" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + mkdir -p "${LOCAL_REPO}" cd "${LOCAL_REPO}" mkdir src @@ -116,12 +121,12 @@ echo CONF=$("${JUST_MR}" --local-build-root "${LBR}" \ --rc "${RCFILE}" \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" grep eg "${OUT}/out.txt" diff --git a/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh b/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh index 48effd5c..182aa35f 100644 --- a/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh +++ b/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh @@ -25,6 +25,11 @@ readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT3="${TEST_TMPDIR}/out3" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" @@ -70,13 +75,13 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep x "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information @@ -112,7 +117,7 @@ echo "${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ --fetch-absent install -o "${OUT3}" 2>&1 grep xx "${OUT3}/out.txt" diff --git a/test/end-to-end/just-mr/fetch-absent-archives.sh b/test/end-to-end/just-mr/fetch-absent-archives.sh index a85c95ee..4f4f6704 100644 --- a/test/end-to-end/just-mr/fetch-absent-archives.sh +++ b/test/end-to-end/just-mr/fetch-absent-archives.sh @@ -29,6 +29,11 @@ readonly OUT3="${TEST_TMPDIR}/out3" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" readonly OUT_DISTDIR="${TEST_TMPDIR}/out4" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" @@ -68,13 +73,13 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information @@ -114,7 +119,7 @@ EOF "${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ --fetch-absent install -o "${OUT3}" 2>&1 grep 42 "${OUT3}/out.txt" @@ -153,7 +158,7 @@ echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 grep 42 "${OUT_NON_ABSENT}/out.txt" @@ -164,7 +169,7 @@ echo mkdir -p "${OUT_DISTDIR}" "${JUST_MR}" --norc --local-build-root "${LBR_FOR_FETCH}" \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ fetch -o "${OUT_DISTDIR}" 2>&1 FETCHED_CONTENT=$(git hash-object "${OUT_DISTDIR}"/data.tar) echo diff --git a/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh b/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh index ade94031..d7cf70ef 100644 --- a/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh +++ b/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh @@ -26,6 +26,11 @@ readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out3" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" @@ -74,13 +79,13 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information @@ -121,7 +126,7 @@ echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 grep 42 "${OUT_NON_ABSENT}/out.txt" diff --git a/test/end-to-end/just-mr/fetch-absent-git-tree.sh b/test/end-to-end/just-mr/fetch-absent-git-tree.sh index fb5d516a..b0669b15 100644 --- a/test/end-to-end/just-mr/fetch-absent-git-tree.sh +++ b/test/end-to-end/just-mr/fetch-absent-git-tree.sh @@ -26,6 +26,11 @@ readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + mkdir work cd work touch ROOT @@ -61,12 +66,12 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" @@ -103,7 +108,7 @@ echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 grep 42 "${OUT_NON_ABSENT}/out.txt" diff --git a/test/end-to-end/just-mr/fetch-absent.sh b/test/end-to-end/just-mr/fetch-absent.sh index 30267998..41cde7a8 100644 --- a/test/end-to-end/just-mr/fetch-absent.sh +++ b/test/end-to-end/just-mr/fetch-absent.sh @@ -27,6 +27,11 @@ readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT3="${TEST_TMPDIR}/out3" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + mkdir work cd work touch ROOT @@ -64,13 +69,13 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information @@ -110,7 +115,7 @@ cat > targets/TARGETS <<'EOF' EOF "${JUST_MR}" --norc --local-build-root "${LBR}" \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ -L '["env", "PATH='"${PATH}"'"]' \ --just "${JUST}" \ --fetch-absent install -o "${OUT3}" 2>&1 @@ -150,7 +155,7 @@ cat repos.json echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ -L '["env", "PATH='"${PATH}"'"]' \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 diff --git a/test/end-to-end/just-mr/stay-local.sh b/test/end-to-end/just-mr/stay-local.sh index 1e411780..531630db 100644 --- a/test/end-to-end/just-mr/stay-local.sh +++ b/test/end-to-end/just-mr/stay-local.sh @@ -29,11 +29,6 @@ readonly EMPTY="${TEST_TMPDIR}/empty-directory" readonly SERVER="${PWD}/utils/null-server" readonly SERVER_STATE="${TEST_TMPDIR}/server" -COMPAT="" -if [ "${COMPATIBLE:-}" = "YES" ]; then - COMPAT="--compatible" -fi - ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" @@ -84,7 +79,7 @@ cat repos.json echo mkdir -p "${LOG}" "${JUST_MR}" --norc --local-build-root "${LBR}" \ - -r "127.0.0.1:${port}" ${COMPAT} \ + -r "127.0.0.1:${port}" \ --log-limit 5 -f "${LOG}/log" \ --distdir ../src \ setup > conf.json @@ -96,14 +91,14 @@ echo cat $(cat conf.json) echo # As a distdir (local directory!) was provided with all needed files, -# no attempty should be made to contact the remote-execution endpoint +# no attempt should be made to contact the remote-execution endpoint echo [ -f "${SERVER_STATE}/access" ] && cat "${SERVER_STATE}/access" && exit 1 || : # The obtained configuraiton should be suitable for building, also remotely "${JUST}" install -C "$(cat conf.json)" -o "${OUT}" \ --local-build-root "${LBR}" \ - -r "${REMOTE_EXECUTION_ADDRESS}" $COMPAT 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" 2>&1 echo cat "${OUT}/archive_id" [ $(cat "${OUT}/archive_id") = "${ARCHIVE_CONTENT}" ] @@ -114,7 +109,7 @@ echo # distdir is empty mkdir -p "${EMPTY}" "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR2}" \ - -r "${REMOTE_EXECUTION_ADDRESS}" $COMPAT \ + -r "${REMOTE_EXECUTION_ADDRESS}" \ --distdir ${EMPTY} \ install -o "${OUT2}" 2>&1 cat "${OUT2}/archive_id" diff --git a/test/end-to-end/serve-service/TARGETS b/test/end-to-end/serve-service/TARGETS index 3ab54a14..f91cbd8e 100644 --- a/test/end-to-end/serve-service/TARGETS +++ b/test/end-to-end/serve-service/TARGETS @@ -173,7 +173,7 @@ } , "TESTS (unconfigured)": { "type": ["@", "rules", "test", "suite"] - , "arguments_config": ["TEST_COMPATIBLE_REMOTE", "TEST_STANDALONE_SERVE"] + , "arguments_config": ["TEST_STANDALONE_SERVE"] , "stage": [ { "type": "if" , "cond": {"type": "var", "name": "TEST_STANDALONE_SERVE"} @@ -196,12 +196,8 @@ , "describe" , "deduplication-of-serve-requests" , "failure-report" + , "serve-tree" ] - , { "type": "if" - , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} - , "then": [] - , "else": ["serve-tree"] - } , [ ["./", "serve-archive-root", "TESTS"] , ["./", "serve-git-root", "TESTS"] , ["./", "serve-file-root", "TESTS"] diff --git a/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known-upload.sh b/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known-upload.sh index 502aaecb..5b8404ab 100644 --- a/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known-upload.sh +++ b/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known-upload.sh @@ -18,7 +18,7 @@ # This test checks that an absent root can successfully be made in the presence # of the serve endpoint in the situation where we already have the file # association (i.e., we know the unresolved root tree) and the serve endpoint -# does not know the archive. The upload can only happen in native mode. +# does not know the archive. # # The test archive contains symlinks to be resolved, which tests also the # resolved tree file association. @@ -98,35 +98,17 @@ rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as # absent. For a serve endpoint that does not have the archive blob available, # this will require uploading the locally-known root tree to remote CAS, from -# where the serve endpoint will pick it up. This can only happen in native mode. -if [ -z "${COMPAT}" ]; then - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent) - cat "${CONF}" - echo - test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent 2>&1 && exit 1 || : - echo Failed as expected -fi +# where the serve endpoint will pick it up. +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup absent) +cat "${CONF}" +echo +test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known.sh b/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known.sh index b882bc2d..4d20deb4 100644 --- a/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known.sh +++ b/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known.sh @@ -96,8 +96,8 @@ TREE=$(jq -r '.repositories.present.workspace_root[1]' "${CONF}") rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as -# absent. This serve endpoint known already the archive, so it should be able to -# set it up even if in compatible mode. +# absent. This serve endpoint knows already the archive, so it should be able to +# set it up. ${JUST} gc --local-build-root ${LBR} 2>&1 ${JUST} gc --local-build-root ${LBR} 2>&1 diff --git a/test/end-to-end/serve-service/serve-archive-root/resolved-absent.sh b/test/end-to-end/serve-service/serve-archive-root/resolved-absent.sh index 07568390..fb31b959 100644 --- a/test/end-to-end/serve-service/serve-archive-root/resolved-absent.sh +++ b/test/end-to-end/serve-service/serve-archive-root/resolved-absent.sh @@ -106,32 +106,15 @@ cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" -# Check that serve can provide this tree as present in a clean build root. This -# can happen however only in native mode. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present) - cat "${CONF}" - echo - test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || : - echo Failed as expected -fi +# Check that serve can provide this tree as present in a clean build root. +rm -rf "${LBR}" +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup present) +cat "${CONF}" +echo +test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/resolved-present.sh b/test/end-to-end/serve-service/serve-archive-root/resolved-present.sh index d2ecf13c..bd9b17c9 100644 --- a/test/end-to-end/serve-service/serve-archive-root/resolved-present.sh +++ b/test/end-to-end/serve-service/serve-archive-root/resolved-present.sh @@ -15,8 +15,9 @@ ### -# This test checks 3 of the options to make a present root for an archive, where: -# - archive is in local distfile; +# This test checks 3 of the options to make a present root for an archive, +# where: +# - archive is in a local distfile; # - there is already a file association to the unresolved root tree; # - we receive the archive content from serve endpoint via the remote CAS. # @@ -99,48 +100,31 @@ cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" -# We now test if the serve endpoint can provide us the root. This can only -# happen in we're in native mode. -if [ -z "${COMPAT}" ]; then - - # In a clean build root, ask serve to set up the root for us, from scratch - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - - # Double-check the file association was created and root remains available - # without the remote endpoints - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# We now test if the serve endpoint can provide us the root. +# In a clean build root, ask serve to set up the root for us, from scratch +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" + +# Double-check the file association was created and root remains available +# without the remote endpoints +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known-upload.sh b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known-upload.sh index c27c8066..f087b3d6 100644 --- a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known-upload.sh +++ b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known-upload.sh @@ -18,7 +18,7 @@ # This test checks that an absent root can successfully be made in the presence # of the serve endpoint in the situation where we already have the file # association (i.e., we know the unresolved root tree) and the serve endpoint -# does not know the archive. The upload can only happen in native mode. +# does not know the archive. # # The test archive does not contain symlinks. ## @@ -96,35 +96,17 @@ rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as # absent. For a serve endpoint that does not have the archive blob available, # this will require uploading the locally-known root tree to remote CAS, from -# where the serve endpoint will pick it up. This can only happen in native mode. -if [ -z "${COMPAT}" ]; then - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent) - cat "${CONF}" - echo - test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent 2>&1 && exit 1 || : - echo Failed as expected -fi +# where the serve endpoint will pick it up. +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup absent) +cat "${CONF}" +echo +test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known.sh b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known.sh index 8be8aa0f..3f1fd7e2 100644 --- a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known.sh +++ b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known.sh @@ -94,8 +94,8 @@ TREE=$(jq -r '.repositories.present.workspace_root[1]' "${CONF}") rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as -# absent. This serve endpoint known already the archive, so it should be able to -# set it up even if in compatible mode. +# absent. This serve endpoint knows already the archive, so it should be able to +# set it up. ${JUST} gc --local-build-root ${LBR} 2>&1 ${JUST} gc --local-build-root ${LBR} 2>&1 diff --git a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent.sh b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent.sh index e304c9f2..46ed6c37 100644 --- a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent.sh +++ b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent.sh @@ -104,32 +104,15 @@ cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" -# Check that serve can provide this tree as present in a clean build root. This -# can happen however only in native mode. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present) - cat "${CONF}" - echo - test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || : - echo Failed as expected -fi +# Check that serve can provide this tree as present in a clean build root. +rm -rf "${LBR}" +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup present) +cat "${CONF}" +echo +test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/unresolved-present.sh b/test/end-to-end/serve-service/serve-archive-root/unresolved-present.sh index e25a72bf..ee1e1dc2 100644 --- a/test/end-to-end/serve-service/serve-archive-root/unresolved-present.sh +++ b/test/end-to-end/serve-service/serve-archive-root/unresolved-present.sh @@ -15,8 +15,9 @@ ### -# This test checks 3 of the options to make a present root for an archive, where: -# - archive is in local distfile; +# This test checks 3 of the options to make a present root for an archive, +# where: +# - archive is in a local distfile; # - there is already a file association to the unresolved root tree; # - we receive the archive content from serve endpoint via the remote CAS. # @@ -97,48 +98,31 @@ cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" -# We now test if the serve endpoint can provide us the root. This can only -# happen in we're in native mode. -if [ -z "${COMPAT}" ]; then - - # In a clean build root, ask serve to set up the root for us, from scratch - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - - # Double-check the file association was created and root remains available - # without the remote endpoints - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# We now test if the serve endpoint can provide us the root. +# In a clean build root, ask serve to set up the root for us, from scratch +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" + +# Double-check the file association was created and root remains available +# without the remote endpoints +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-distdir-root/absent.sh b/test/end-to-end/serve-service/serve-distdir-root/absent.sh index 241297c0..e44d10ea 100644 --- a/test/end-to-end/serve-service/serve-distdir-root/absent.sh +++ b/test/end-to-end/serve-service/serve-distdir-root/absent.sh @@ -119,32 +119,15 @@ echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" # Check that serve can provide now this tree as present in a clean build root. -# This can happen only in native mode. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present) - cat "${CONF}" - echo - test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || : - echo Failed as expected -fi +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup present) +cat "${CONF}" +echo +test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh b/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh index 18dff9e0..8696659b 100644 --- a/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh +++ b/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh @@ -78,7 +78,7 @@ mkdir -p "${OUT}" grep 'HELLO WORLD' "${OUT}/out.txt" -# also verify that the repo config has the repository abent +# also verify that the repo config has the repository absent CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" ${ENDPOINT_ARGS} setup) echo diff --git a/test/end-to-end/serve-service/serve-distdir-root/present.sh b/test/end-to-end/serve-service/serve-distdir-root/present.sh index 8a205182..b0bf7727 100644 --- a/test/end-to-end/serve-service/serve-distdir-root/present.sh +++ b/test/end-to-end/serve-service/serve-distdir-root/present.sh @@ -17,7 +17,7 @@ ### # This test checks 3 of the options to make a present root for a distidr # repository, where: -# - archives are in local distfile; +# - archives are in a local distfile; # - there is already a file association to the distdir root tree; # - we receive the distdir root from serve endpoint via the remote CAS. ## @@ -113,48 +113,31 @@ cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" -# We now test if the serve endpoint can provide us the present root. This can -# only happen in we're in native mode. -if [ -z "${COMPAT}" ]; then - - # In a clean build root, ask serve to set up the root for us, from scratch - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - - # Double-check the file association was created and root remains available - # without the remote endpoints - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# We now test if the serve endpoint can provide us the present root. +# In a clean build root, ask serve to set up the root for us, from scratch +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" + +# Double-check the file association was created and root remains available +# without the remote endpoints +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-distdir-root/upload.sh b/test/end-to-end/serve-service/serve-distdir-root/upload.sh index ff33599d..841a51db 100644 --- a/test/end-to-end/serve-service/serve-distdir-root/upload.sh +++ b/test/end-to-end/serve-service/serve-distdir-root/upload.sh @@ -17,7 +17,6 @@ ### # This test checks that an absent distdir root can be successfully computed # locally and then uploaded to a serve endpoint that does not know the root. -# The upload can only happen in native mode. ## set -eu @@ -108,35 +107,17 @@ rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as # absent. For a serve endpoint that does not have the archive blob available, # this will require uploading the locally-known root tree to remote CAS, from -# where the serve endpoint will pick it up. This can only happen in native mode. -if [ -z "${COMPAT}" ]; then - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent) - cat "${CONF}" - echo - test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent 2>&1 && exit 1 || : - echo Failed as expected -fi +# where the serve endpoint will pick it up. +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup absent) +cat "${CONF}" +echo +test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-file-root/upload.sh b/test/end-to-end/serve-service/serve-file-root/upload.sh index 3362aa0d..01e35ab9 100644 --- a/test/end-to-end/serve-service/serve-file-root/upload.sh +++ b/test/end-to-end/serve-service/serve-file-root/upload.sh @@ -86,8 +86,6 @@ EOF # Setup an absent root from local path. Even if root is present, if a serve # endpoint is given then we try to set it up there as well. As this serve # endpoint does not know the tree, it will try to upload through the remote CAS. -# The upload succeeds if remote in native mode, but fails (non-fatally) in -# compatible mode. CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ @@ -98,31 +96,15 @@ echo test $(jq -r '.repositories.present_file.workspace_root[1]' "${CONF}") = "${TREE}" # Check in a clean local build root that the serve endpoint now has the root -# tree. This can only work in native mode, where the root was actually uploaded. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent_git_tree) - cat "${CONF}" - echo - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent_git_tree 2>&1 && exit 1 || : - echo Failed as expected -fi +# tree. +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup absent_git_tree) +cat "${CONF}" +echo echo OK diff --git a/test/end-to-end/serve-service/serve-git-root/absent-upload.sh b/test/end-to-end/serve-service/serve-git-root/absent-upload.sh index 2fa54512..9ec592d0 100644 --- a/test/end-to-end/serve-service/serve-git-root/absent-upload.sh +++ b/test/end-to-end/serve-service/serve-git-root/absent-upload.sh @@ -16,8 +16,7 @@ ### # This test checks that an absent root known in a local checkout can be -# successfully uploaded to a serve endpoint. This can only succeed in native -# mode. +# successfully uploaded to a serve endpoint. ## set -eu @@ -79,30 +78,14 @@ EOF # Setup an absent root from a local checkout. For a serve endpoint that does # not have the commit available, this will upload the locally-known root tree -# to remote CAS, from where the serve endpoint will pick it up. This requires -# that the remotes are in native mode. -if [ -z "${COMPAT}" ]; then - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${SUBTREE}" - -else - - echo --- - echo Checking expected failures - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# to remote CAS, from where the serve endpoint will pick it up. +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${SUBTREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-git-root/absent.sh b/test/end-to-end/serve-service/serve-git-root/absent.sh index ab808d53..97d2f7ea 100644 --- a/test/end-to-end/serve-service/serve-git-root/absent.sh +++ b/test/end-to-end/serve-service/serve-git-root/absent.sh @@ -71,8 +71,7 @@ EOF # Run the checks ## -# Compute absent root by asking serve to set it up from scratch. This works also -# in compatible mode. +# Compute absent root by asking serve to set it up from scratch. rm -rf "${LBR}" CONF=$("${JUST_MR}" --norc -C repos.json \ @@ -85,31 +84,15 @@ echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE_0}" # Check that serve can provide also a subtree of this tree as present in a clean -# build root. This can only happen if remotes are in native mode. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present) - cat "${CONF}" - echo - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || : - echo Failed as expected -fi +# build root. +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup present) +cat "${CONF}" +echo echo OK diff --git a/test/end-to-end/serve-service/serve-git-root/present.sh b/test/end-to-end/serve-service/serve-git-root/present.sh index 51723be2..0c359ec8 100644 --- a/test/end-to-end/serve-service/serve-git-root/present.sh +++ b/test/end-to-end/serve-service/serve-git-root/present.sh @@ -61,70 +61,54 @@ EOF # Run the checks ## -# Compute present root by asking the serve endpoint to set it up for us. This -# requires remotes in native mode. -if [ -z "${COMPAT}" ]; then - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" - - # Compute present root locally from now populated Git cache - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" - - # Check that the subdir is also working correctly - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - cat > repos.json <<EOF +# Compute present root by asking the serve endpoint to set it up for us. +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" + +# Compute present root locally from now populated Git cache +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" + +# Check that the subdir is also working correctly +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +cat > repos.json <<EOF { "repositories": - { "main": - { "repository": - { "type": "git" - , "commit": "$COMMIT_0" - , "repository": "http://non-existent.example.org/data.git" - , "branch": "master" - , "subdir": "repo" - } +{ "main": + { "repository": + { "type": "git" + , "commit": "$COMMIT_0" + , "repository": "http://non-existent.example.org/data.git" + , "branch": "master" + , "subdir": "repo" } } } +} EOF - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - -else - - echo --- - echo Checking expected failures - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo echo OK diff --git a/test/end-to-end/serve-service/serve-git-tree-root/absent.sh b/test/end-to-end/serve-service/serve-git-tree-root/absent.sh index 1e4b5f05..ddfe33d8 100644 --- a/test/end-to-end/serve-service/serve-git-tree-root/absent.sh +++ b/test/end-to-end/serve-service/serve-git-tree-root/absent.sh @@ -60,8 +60,7 @@ EOF # Run the checks ## -# Compute absent root by asking serve to set it up from scratch. This works also -# in compatible mode. +# Compute absent root by asking serve to set it up from scratch. rm -rf "${LBR}" CONF=$("${JUST_MR}" --norc -C repos.json \ diff --git a/test/end-to-end/serve-service/serve-git-tree-root/present.sh b/test/end-to-end/serve-service/serve-git-tree-root/present.sh index 2a704b52..4b05f9b5 100644 --- a/test/end-to-end/serve-service/serve-git-tree-root/present.sh +++ b/test/end-to-end/serve-service/serve-git-tree-root/present.sh @@ -16,7 +16,7 @@ ### # This test checks if we can make a present root for a Git-tree repository -# using the serve endpoint. This can only succeed in native mode. +# using the serve endpoint. ## set -eu @@ -59,43 +59,27 @@ EOF # Run the checks ## -# Compute present root by asking the serve endpoint to set it up for us. This -# requires remotes in native mode. -if [ -z "${COMPAT}" ]; then - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" - - # Compute present root locally from now populated Git cache - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" - -else - - echo --- - echo Checking expected failures - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# Compute present root by asking the serve endpoint to set it up for us. +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" + +# Compute present root locally from now populated Git cache +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" echo OK diff --git a/test/end-to-end/serve-service/serve-tree.sh b/test/end-to-end/serve-service/serve-tree.sh index 75573964..d95a890b 100755 --- a/test/end-to-end/serve-service/serve-tree.sh +++ b/test/end-to-end/serve-service/serve-tree.sh @@ -32,7 +32,12 @@ mkdir -p "${DISTDIR}" cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) -REMOTE="-r ${REMOTE_EXECUTION_ADDRESS}" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + +REMOTE="-r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT}" mkdir work cd work @@ -70,7 +75,7 @@ echo echo echo Local build "${JUST_MR}" --norc --local-build-root "${LBR_A}" --just "${JUST}" \ - --distdir "${DISTDIR}" build \ + --distdir "${DISTDIR}" ${COMPAT} build \ --log-limit 4 \ --dump-artifacts local.json 2>&1 |