diff options
author | Paul Cristian Sarbu <paul.cristian.sarbu@huawei.com> | 2024-10-21 15:37:44 +0200 |
---|---|---|
committer | Paul Cristian Sarbu <paul.cristian.sarbu@huawei.com> | 2024-10-25 13:00:43 +0200 |
commit | e6127d3cbd4fb8b3dd82bc740692877b72f977b7 (patch) | |
tree | b229db4b17c4bcfef5ea9ea8deed61f355bf151d | |
parent | 5e0fea2f02f8e721f44ea23302ade010e9f9bb97 (diff) | |
download | justbuild-e6127d3cbd4fb8b3dd82bc740692877b72f977b7.tar.gz |
Enable compatible mode for just-mr and SourceTree serve service...
...by using the new local api that can handle any remote endpoint,
irrespective of protocol.
Also ensure all tests for the serve service are now being run both
in native and compatible modes.
35 files changed, 383 insertions, 599 deletions
diff --git a/src/buildtool/serve_api/serve_service/TARGETS b/src/buildtool/serve_api/serve_service/TARGETS index 27e0d58c..80c79f2f 100644 --- a/src/buildtool/serve_api/serve_service/TARGETS +++ b/src/buildtool/serve_api/serve_service/TARGETS @@ -74,6 +74,8 @@ , ["src/buildtool/execution_api/execution_service", "cas_server"] , ["src/buildtool/execution_api/execution_service", "execution_server"] , ["src/buildtool/execution_api/execution_service", "operations_server"] + , ["src/buildtool/execution_api/local", "local"] + , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/logging", "log_level"] , ["src/buildtool/storage", "config"] , ["src/buildtool/storage", "storage"] diff --git a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp index f03a21e8..88fad61a 100644 --- a/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp +++ b/src/buildtool/serve_api/serve_service/serve_server_implementation.cpp @@ -37,6 +37,8 @@ #include "src/buildtool/execution_api/execution_service/cas_server.hpp" #include "src/buildtool/execution_api/execution_service/execution_server.hpp" #include "src/buildtool/execution_api/execution_service/operations_server.hpp" +#include "src/buildtool/execution_api/local/local_api.hpp" +#include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/file_system/file_system_manager.hpp" #include "src/buildtool/file_system/git_repo.hpp" #include "src/buildtool/logging/log_level.hpp" @@ -132,7 +134,9 @@ auto ServeServerImpl::Run( std::unique_ptr<StorageConfig> secondary_storage_config = nullptr; std::unique_ptr<Storage> secondary_storage = nullptr; std::unique_ptr<LocalContext> secondary_local_context = nullptr; - if (not ProtocolTraits::IsNative(hash_type)) { + IExecutionApi::Ptr secondary_local_api = nullptr; + auto const is_compat = not ProtocolTraits::IsNative(hash_type); + if (is_compat) { auto config = StorageConfig::Builder{} .SetBuildRoot(local_context->storage_config->build_root) @@ -150,16 +154,28 @@ auto ServeServerImpl::Run( LocalContext{.exec_config = local_context->exec_config, .storage_config = &*secondary_storage_config, .storage = &*secondary_storage}); + secondary_local_api = + std::make_shared<LocalApi>(&*secondary_local_context); } - SourceTreeService sts{&serve_config, - &apis, - /*native_context=*/secondary_local_context != nullptr - ? &*secondary_local_context - : local_context, - /*compat_context=*/secondary_local_context != nullptr - ? &*local_context - : nullptr}; + // setup the overall local api, aware of compatibility + IExecutionApi::Ptr mr_local_api = std::make_shared<MRLocalApi>( + is_compat ? &*secondary_local_context : local_context, + is_compat ? &*secondary_local_api : &*apis.local, + is_compat ? &*local_context : nullptr, + is_compat ? &*apis.local : nullptr); + // setup the apis to pass to SourceTreeService + auto const mr_apis = ApiBundle{.hash_function = apis.hash_function, + .local = mr_local_api, + .remote = apis.remote}; + + SourceTreeService sts{ + &serve_config, + &mr_apis, + is_compat ? &*secondary_local_context + : local_context, // native_context + is_compat ? &*local_context : nullptr // compat_context + }; // set up the server grpc::ServerBuilder builder; diff --git a/src/other_tools/just_mr/TARGETS b/src/other_tools/just_mr/TARGETS index a1180e56..002e8fda 100644 --- a/src/other_tools/just_mr/TARGETS +++ b/src/other_tools/just_mr/TARGETS @@ -131,6 +131,7 @@ , ["src/buildtool/execution_api/remote", "bazel"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] + , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "retry"] , ["src/buildtool/multithreading", "async_map_utils"] @@ -206,6 +207,7 @@ , ["src/buildtool/execution_api/remote", "bazel"] , ["src/buildtool/execution_api/remote", "config"] , ["src/buildtool/execution_api/remote", "context"] + , ["src/buildtool/execution_api/serve", "mr_local_api"] , ["src/buildtool/file_system/symlinks_map", "resolve_symlinks_map"] , ["src/buildtool/logging", "logging"] , ["src/buildtool/main", "retry"] diff --git a/src/other_tools/just_mr/fetch.cpp b/src/other_tools/just_mr/fetch.cpp index fdea8812..99f21eb1 100644 --- a/src/other_tools/just_mr/fetch.cpp +++ b/src/other_tools/just_mr/fetch.cpp @@ -32,6 +32,7 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" +#include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" #include "src/buildtool/main/retry.hpp" @@ -341,6 +342,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, std::unique_ptr<Storage> compat_storage = nullptr; std::unique_ptr<LocalContext> compat_local_context = nullptr; std::optional<LockFile> compat_lock = std::nullopt; + IExecutionApi::Ptr compat_local_api = nullptr; if (common_args.compatible) { auto config = StorageConfig::Builder{} .SetBuildRoot(native_storage_config.build_root) @@ -366,8 +368,16 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, "Failed to acquire compatible storage gc lock"); return kExitConfigError; } + compat_local_api = std::make_shared<LocalApi>(&*compat_local_context); } + // setup the overall local api, aware of compatibility + IExecutionApi::Ptr mr_local_api = std::make_shared<MRLocalApi>( + &native_local_context, + &*native_local_api, + common_args.compatible ? &*compat_local_context : nullptr, + common_args.compatible ? &*compat_local_api : nullptr); + // setup authentication config auto const auth_config = JustMR::Utils::CreateAuthConfig(auth_args); if (not auth_config) { @@ -404,8 +414,7 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, config, &hash_fct); } - bool const has_remote_api = - remote_api != nullptr and not common_args.compatible; + bool const has_remote_api = remote_api != nullptr; // pack the remote context RemoteContext const remote_context{.auth = &*auth_config, @@ -420,8 +429,8 @@ auto MultiRepoFetch(std::shared_ptr<Configuration> const& config, } auto const apis = ApiBundle{.hash_function = hash_fct, - .local = native_local_api, - .remote = has_remote_api ? remote_api : native_local_api}; + .local = mr_local_api, + .remote = has_remote_api ? remote_api : mr_local_api}; auto serve = ServeApi::Create( *serve_config, compat_local_context != nullptr diff --git a/src/other_tools/just_mr/main.cpp b/src/other_tools/just_mr/main.cpp index bcc42bac..e4eaa18c 100644 --- a/src/other_tools/just_mr/main.cpp +++ b/src/other_tools/just_mr/main.cpp @@ -365,13 +365,6 @@ auto main(int argc, char* argv[]) -> int { // Run subcommands known to just and `do` if (arguments.cmd == SubCommand::kJustDo or arguments.cmd == SubCommand::kJustSubCmd) { - // check setup configuration arguments for validity - if (arguments.common.compatible and arguments.common.fetch_absent) { - Logger::Log(LogLevel::Error, - "Fetching absent repositories only available in " - "native mode!"); - return kExitConfigError; - } return CallJust(config_file, arguments.common, arguments.setup, @@ -402,13 +395,6 @@ auto main(int argc, char* argv[]) -> int { // Run subcommand `setup` or `setup-env` if (arguments.cmd == SubCommand::kSetup or arguments.cmd == SubCommand::kSetupEnv) { - // check setup configuration arguments for validity - if (arguments.common.compatible and arguments.common.fetch_absent) { - Logger::Log(LogLevel::Error, - "Fetching absent repositories only available in " - "native mode!"); - return kExitConfigError; - } auto mr_config_path = MultiRepoSetup( config, arguments.common, @@ -442,23 +428,6 @@ auto main(int argc, char* argv[]) -> int { // Run subcommand `fetch` if (arguments.cmd == SubCommand::kFetch) { - // check fetch configuration arguments for validity - if (arguments.common.compatible) { - if (arguments.common.remote_execution_address and - arguments.fetch.backup_to_remote) { - Logger::Log( - LogLevel::Error, - "Remote backup for fetched archives only available " - "in native mode!"); - return kExitConfigError; - } - if (arguments.common.fetch_absent) { - Logger::Log(LogLevel::Error, - "Fetching absent repositories only available " - "in native mode!"); - return kExitConfigError; - } - } return MultiRepoFetch(config, arguments.common, arguments.setup, diff --git a/src/other_tools/just_mr/setup.cpp b/src/other_tools/just_mr/setup.cpp index f6ada636..4ab032de 100644 --- a/src/other_tools/just_mr/setup.cpp +++ b/src/other_tools/just_mr/setup.cpp @@ -34,6 +34,7 @@ #include "src/buildtool/execution_api/remote/bazel/bazel_api.hpp" #include "src/buildtool/execution_api/remote/config.hpp" #include "src/buildtool/execution_api/remote/context.hpp" +#include "src/buildtool/execution_api/serve/mr_local_api.hpp" #include "src/buildtool/file_system/symlinks_map/resolve_symlinks_map.hpp" #include "src/buildtool/logging/log_level.hpp" #include "src/buildtool/logging/logger.hpp" @@ -153,6 +154,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, std::unique_ptr<Storage> compat_storage = nullptr; std::unique_ptr<LocalContext> compat_local_context = nullptr; std::optional<LockFile> compat_lock = std::nullopt; + IExecutionApi::Ptr compat_local_api = nullptr; if (common_args.compatible) { auto config = StorageConfig::Builder{} .SetBuildRoot(native_storage_config.build_root) @@ -178,8 +180,16 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, "Failed to acquire compatible storage gc lock"); return std::nullopt; } + compat_local_api = std::make_shared<LocalApi>(&*compat_local_context); } + // setup the overall local api, aware of compatibility + IExecutionApi::Ptr mr_local_api = std::make_shared<MRLocalApi>( + &native_local_context, + &*native_local_api, + common_args.compatible ? &*compat_local_context : nullptr, + common_args.compatible ? &*compat_local_api : nullptr); + // setup authentication config auto const auth_config = JustMR::Utils::CreateAuthConfig(auth_args); if (not auth_config) { @@ -216,8 +226,7 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, config, &hash_fct); } - bool const has_remote_api = - remote_api != nullptr and not common_args.compatible; + bool const has_remote_api = remote_api != nullptr; // pack the remote context RemoteContext const remote_context{.auth = &*auth_config, @@ -232,8 +241,8 @@ auto MultiRepoSetup(std::shared_ptr<Configuration> const& config, } auto const apis = ApiBundle{.hash_function = hash_fct, - .local = native_local_api, - .remote = has_remote_api ? remote_api : native_local_api}; + .local = mr_local_api, + .remote = has_remote_api ? remote_api : mr_local_api}; auto serve = ServeApi::Create( *serve_config, compat_local_context != nullptr diff --git a/test/buildtool/serve_api/TARGETS b/test/buildtool/serve_api/TARGETS index 69455203..89e2379c 100644 --- a/test/buildtool/serve_api/TARGETS +++ b/test/buildtool/serve_api/TARGETS @@ -21,13 +21,7 @@ } , "TESTS": { "type": ["@", "rules", "test", "suite"] - , "arguments_config": ["TEST_COMPATIBLE_REMOTE"] , "stage": ["serve_api"] - , "deps": - { "type": "if" - , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} - , "then": [] - , "else": ["source_tree_client"] - } + , "deps": ["source_tree_client"] } } diff --git a/test/end-to-end/just-mr/TARGETS b/test/end-to-end/just-mr/TARGETS index 5a47b7a4..636e5383 100644 --- a/test/end-to-end/just-mr/TARGETS +++ b/test/end-to-end/just-mr/TARGETS @@ -243,18 +243,15 @@ , { "type": "if" , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} , "then": [] - , "else": - [ "fetch-remote" - , "fetch-remote-git-tree" - , "fetch-absent" - , "fetch-absent-git-tree" - , "absent-config" - , "fetch-absent-archives" - , "fetch-absent-archives-symlinks" - , "fetch-absent-distdir-archive" - , "stay-local" - ] + , "else": ["fetch-remote", "fetch-remote-git-tree", "stay-local"] } + , [ "fetch-absent" + , "fetch-absent-git-tree" + , "absent-config" + , "fetch-absent-archives" + , "fetch-absent-archives-symlinks" + , "fetch-absent-distdir-archive" + ] ] } } diff --git a/test/end-to-end/just-mr/absent-config.sh b/test/end-to-end/just-mr/absent-config.sh index 236325c8..9bfd41a6 100644 --- a/test/end-to-end/just-mr/absent-config.sh +++ b/test/end-to-end/just-mr/absent-config.sh @@ -26,6 +26,11 @@ readonly RCFILE="${TEST_TMPDIR}/mrrc.json" readonly OUT="${TEST_TMPDIR}/out" readonly LOCAL_REPO="${TEST_TMPDIR}/local-repository" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + mkdir -p "${LOCAL_REPO}" cd "${LOCAL_REPO}" mkdir src @@ -116,12 +121,12 @@ echo CONF=$("${JUST_MR}" --local-build-root "${LBR}" \ --rc "${RCFILE}" \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" grep eg "${OUT}/out.txt" diff --git a/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh b/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh index 48effd5c..182aa35f 100644 --- a/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh +++ b/test/end-to-end/just-mr/fetch-absent-archives-symlinks.sh @@ -25,6 +25,11 @@ readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT3="${TEST_TMPDIR}/out3" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" @@ -70,13 +75,13 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep x "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information @@ -112,7 +117,7 @@ echo "${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ --fetch-absent install -o "${OUT3}" 2>&1 grep xx "${OUT3}/out.txt" diff --git a/test/end-to-end/just-mr/fetch-absent-archives.sh b/test/end-to-end/just-mr/fetch-absent-archives.sh index a85c95ee..4f4f6704 100644 --- a/test/end-to-end/just-mr/fetch-absent-archives.sh +++ b/test/end-to-end/just-mr/fetch-absent-archives.sh @@ -29,6 +29,11 @@ readonly OUT3="${TEST_TMPDIR}/out3" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" readonly OUT_DISTDIR="${TEST_TMPDIR}/out4" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" @@ -68,13 +73,13 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information @@ -114,7 +119,7 @@ EOF "${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ --fetch-absent install -o "${OUT3}" 2>&1 grep 42 "${OUT3}/out.txt" @@ -153,7 +158,7 @@ echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 grep 42 "${OUT_NON_ABSENT}/out.txt" @@ -164,7 +169,7 @@ echo mkdir -p "${OUT_DISTDIR}" "${JUST_MR}" --norc --local-build-root "${LBR_FOR_FETCH}" \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ fetch -o "${OUT_DISTDIR}" 2>&1 FETCHED_CONTENT=$(git hash-object "${OUT_DISTDIR}"/data.tar) echo diff --git a/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh b/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh index ade94031..d7cf70ef 100644 --- a/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh +++ b/test/end-to-end/just-mr/fetch-absent-distdir-archive.sh @@ -26,6 +26,11 @@ readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out3" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" @@ -74,13 +79,13 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information @@ -121,7 +126,7 @@ echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 grep 42 "${OUT_NON_ABSENT}/out.txt" diff --git a/test/end-to-end/just-mr/fetch-absent-git-tree.sh b/test/end-to-end/just-mr/fetch-absent-git-tree.sh index fb5d516a..b0669b15 100644 --- a/test/end-to-end/just-mr/fetch-absent-git-tree.sh +++ b/test/end-to-end/just-mr/fetch-absent-git-tree.sh @@ -26,6 +26,11 @@ readonly OUT="${TEST_TMPDIR}/out" readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + mkdir work cd work touch ROOT @@ -61,12 +66,12 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" @@ -103,7 +108,7 @@ echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 grep 42 "${OUT_NON_ABSENT}/out.txt" diff --git a/test/end-to-end/just-mr/fetch-absent.sh b/test/end-to-end/just-mr/fetch-absent.sh index 30267998..41cde7a8 100644 --- a/test/end-to-end/just-mr/fetch-absent.sh +++ b/test/end-to-end/just-mr/fetch-absent.sh @@ -27,6 +27,11 @@ readonly OUT2="${TEST_TMPDIR}/out2" readonly OUT3="${TEST_TMPDIR}/out3" readonly OUT_NON_ABSENT="${TEST_TMPDIR}/out4" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + mkdir work cd work touch ROOT @@ -64,13 +69,13 @@ echo CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \ -L '["env", "PATH='"${PATH}"'"]' \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ --fetch-absent setup) cat $CONF echo "${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \ -L '["env", "PATH='"${PATH}"'"]' \ - -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" ${COMPAT} -o "${OUT}" 2>&1 grep 42 "${OUT}/out.txt" # As the last call of just-mr had --fetch-absent, all relevent information @@ -110,7 +115,7 @@ cat > targets/TARGETS <<'EOF' EOF "${JUST_MR}" --norc --local-build-root "${LBR}" \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ -L '["env", "PATH='"${PATH}"'"]' \ --just "${JUST}" \ --fetch-absent install -o "${OUT3}" 2>&1 @@ -150,7 +155,7 @@ cat repos.json echo "${JUST_MR}" --norc --local-build-root "${LBR_NON_ABSENT}" \ --remote-serve-address ${SERVE} \ - -r ${REMOTE_EXECUTION_ADDRESS} \ + -r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT} \ -L '["env", "PATH='"${PATH}"'"]' \ --just "${JUST}" \ install -o "${OUT_NON_ABSENT}" 2>&1 diff --git a/test/end-to-end/just-mr/stay-local.sh b/test/end-to-end/just-mr/stay-local.sh index 1e411780..531630db 100644 --- a/test/end-to-end/just-mr/stay-local.sh +++ b/test/end-to-end/just-mr/stay-local.sh @@ -29,11 +29,6 @@ readonly EMPTY="${TEST_TMPDIR}/empty-directory" readonly SERVER="${PWD}/utils/null-server" readonly SERVER_STATE="${TEST_TMPDIR}/server" -COMPAT="" -if [ "${COMPATIBLE:-}" = "YES" ]; then - COMPAT="--compatible" -fi - ARCHIVE_CONTENT=$(git hash-object src/data.tar) echo "Archive has content $ARCHIVE_CONTENT" @@ -84,7 +79,7 @@ cat repos.json echo mkdir -p "${LOG}" "${JUST_MR}" --norc --local-build-root "${LBR}" \ - -r "127.0.0.1:${port}" ${COMPAT} \ + -r "127.0.0.1:${port}" \ --log-limit 5 -f "${LOG}/log" \ --distdir ../src \ setup > conf.json @@ -96,14 +91,14 @@ echo cat $(cat conf.json) echo # As a distdir (local directory!) was provided with all needed files, -# no attempty should be made to contact the remote-execution endpoint +# no attempt should be made to contact the remote-execution endpoint echo [ -f "${SERVER_STATE}/access" ] && cat "${SERVER_STATE}/access" && exit 1 || : # The obtained configuraiton should be suitable for building, also remotely "${JUST}" install -C "$(cat conf.json)" -o "${OUT}" \ --local-build-root "${LBR}" \ - -r "${REMOTE_EXECUTION_ADDRESS}" $COMPAT 2>&1 + -r "${REMOTE_EXECUTION_ADDRESS}" 2>&1 echo cat "${OUT}/archive_id" [ $(cat "${OUT}/archive_id") = "${ARCHIVE_CONTENT}" ] @@ -114,7 +109,7 @@ echo # distdir is empty mkdir -p "${EMPTY}" "${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR2}" \ - -r "${REMOTE_EXECUTION_ADDRESS}" $COMPAT \ + -r "${REMOTE_EXECUTION_ADDRESS}" \ --distdir ${EMPTY} \ install -o "${OUT2}" 2>&1 cat "${OUT2}/archive_id" diff --git a/test/end-to-end/serve-service/TARGETS b/test/end-to-end/serve-service/TARGETS index 3ab54a14..f91cbd8e 100644 --- a/test/end-to-end/serve-service/TARGETS +++ b/test/end-to-end/serve-service/TARGETS @@ -173,7 +173,7 @@ } , "TESTS (unconfigured)": { "type": ["@", "rules", "test", "suite"] - , "arguments_config": ["TEST_COMPATIBLE_REMOTE", "TEST_STANDALONE_SERVE"] + , "arguments_config": ["TEST_STANDALONE_SERVE"] , "stage": [ { "type": "if" , "cond": {"type": "var", "name": "TEST_STANDALONE_SERVE"} @@ -196,12 +196,8 @@ , "describe" , "deduplication-of-serve-requests" , "failure-report" + , "serve-tree" ] - , { "type": "if" - , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"} - , "then": [] - , "else": ["serve-tree"] - } , [ ["./", "serve-archive-root", "TESTS"] , ["./", "serve-git-root", "TESTS"] , ["./", "serve-file-root", "TESTS"] diff --git a/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known-upload.sh b/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known-upload.sh index 502aaecb..5b8404ab 100644 --- a/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known-upload.sh +++ b/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known-upload.sh @@ -18,7 +18,7 @@ # This test checks that an absent root can successfully be made in the presence # of the serve endpoint in the situation where we already have the file # association (i.e., we know the unresolved root tree) and the serve endpoint -# does not know the archive. The upload can only happen in native mode. +# does not know the archive. # # The test archive contains symlinks to be resolved, which tests also the # resolved tree file association. @@ -98,35 +98,17 @@ rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as # absent. For a serve endpoint that does not have the archive blob available, # this will require uploading the locally-known root tree to remote CAS, from -# where the serve endpoint will pick it up. This can only happen in native mode. -if [ -z "${COMPAT}" ]; then - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent) - cat "${CONF}" - echo - test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent 2>&1 && exit 1 || : - echo Failed as expected -fi +# where the serve endpoint will pick it up. +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup absent) +cat "${CONF}" +echo +test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known.sh b/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known.sh index b882bc2d..4d20deb4 100644 --- a/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known.sh +++ b/test/end-to-end/serve-service/serve-archive-root/resolved-absent-known.sh @@ -96,8 +96,8 @@ TREE=$(jq -r '.repositories.present.workspace_root[1]' "${CONF}") rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as -# absent. This serve endpoint known already the archive, so it should be able to -# set it up even if in compatible mode. +# absent. This serve endpoint knows already the archive, so it should be able to +# set it up. ${JUST} gc --local-build-root ${LBR} 2>&1 ${JUST} gc --local-build-root ${LBR} 2>&1 diff --git a/test/end-to-end/serve-service/serve-archive-root/resolved-absent.sh b/test/end-to-end/serve-service/serve-archive-root/resolved-absent.sh index 07568390..fb31b959 100644 --- a/test/end-to-end/serve-service/serve-archive-root/resolved-absent.sh +++ b/test/end-to-end/serve-service/serve-archive-root/resolved-absent.sh @@ -106,32 +106,15 @@ cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" -# Check that serve can provide this tree as present in a clean build root. This -# can happen however only in native mode. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present) - cat "${CONF}" - echo - test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || : - echo Failed as expected -fi +# Check that serve can provide this tree as present in a clean build root. +rm -rf "${LBR}" +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup present) +cat "${CONF}" +echo +test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/resolved-present.sh b/test/end-to-end/serve-service/serve-archive-root/resolved-present.sh index d2ecf13c..bd9b17c9 100644 --- a/test/end-to-end/serve-service/serve-archive-root/resolved-present.sh +++ b/test/end-to-end/serve-service/serve-archive-root/resolved-present.sh @@ -15,8 +15,9 @@ ### -# This test checks 3 of the options to make a present root for an archive, where: -# - archive is in local distfile; +# This test checks 3 of the options to make a present root for an archive, +# where: +# - archive is in a local distfile; # - there is already a file association to the unresolved root tree; # - we receive the archive content from serve endpoint via the remote CAS. # @@ -99,48 +100,31 @@ cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" -# We now test if the serve endpoint can provide us the root. This can only -# happen in we're in native mode. -if [ -z "${COMPAT}" ]; then - - # In a clean build root, ask serve to set up the root for us, from scratch - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - - # Double-check the file association was created and root remains available - # without the remote endpoints - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# We now test if the serve endpoint can provide us the root. +# In a clean build root, ask serve to set up the root for us, from scratch +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" + +# Double-check the file association was created and root remains available +# without the remote endpoints +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known-upload.sh b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known-upload.sh index c27c8066..f087b3d6 100644 --- a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known-upload.sh +++ b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known-upload.sh @@ -18,7 +18,7 @@ # This test checks that an absent root can successfully be made in the presence # of the serve endpoint in the situation where we already have the file # association (i.e., we know the unresolved root tree) and the serve endpoint -# does not know the archive. The upload can only happen in native mode. +# does not know the archive. # # The test archive does not contain symlinks. ## @@ -96,35 +96,17 @@ rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as # absent. For a serve endpoint that does not have the archive blob available, # this will require uploading the locally-known root tree to remote CAS, from -# where the serve endpoint will pick it up. This can only happen in native mode. -if [ -z "${COMPAT}" ]; then - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent) - cat "${CONF}" - echo - test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent 2>&1 && exit 1 || : - echo Failed as expected -fi +# where the serve endpoint will pick it up. +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup absent) +cat "${CONF}" +echo +test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known.sh b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known.sh index 8be8aa0f..3f1fd7e2 100644 --- a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known.sh +++ b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent-known.sh @@ -94,8 +94,8 @@ TREE=$(jq -r '.repositories.present.workspace_root[1]' "${CONF}") rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as -# absent. This serve endpoint known already the archive, so it should be able to -# set it up even if in compatible mode. +# absent. This serve endpoint knows already the archive, so it should be able to +# set it up. ${JUST} gc --local-build-root ${LBR} 2>&1 ${JUST} gc --local-build-root ${LBR} 2>&1 diff --git a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent.sh b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent.sh index e304c9f2..46ed6c37 100644 --- a/test/end-to-end/serve-service/serve-archive-root/unresolved-absent.sh +++ b/test/end-to-end/serve-service/serve-archive-root/unresolved-absent.sh @@ -104,32 +104,15 @@ cat "${CONF}" echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" -# Check that serve can provide this tree as present in a clean build root. This -# can happen however only in native mode. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present) - cat "${CONF}" - echo - test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || : - echo Failed as expected -fi +# Check that serve can provide this tree as present in a clean build root. +rm -rf "${LBR}" +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup present) +cat "${CONF}" +echo +test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-archive-root/unresolved-present.sh b/test/end-to-end/serve-service/serve-archive-root/unresolved-present.sh index e25a72bf..ee1e1dc2 100644 --- a/test/end-to-end/serve-service/serve-archive-root/unresolved-present.sh +++ b/test/end-to-end/serve-service/serve-archive-root/unresolved-present.sh @@ -15,8 +15,9 @@ ### -# This test checks 3 of the options to make a present root for an archive, where: -# - archive is in local distfile; +# This test checks 3 of the options to make a present root for an archive, +# where: +# - archive is in a local distfile; # - there is already a file association to the unresolved root tree; # - we receive the archive content from serve endpoint via the remote CAS. # @@ -97,48 +98,31 @@ cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" -# We now test if the serve endpoint can provide us the root. This can only -# happen in we're in native mode. -if [ -z "${COMPAT}" ]; then - - # In a clean build root, ask serve to set up the root for us, from scratch - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - - # Double-check the file association was created and root remains available - # without the remote endpoints - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# We now test if the serve endpoint can provide us the root. +# In a clean build root, ask serve to set up the root for us, from scratch +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" + +# Double-check the file association was created and root remains available +# without the remote endpoints +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-distdir-root/absent.sh b/test/end-to-end/serve-service/serve-distdir-root/absent.sh index 241297c0..e44d10ea 100644 --- a/test/end-to-end/serve-service/serve-distdir-root/absent.sh +++ b/test/end-to-end/serve-service/serve-distdir-root/absent.sh @@ -119,32 +119,15 @@ echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" # Check that serve can provide now this tree as present in a clean build root. -# This can happen only in native mode. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present) - cat "${CONF}" - echo - test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || : - echo Failed as expected -fi +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup present) +cat "${CONF}" +echo +test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh b/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh index 18dff9e0..8696659b 100644 --- a/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh +++ b/test/end-to-end/serve-service/serve-distdir-root/foreign-file.sh @@ -78,7 +78,7 @@ mkdir -p "${OUT}" grep 'HELLO WORLD' "${OUT}/out.txt" -# also verify that the repo config has the repository abent +# also verify that the repo config has the repository absent CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" ${ENDPOINT_ARGS} setup) echo diff --git a/test/end-to-end/serve-service/serve-distdir-root/present.sh b/test/end-to-end/serve-service/serve-distdir-root/present.sh index 8a205182..b0bf7727 100644 --- a/test/end-to-end/serve-service/serve-distdir-root/present.sh +++ b/test/end-to-end/serve-service/serve-distdir-root/present.sh @@ -17,7 +17,7 @@ ### # This test checks 3 of the options to make a present root for a distidr # repository, where: -# - archives are in local distfile; +# - archives are in a local distfile; # - there is already a file association to the distdir root tree; # - we receive the distdir root from serve endpoint via the remote CAS. ## @@ -113,48 +113,31 @@ cat "${CONF}" echo test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" -# We now test if the serve endpoint can provide us the present root. This can -# only happen in we're in native mode. -if [ -z "${COMPAT}" ]; then - - # In a clean build root, ask serve to set up the root for us, from scratch - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - - # Double-check the file association was created and root remains available - # without the remote endpoints - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# We now test if the serve endpoint can provide us the present root. +# In a clean build root, ask serve to set up the root for us, from scratch +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" + +# Double-check the file association was created and root remains available +# without the remote endpoints +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-distdir-root/upload.sh b/test/end-to-end/serve-service/serve-distdir-root/upload.sh index ff33599d..841a51db 100644 --- a/test/end-to-end/serve-service/serve-distdir-root/upload.sh +++ b/test/end-to-end/serve-service/serve-distdir-root/upload.sh @@ -17,7 +17,6 @@ ### # This test checks that an absent distdir root can be successfully computed # locally and then uploaded to a serve endpoint that does not know the root. -# The upload can only happen in native mode. ## set -eu @@ -108,35 +107,17 @@ rm -rf "${DISTDIR}" # While keeping the file association, ask serve endpoint to provide the root as # absent. For a serve endpoint that does not have the archive blob available, # this will require uploading the locally-known root tree to remote CAS, from -# where the serve endpoint will pick it up. This can only happen in native mode. -if [ -z "${COMPAT}" ]; then - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent) - cat "${CONF}" - echo - test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" - -else - - echo --- - echo Checking expected failures - - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent 2>&1 && exit 1 || : - echo Failed as expected -fi +# where the serve endpoint will pick it up. +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup absent) +cat "${CONF}" +echo +test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-file-root/upload.sh b/test/end-to-end/serve-service/serve-file-root/upload.sh index 3362aa0d..01e35ab9 100644 --- a/test/end-to-end/serve-service/serve-file-root/upload.sh +++ b/test/end-to-end/serve-service/serve-file-root/upload.sh @@ -86,8 +86,6 @@ EOF # Setup an absent root from local path. Even if root is present, if a serve # endpoint is given then we try to set it up there as well. As this serve # endpoint does not know the tree, it will try to upload through the remote CAS. -# The upload succeeds if remote in native mode, but fails (non-fatally) in -# compatible mode. CONF=$("${JUST_MR}" --norc -C repos.json \ --just "${JUST}" \ --local-build-root "${LBR}" \ @@ -98,31 +96,15 @@ echo test $(jq -r '.repositories.present_file.workspace_root[1]' "${CONF}") = "${TREE}" # Check in a clean local build root that the serve endpoint now has the root -# tree. This can only work in native mode, where the root was actually uploaded. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent_git_tree) - cat "${CONF}" - echo - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup absent_git_tree 2>&1 && exit 1 || : - echo Failed as expected -fi +# tree. +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup absent_git_tree) +cat "${CONF}" +echo echo OK diff --git a/test/end-to-end/serve-service/serve-git-root/absent-upload.sh b/test/end-to-end/serve-service/serve-git-root/absent-upload.sh index 2fa54512..9ec592d0 100644 --- a/test/end-to-end/serve-service/serve-git-root/absent-upload.sh +++ b/test/end-to-end/serve-service/serve-git-root/absent-upload.sh @@ -16,8 +16,7 @@ ### # This test checks that an absent root known in a local checkout can be -# successfully uploaded to a serve endpoint. This can only succeed in native -# mode. +# successfully uploaded to a serve endpoint. ## set -eu @@ -79,30 +78,14 @@ EOF # Setup an absent root from a local checkout. For a serve endpoint that does # not have the commit available, this will upload the locally-known root tree -# to remote CAS, from where the serve endpoint will pick it up. This requires -# that the remotes are in native mode. -if [ -z "${COMPAT}" ]; then - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${SUBTREE}" - -else - - echo --- - echo Checking expected failures - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# to remote CAS, from where the serve endpoint will pick it up. +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${SUBTREE}" echo OK diff --git a/test/end-to-end/serve-service/serve-git-root/absent.sh b/test/end-to-end/serve-service/serve-git-root/absent.sh index ab808d53..97d2f7ea 100644 --- a/test/end-to-end/serve-service/serve-git-root/absent.sh +++ b/test/end-to-end/serve-service/serve-git-root/absent.sh @@ -71,8 +71,7 @@ EOF # Run the checks ## -# Compute absent root by asking serve to set it up from scratch. This works also -# in compatible mode. +# Compute absent root by asking serve to set it up from scratch. rm -rf "${LBR}" CONF=$("${JUST_MR}" --norc -C repos.json \ @@ -85,31 +84,15 @@ echo test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE_0}" # Check that serve can provide also a subtree of this tree as present in a clean -# build root. This can only happen if remotes are in native mode. -if [ -z "${COMPAT}" ]; then - - rm -rf "${LBR}" - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present) - cat "${CONF}" - echo - -else - - echo --- - echo Checking expected failures - - rm -rf "${LBR}" - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || : - echo Failed as expected -fi +# build root. +rm -rf "${LBR}" + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup present) +cat "${CONF}" +echo echo OK diff --git a/test/end-to-end/serve-service/serve-git-root/present.sh b/test/end-to-end/serve-service/serve-git-root/present.sh index 51723be2..0c359ec8 100644 --- a/test/end-to-end/serve-service/serve-git-root/present.sh +++ b/test/end-to-end/serve-service/serve-git-root/present.sh @@ -61,70 +61,54 @@ EOF # Run the checks ## -# Compute present root by asking the serve endpoint to set it up for us. This -# requires remotes in native mode. -if [ -z "${COMPAT}" ]; then - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" - - # Compute present root locally from now populated Git cache - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" - - # Check that the subdir is also working correctly - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - cat > repos.json <<EOF +# Compute present root by asking the serve endpoint to set it up for us. +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" + +# Compute present root locally from now populated Git cache +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" + +# Check that the subdir is also working correctly +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +cat > repos.json <<EOF { "repositories": - { "main": - { "repository": - { "type": "git" - , "commit": "$COMMIT_0" - , "repository": "http://non-existent.example.org/data.git" - , "branch": "master" - , "subdir": "repo" - } +{ "main": + { "repository": + { "type": "git" + , "commit": "$COMMIT_0" + , "repository": "http://non-existent.example.org/data.git" + , "branch": "master" + , "subdir": "repo" } } } +} EOF - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - -else - - echo --- - echo Checking expected failures - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo echo OK diff --git a/test/end-to-end/serve-service/serve-git-tree-root/absent.sh b/test/end-to-end/serve-service/serve-git-tree-root/absent.sh index 1e4b5f05..ddfe33d8 100644 --- a/test/end-to-end/serve-service/serve-git-tree-root/absent.sh +++ b/test/end-to-end/serve-service/serve-git-tree-root/absent.sh @@ -60,8 +60,7 @@ EOF # Run the checks ## -# Compute absent root by asking serve to set it up from scratch. This works also -# in compatible mode. +# Compute absent root by asking serve to set it up from scratch. rm -rf "${LBR}" CONF=$("${JUST_MR}" --norc -C repos.json \ diff --git a/test/end-to-end/serve-service/serve-git-tree-root/present.sh b/test/end-to-end/serve-service/serve-git-tree-root/present.sh index 2a704b52..4b05f9b5 100644 --- a/test/end-to-end/serve-service/serve-git-tree-root/present.sh +++ b/test/end-to-end/serve-service/serve-git-tree-root/present.sh @@ -16,7 +16,7 @@ ### # This test checks if we can make a present root for a Git-tree repository -# using the serve endpoint. This can only succeed in native mode. +# using the serve endpoint. ## set -eu @@ -59,43 +59,27 @@ EOF # Run the checks ## -# Compute present root by asking the serve endpoint to set it up for us. This -# requires remotes in native mode. -if [ -z "${COMPAT}" ]; then - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" - - # Compute present root locally from now populated Git cache - ${JUST} gc --local-build-root ${LBR} 2>&1 - ${JUST} gc --local-build-root ${LBR} 2>&1 - - CONF=$("${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - setup main) - cat "${CONF}" - echo - test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" - -else - - echo --- - echo Checking expected failures - - "${JUST_MR}" --norc -C repos.json \ - --just "${JUST}" \ - --local-build-root "${LBR}" \ - --log-limit 6 \ - ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || : - echo Failed as expected -fi +# Compute present root by asking the serve endpoint to set it up for us. +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + ${ENDPOINT_ARGS} setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" + +# Compute present root locally from now populated Git cache +${JUST} gc --local-build-root ${LBR} 2>&1 +${JUST} gc --local-build-root ${LBR} 2>&1 + +CONF=$("${JUST_MR}" --norc -C repos.json \ + --just "${JUST}" \ + --local-build-root "${LBR}" \ + --log-limit 6 \ + setup main) +cat "${CONF}" +echo +test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE_0}" echo OK diff --git a/test/end-to-end/serve-service/serve-tree.sh b/test/end-to-end/serve-service/serve-tree.sh index 75573964..d95a890b 100755 --- a/test/end-to-end/serve-service/serve-tree.sh +++ b/test/end-to-end/serve-service/serve-tree.sh @@ -32,7 +32,12 @@ mkdir -p "${DISTDIR}" cp src.tar "${DISTDIR}" HASH=$(git hash-object src.tar) -REMOTE="-r ${REMOTE_EXECUTION_ADDRESS}" +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + +REMOTE="-r ${REMOTE_EXECUTION_ADDRESS} ${COMPAT}" mkdir work cd work @@ -70,7 +75,7 @@ echo echo echo Local build "${JUST_MR}" --norc --local-build-root "${LBR_A}" --just "${JUST}" \ - --distdir "${DISTDIR}" build \ + --distdir "${DISTDIR}" ${COMPAT} build \ --log-limit 4 \ --dump-artifacts local.json 2>&1 |