From 7208a562340074935a9cd119569d8007b6897777 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Fri, 30 May 2025 10:29:03 +0200 Subject: Add end-to-end test for serving complex targets Add a test asking serve to provide a target where both, the dependency as well as the action graph has both a wide node (a target with a large number of direct dependencies) as well as a deep node (a node where the chain of dependencies is long). That target is requested serveral times in parallel to verify that such targets can also be served under load. The process actually tested is the `just serve` process implicit to the "with serve" rule. --- test/end-to-end/serve-service/TARGETS | 15 +++++ test/end-to-end/serve-service/load-test.sh | 78 ++++++++++++++++++++++ test/end-to-end/serve-service/wide-deep-targets.py | 64 ++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 test/end-to-end/serve-service/load-test.sh create mode 100644 test/end-to-end/serve-service/wide-deep-targets.py (limited to 'test/end-to-end') diff --git a/test/end-to-end/serve-service/TARGETS b/test/end-to-end/serve-service/TARGETS index 4830017c..f31720a6 100644 --- a/test/end-to-end/serve-service/TARGETS +++ b/test/end-to-end/serve-service/TARGETS @@ -159,6 +159,20 @@ , "repos": ["interruption-test (data)"] , "keep-dirs": ["out"] } +, "load test (data)": + { "type": "generic" + , "outs": ["ROOT", "TARGETS"] + , "cmds": ["touch ROOT", "python3 wide-deep-targets.py"] + , "deps": ["wide-deep-targets.py"] + } +, "load test": + { "type": ["end-to-end", "with serve"] + , "name": ["load-test"] + , "test": ["load-test.sh"] + , "deps": [["", "mr-tool-under-test"], ["", "tool-under-test"]] + , "repos": ["load test (data)"] + , "keep-dirs": ["out"] + } , "serve-many-targets (data)": { "type": "install" , "files": {"ROOT": "data/ROOT", "TARGETS": "data/targets/TARGETS.transform"} @@ -191,6 +205,7 @@ , "describe" , "failure-report" , "interruption-test" + , "load test" , "serve-export-deps" , "serve-many-targets" , "serve-query-target-cache-value" diff --git a/test/end-to-end/serve-service/load-test.sh b/test/end-to-end/serve-service/load-test.sh new file mode 100644 index 00000000..07773388 --- /dev/null +++ b/test/end-to-end/serve-service/load-test.sh @@ -0,0 +1,78 @@ +#!/bin/sh +# Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +env + +readonly JUST="${PWD}/bin/tool-under-test" +readonly JUST_MR="${PWD}/bin/mr-tool-under-test" +readonly LBR="${TEST_TMPDIR}/local-build-root" +readonly OUT="${PWD}/out" +mkdir -p "${OUT}" + + +COMPAT="" +if [ "${COMPATIBLE:-}" = "YES" ]; then + COMPAT="--compatible" +fi + +mkdir work +cd work +touch ROOT + +cat > repos.json <&1 & +pid="$!" +pids="${pids} ${pid}" +done +echo "Waiting for processes ${pids}" +for pid in ${pids}; do + wait $pid || echo "$pid has exit code $?" +done +echo done +echo + +# Sanity check: files contain some numbers +for i in `seq 1 2` +do grep 123 "${OUT}/out${i}/deep" + grep 123 "${OUT}/out${i}/wide" +done + +echo +echo OK diff --git a/test/end-to-end/serve-service/wide-deep-targets.py b/test/end-to-end/serve-service/wide-deep-targets.py new file mode 100644 index 00000000..13e2a81c --- /dev/null +++ b/test/end-to-end/serve-service/wide-deep-targets.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright 2025 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from typing import Any, Dict, cast + +SIZE=1000 + +deep : Dict[str, Any] = ( + {"deep%d (unexported)" % (i,) : + {"type": "generic", + "deps": ["deep%d" % (i-1,)] if i > 0 else [], + "outs": ["out%d" % (i,)], + "cmds": ["cat out%d > out%d" % (i-1, i) + if i > 0 else "true", + "echo %d >> out%d" % (i, i)]} + for i in range(SIZE)} + | {"deep%d" % (i,) : + {"type": "export", "target": "deep%d (unexported)" %(i,)} + for i in range(SIZE)} +) + +wide : Dict[str, Any] = ( + cast(Dict[str, Any], + {"wide": + {"type": "generic", + "outs": ["out"], + "cmds": ["for f in $(ls wide* | sort) ; do cat $f >> out ; done"], + "deps": ["wide%d" % (i,) for i in range(SIZE)] + }}) + | cast(Dict[str, Any], + {"wide%d (unexported)" % (i,) : + {"type": "generic", + "outs": ["wide%d" % (i,)], + "cmds": ["echo %d > wide%d" % (i,i)]} + for i in range(SIZE)}) + | cast(Dict[str, Any], + {"wide%d" % (i,) : + {"type": "export", "target": "wide%d (unexported)" %(i,)} + for i in range(SIZE)}) +) + +total : Dict[str, Any] = ( + { "": {"type": "export", "target": "all"}, + "all": {"type": "install", + "files": {"deep": "deep%d" % (SIZE-1,), + "wide": "wide"}}} + | deep | wide) + +with open("TARGETS", "w") as f: + json.dump(total, f) -- cgit v1.2.3