From 0c0754a42f891d2e3b06cf696153191519ce5081 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 12:42:55 +0200 Subject: ["shell/test", "script"] Rename implicit dependencies ... and document at the appropriate places what can be overwritten by setting those targets. --- shell/test/RULES | 14 ++++++++-- shell/test/runner | 58 +++++++++++++++++++++++++++++++++++++++ shell/test/summarizer | 67 ++++++++++++++++++++++++++++++++++++++++++++++ shell/test/test_runner.sh | 58 --------------------------------------- shell/test/test_summary.py | 67 ---------------------------------------------- 5 files changed, 137 insertions(+), 127 deletions(-) create mode 100755 shell/test/runner create mode 100755 shell/test/summarizer delete mode 100755 shell/test/test_runner.sh delete mode 100755 shell/test/test_summary.py (limited to 'shell/test') diff --git a/shell/test/RULES b/shell/test/RULES index 117a060..c4f8569 100644 --- a/shell/test/RULES +++ b/shell/test/RULES @@ -20,6 +20,11 @@ , "TEST_TMPDIR. The test should not assume write permissions" , "outside the working directory and the TEST_TMPDIR." , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." + , "" + , "This running of the test is carried out by the implicit dependency" + , "on the target \"runner\". By setting this target in the target layer" + , "of this rues repository (instead of letting it default to the" + , "respective file), the shell test environment can be modified globally." ] , "name": [ "A name for the test, used in reporting, as well as for staging" @@ -39,6 +44,12 @@ { "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." + , "" + , "The individual test runs will be summarized by the implict dependency" + , "on the target \"summarizer\". By setting this target in the target" + , "in the target layer of this rues repository (instead of letting it" + , "default to the respective file) the layout of the summary can be" + , "changed globally." ] , "TEST_ENV": ["The environment for executing the test runner."] , "TIMEOUT_SCALE": @@ -72,8 +83,7 @@ , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" , "argument, this gives an easy way of defining test suites." ] - , "implicit": - {"runner": ["test_runner.sh"], "summarizer": ["test_summary.py"]} + , "implicit": {"runner": ["runner"], "summarizer": ["summarizer"]} , "imports": { "test-result": "test-result" , "action": "test-action" diff --git a/shell/test/runner b/shell/test/runner new file mode 100755 index 0000000..a99d5b7 --- /dev/null +++ b/shell/test/runner @@ -0,0 +1,58 @@ +#!/bin/sh +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ensure all required outputs are present +touch stdout +touch stderr +RESULT=UNKNOWN +echo "${RESULT}" > result +echo UNKNOWN > time-start +echo UNKNOWN > time-stop + +mkdir scratch +export TEST_TMPDIR=$(realpath scratch) +export TMPDIR="${TEST_TMPDIR}" + +# Change to the working directory; note: while unlikely, the test +# might not have test data, so we have to ensure the presence of +# the work directory. +mkdir -p work +cd work + +date +%s > ../time-start +# TODO: +# - proper wrapping with timeout +if sh ../test.sh > ../stdout 2> ../stderr +then + RESULT=PASS +else + RESULT=FAIL +fi +date +%s > ../time-stop + +# Ensure all the promissed output files in the work directory +# are present, even if the test failed to create them. +for f in "$@" +do + touch "./${f}" +done + +echo "${RESULT}" > ../result + +if [ "${RESULT}" '!=' PASS ] +then + exit 1; +fi diff --git a/shell/test/summarizer b/shell/test/summarizer new file mode 100755 index 0000000..0b5e656 --- /dev/null +++ b/shell/test/summarizer @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import time + +RESULTS = {} + +time_start = time.time() +time_stop = 0 + +for attempt in os.listdir("."): + if os.path.isdir(attempt): + with open(os.path.join(attempt, "result")) as f: + result = f.read().strip() + RESULTS[result] = RESULTS.get(result, []) + [int(attempt)] + try: + with open(os.path.join(attempt, "time-start")) as f: + time_start = min(time_start, float(f.read().strip())) + except: + pass + try: + with open(os.path.join(attempt, "time-stop")) as f: + time_stop = max(time_start, float(f.read().strip())) + except: + pass + +result = "UNKNOWN" +if set(RESULTS.keys()) <= set(["PASS", "FAIL"]): + if not RESULTS.get("FAIL"): + result = "PASS" + elif not RESULTS.get("PASS"): + result = "FAIL" + else: + result = "FLAKY" +with open("result", "w") as f: + f.write("%s\n" % (result,)) + +with open("time-start", "w") as f: + f.write("%d\n" % (time_start,)) +with open("time-stop", "w") as f: + f.write("%d\n" % (time_stop,)) + +with open("stdout", "w") as f: + f.write("Summary: %s\n\n" % (result,)) + f.write("PASS: %s\n" % (sorted(RESULTS.get("PASS", [])),)) + f.write("FAIL: %s\n" % (sorted(RESULTS.get("FAIL", [])),)) + RESULTS.pop("PASS", None) + RESULTS.pop("FAIL", None) + if RESULTS: + f.write("\nother results: %r\n" % (RESULTS,)) + +with open("stderr", "w") as f: + pass diff --git a/shell/test/test_runner.sh b/shell/test/test_runner.sh deleted file mode 100755 index a99d5b7..0000000 --- a/shell/test/test_runner.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# ensure all required outputs are present -touch stdout -touch stderr -RESULT=UNKNOWN -echo "${RESULT}" > result -echo UNKNOWN > time-start -echo UNKNOWN > time-stop - -mkdir scratch -export TEST_TMPDIR=$(realpath scratch) -export TMPDIR="${TEST_TMPDIR}" - -# Change to the working directory; note: while unlikely, the test -# might not have test data, so we have to ensure the presence of -# the work directory. -mkdir -p work -cd work - -date +%s > ../time-start -# TODO: -# - proper wrapping with timeout -if sh ../test.sh > ../stdout 2> ../stderr -then - RESULT=PASS -else - RESULT=FAIL -fi -date +%s > ../time-stop - -# Ensure all the promissed output files in the work directory -# are present, even if the test failed to create them. -for f in "$@" -do - touch "./${f}" -done - -echo "${RESULT}" > ../result - -if [ "${RESULT}" '!=' PASS ] -then - exit 1; -fi diff --git a/shell/test/test_summary.py b/shell/test/test_summary.py deleted file mode 100755 index 0b5e656..0000000 --- a/shell/test/test_summary.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import time - -RESULTS = {} - -time_start = time.time() -time_stop = 0 - -for attempt in os.listdir("."): - if os.path.isdir(attempt): - with open(os.path.join(attempt, "result")) as f: - result = f.read().strip() - RESULTS[result] = RESULTS.get(result, []) + [int(attempt)] - try: - with open(os.path.join(attempt, "time-start")) as f: - time_start = min(time_start, float(f.read().strip())) - except: - pass - try: - with open(os.path.join(attempt, "time-stop")) as f: - time_stop = max(time_start, float(f.read().strip())) - except: - pass - -result = "UNKNOWN" -if set(RESULTS.keys()) <= set(["PASS", "FAIL"]): - if not RESULTS.get("FAIL"): - result = "PASS" - elif not RESULTS.get("PASS"): - result = "FAIL" - else: - result = "FLAKY" -with open("result", "w") as f: - f.write("%s\n" % (result,)) - -with open("time-start", "w") as f: - f.write("%d\n" % (time_start,)) -with open("time-stop", "w") as f: - f.write("%d\n" % (time_stop,)) - -with open("stdout", "w") as f: - f.write("Summary: %s\n\n" % (result,)) - f.write("PASS: %s\n" % (sorted(RESULTS.get("PASS", [])),)) - f.write("FAIL: %s\n" % (sorted(RESULTS.get("FAIL", [])),)) - RESULTS.pop("PASS", None) - RESULTS.pop("FAIL", None) - if RESULTS: - f.write("\nother results: %r\n" % (RESULTS,)) - -with open("stderr", "w") as f: - pass -- cgit v1.2.3