From 0c0754a42f891d2e3b06cf696153191519ce5081 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 12:42:55 +0200 Subject: ["shell/test", "script"] Rename implicit dependencies ... and document at the appropriate places what can be overwritten by setting those targets. --- CC/test/RULES | 5 +++- shell/test/RULES | 14 ++++++++-- shell/test/runner | 58 +++++++++++++++++++++++++++++++++++++++ shell/test/summarizer | 67 ++++++++++++++++++++++++++++++++++++++++++++++ shell/test/test_runner.sh | 58 --------------------------------------- shell/test/test_summary.py | 67 ---------------------------------------------- 6 files changed, 141 insertions(+), 128 deletions(-) create mode 100755 shell/test/runner create mode 100755 shell/test/summarizer delete mode 100755 shell/test/test_runner.sh delete mode 100755 shell/test/test_summary.py diff --git a/CC/test/RULES b/CC/test/RULES index 8dba363..41507d7 100644 --- a/CC/test/RULES +++ b/CC/test/RULES @@ -30,7 +30,7 @@ , "implicit": { "defaults": [["./", "..", "defaults"]] , "runner": ["test_runner.py"] - , "summarizer": [["./", "../../shell/test", "test_summary.py"]] + , "summarizer": [["./", "../../shell/test", "summarizer"]] } , "field_doc": { "name": @@ -88,6 +88,9 @@ , "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." + , "" + , "Test runs are summarized by the [\"shell/test\", \"summarizer\"] that" + , "is also used by shell tests." ] , "TARGET_ARCH": [ "The architecture to build the test for." diff --git a/shell/test/RULES b/shell/test/RULES index 117a060..c4f8569 100644 --- a/shell/test/RULES +++ b/shell/test/RULES @@ -20,6 +20,11 @@ , "TEST_TMPDIR. The test should not assume write permissions" , "outside the working directory and the TEST_TMPDIR." , "For convenience, the environment variable TMPDIR is also set to TEST_TMPDIR." + , "" + , "This running of the test is carried out by the implicit dependency" + , "on the target \"runner\". By setting this target in the target layer" + , "of this rues repository (instead of letting it default to the" + , "respective file), the shell test environment can be modified globally." ] , "name": [ "A name for the test, used in reporting, as well as for staging" @@ -39,6 +44,12 @@ { "RUNS_PER_TEST": [ "The number of times the test should be run in order to detect flakyness." , "If set, no test action will be taken from cache." + , "" + , "The individual test runs will be summarized by the implict dependency" + , "on the target \"summarizer\". By setting this target in the target" + , "in the target layer of this rues repository (instead of letting it" + , "default to the respective file) the layout of the summary can be" + , "changed globally." ] , "TEST_ENV": ["The environment for executing the test runner."] , "TIMEOUT_SCALE": @@ -72,8 +83,7 @@ , "As the built-in \"install\" rule only takes the runfiles of its \"deps\"" , "argument, this gives an easy way of defining test suites." ] - , "implicit": - {"runner": ["test_runner.sh"], "summarizer": ["test_summary.py"]} + , "implicit": {"runner": ["runner"], "summarizer": ["summarizer"]} , "imports": { "test-result": "test-result" , "action": "test-action" diff --git a/shell/test/runner b/shell/test/runner new file mode 100755 index 0000000..a99d5b7 --- /dev/null +++ b/shell/test/runner @@ -0,0 +1,58 @@ +#!/bin/sh +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ensure all required outputs are present +touch stdout +touch stderr +RESULT=UNKNOWN +echo "${RESULT}" > result +echo UNKNOWN > time-start +echo UNKNOWN > time-stop + +mkdir scratch +export TEST_TMPDIR=$(realpath scratch) +export TMPDIR="${TEST_TMPDIR}" + +# Change to the working directory; note: while unlikely, the test +# might not have test data, so we have to ensure the presence of +# the work directory. +mkdir -p work +cd work + +date +%s > ../time-start +# TODO: +# - proper wrapping with timeout +if sh ../test.sh > ../stdout 2> ../stderr +then + RESULT=PASS +else + RESULT=FAIL +fi +date +%s > ../time-stop + +# Ensure all the promissed output files in the work directory +# are present, even if the test failed to create them. +for f in "$@" +do + touch "./${f}" +done + +echo "${RESULT}" > ../result + +if [ "${RESULT}" '!=' PASS ] +then + exit 1; +fi diff --git a/shell/test/summarizer b/shell/test/summarizer new file mode 100755 index 0000000..0b5e656 --- /dev/null +++ b/shell/test/summarizer @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import time + +RESULTS = {} + +time_start = time.time() +time_stop = 0 + +for attempt in os.listdir("."): + if os.path.isdir(attempt): + with open(os.path.join(attempt, "result")) as f: + result = f.read().strip() + RESULTS[result] = RESULTS.get(result, []) + [int(attempt)] + try: + with open(os.path.join(attempt, "time-start")) as f: + time_start = min(time_start, float(f.read().strip())) + except: + pass + try: + with open(os.path.join(attempt, "time-stop")) as f: + time_stop = max(time_start, float(f.read().strip())) + except: + pass + +result = "UNKNOWN" +if set(RESULTS.keys()) <= set(["PASS", "FAIL"]): + if not RESULTS.get("FAIL"): + result = "PASS" + elif not RESULTS.get("PASS"): + result = "FAIL" + else: + result = "FLAKY" +with open("result", "w") as f: + f.write("%s\n" % (result,)) + +with open("time-start", "w") as f: + f.write("%d\n" % (time_start,)) +with open("time-stop", "w") as f: + f.write("%d\n" % (time_stop,)) + +with open("stdout", "w") as f: + f.write("Summary: %s\n\n" % (result,)) + f.write("PASS: %s\n" % (sorted(RESULTS.get("PASS", [])),)) + f.write("FAIL: %s\n" % (sorted(RESULTS.get("FAIL", [])),)) + RESULTS.pop("PASS", None) + RESULTS.pop("FAIL", None) + if RESULTS: + f.write("\nother results: %r\n" % (RESULTS,)) + +with open("stderr", "w") as f: + pass diff --git a/shell/test/test_runner.sh b/shell/test/test_runner.sh deleted file mode 100755 index a99d5b7..0000000 --- a/shell/test/test_runner.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# ensure all required outputs are present -touch stdout -touch stderr -RESULT=UNKNOWN -echo "${RESULT}" > result -echo UNKNOWN > time-start -echo UNKNOWN > time-stop - -mkdir scratch -export TEST_TMPDIR=$(realpath scratch) -export TMPDIR="${TEST_TMPDIR}" - -# Change to the working directory; note: while unlikely, the test -# might not have test data, so we have to ensure the presence of -# the work directory. -mkdir -p work -cd work - -date +%s > ../time-start -# TODO: -# - proper wrapping with timeout -if sh ../test.sh > ../stdout 2> ../stderr -then - RESULT=PASS -else - RESULT=FAIL -fi -date +%s > ../time-stop - -# Ensure all the promissed output files in the work directory -# are present, even if the test failed to create them. -for f in "$@" -do - touch "./${f}" -done - -echo "${RESULT}" > ../result - -if [ "${RESULT}" '!=' PASS ] -then - exit 1; -fi diff --git a/shell/test/test_summary.py b/shell/test/test_summary.py deleted file mode 100755 index 0b5e656..0000000 --- a/shell/test/test_summary.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import time - -RESULTS = {} - -time_start = time.time() -time_stop = 0 - -for attempt in os.listdir("."): - if os.path.isdir(attempt): - with open(os.path.join(attempt, "result")) as f: - result = f.read().strip() - RESULTS[result] = RESULTS.get(result, []) + [int(attempt)] - try: - with open(os.path.join(attempt, "time-start")) as f: - time_start = min(time_start, float(f.read().strip())) - except: - pass - try: - with open(os.path.join(attempt, "time-stop")) as f: - time_stop = max(time_start, float(f.read().strip())) - except: - pass - -result = "UNKNOWN" -if set(RESULTS.keys()) <= set(["PASS", "FAIL"]): - if not RESULTS.get("FAIL"): - result = "PASS" - elif not RESULTS.get("PASS"): - result = "FAIL" - else: - result = "FLAKY" -with open("result", "w") as f: - f.write("%s\n" % (result,)) - -with open("time-start", "w") as f: - f.write("%d\n" % (time_start,)) -with open("time-stop", "w") as f: - f.write("%d\n" % (time_stop,)) - -with open("stdout", "w") as f: - f.write("Summary: %s\n\n" % (result,)) - f.write("PASS: %s\n" % (sorted(RESULTS.get("PASS", [])),)) - f.write("FAIL: %s\n" % (sorted(RESULTS.get("FAIL", [])),)) - RESULTS.pop("PASS", None) - RESULTS.pop("FAIL", None) - if RESULTS: - f.write("\nother results: %r\n" % (RESULTS,)) - -with open("stderr", "w") as f: - pass -- cgit v1.2.3 From 4fa5efa1f520fd58da16f09d74998c944bc3a14d Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 14:39:06 +0200 Subject: ["CC/auto", "config_file"] rename implicit target ... and mention it at the appropriate part of the documentation. While there, also provide a default TARGETS file. --- CC/auto/RULES | 7 ++++- CC/auto/TARGETS | 1 + CC/auto/config_runner.py | 76 ------------------------------------------------ CC/auto/runner | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 77 deletions(-) create mode 100644 CC/auto/TARGETS delete mode 100755 CC/auto/config_runner.py create mode 100755 CC/auto/runner diff --git a/CC/auto/RULES b/CC/auto/RULES index 0e156c9..860db01 100644 --- a/CC/auto/RULES +++ b/CC/auto/RULES @@ -1015,6 +1015,11 @@ , "target configuration. In the usual case, a target using this rule is" , "configured by depending on it from a target that uses the built-in" , "\"configure\" rule." + , "" + , "The actual generation of the header file from the template" + , "is done by the implicit dependency on the \"runner\" target which" + , "can be changed globally by setting this target in the" + , "target layer of this repository." ] , "field_doc": { "output": @@ -1045,7 +1050,7 @@ , "last": "last_list_entry" , "stage_singleton_field": ["", "stage_singleton_field"] } - , "implicit": {"runner": ["config_runner.py"]} + , "implicit": {"runner": ["runner"]} , "expression": { "type": "let*" , "bindings": diff --git a/CC/auto/TARGETS b/CC/auto/TARGETS new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/CC/auto/TARGETS @@ -0,0 +1 @@ +{} diff --git a/CC/auto/config_runner.py b/CC/auto/config_runner.py deleted file mode 100755 index f938c27..0000000 --- a/CC/auto/config_runner.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/python3 - -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import re -from sys import argv -from re import sub, match - -input_file = argv[1] -param_file = argv[2] -magic_string = argv[3] -at_only = argv[4] == "true" - - -with open(param_file) as f: - param = json.loads(f.read()) - -# In many cases, CMake simply defines some variables (without any associated -# value). We handle this situation by assigning to the boolean True the empty -# string. Note that no False value should be found, because the right way to set -# a variable to False in the TARGETS file is to *do not mention* that variable -# at all. -for k, v in param.items(): - if isinstance(v, bool): - param[k] = "" - -with open(input_file) as i: - with open("out", "w") as o: - for line in i.readlines(): - if x := re.search( - r"#(.*)(" + magic_string + r" )([ \t]*)([a-zA-Z0-9_]+)", line - ): - # get the VAR - key = x.groups()[-1] - if key in param: - line = sub( - f"{x.group()[1:]}", - f"{x.groups()[0]}define {x.groups()[2]}{key} {param[key]}", - line, - ) - else: - line = f"/* #undef {x.groups()[-1]} */\n" - if x := re.search( - r"#(.*)(" + magic_string + "01 )([ \t]*)([a-zA-Z0-9_]+)", line - ): - # get the VAR - key = x.groups()[-1] - line = sub( - f"{x.group()[1:]}", - f"{x.groups()[0]}define {x.groups()[2]}{key} " - + str(1 if key in param else 0), - line, - ) - if match("#[ \t]*define", line): - if x := re.search(r"@([a-zA-Z0-9-_]+)@", line): - key = x.groups()[0] - line = sub(x.group(), param.get(key, ""), line) - if not at_only: - if x := re.search(r"\${([a-zA-Z0-9-_]+)}", line): - key = x.groups()[0] - line = sub(r"\${" + key + r"}", param.get(key, ""), line) - - print(line, end="", file=o) diff --git a/CC/auto/runner b/CC/auto/runner new file mode 100755 index 0000000..f938c27 --- /dev/null +++ b/CC/auto/runner @@ -0,0 +1,76 @@ +#!/usr/bin/python3 + +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +from sys import argv +from re import sub, match + +input_file = argv[1] +param_file = argv[2] +magic_string = argv[3] +at_only = argv[4] == "true" + + +with open(param_file) as f: + param = json.loads(f.read()) + +# In many cases, CMake simply defines some variables (without any associated +# value). We handle this situation by assigning to the boolean True the empty +# string. Note that no False value should be found, because the right way to set +# a variable to False in the TARGETS file is to *do not mention* that variable +# at all. +for k, v in param.items(): + if isinstance(v, bool): + param[k] = "" + +with open(input_file) as i: + with open("out", "w") as o: + for line in i.readlines(): + if x := re.search( + r"#(.*)(" + magic_string + r" )([ \t]*)([a-zA-Z0-9_]+)", line + ): + # get the VAR + key = x.groups()[-1] + if key in param: + line = sub( + f"{x.group()[1:]}", + f"{x.groups()[0]}define {x.groups()[2]}{key} {param[key]}", + line, + ) + else: + line = f"/* #undef {x.groups()[-1]} */\n" + if x := re.search( + r"#(.*)(" + magic_string + "01 )([ \t]*)([a-zA-Z0-9_]+)", line + ): + # get the VAR + key = x.groups()[-1] + line = sub( + f"{x.group()[1:]}", + f"{x.groups()[0]}define {x.groups()[2]}{key} " + + str(1 if key in param else 0), + line, + ) + if match("#[ \t]*define", line): + if x := re.search(r"@([a-zA-Z0-9-_]+)@", line): + key = x.groups()[0] + line = sub(x.group(), param.get(key, ""), line) + if not at_only: + if x := re.search(r"\${([a-zA-Z0-9-_]+)}", line): + key = x.groups()[0] + line = sub(r"\${" + key + r"}", param.get(key, ""), line) + + print(line, end="", file=o) -- cgit v1.2.3 From 78540814ad421bc02a4f6c7396d81c38c95ef503 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Thu, 17 Aug 2023 14:08:22 +0200 Subject: ["CC/test", "test"] rename implict dependency ... and document at the appropriate place --- CC/test/RULES | 15 +++++++++-- CC/test/runner | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++ CC/test/test_runner.py | 69 -------------------------------------------------- 3 files changed, 82 insertions(+), 71 deletions(-) create mode 100755 CC/test/runner delete mode 100755 CC/test/test_runner.py diff --git a/CC/test/RULES b/CC/test/RULES index 41507d7..af413a9 100644 --- a/CC/test/RULES +++ b/CC/test/RULES @@ -29,7 +29,7 @@ ] , "implicit": { "defaults": [["./", "..", "defaults"]] - , "runner": ["test_runner.py"] + , "runner": ["runner"] , "summarizer": [["./", "../../shell/test", "summarizer"]] } , "field_doc": @@ -39,7 +39,18 @@ , "Used to name the test binary as well as for staging the test result" ] , "args": ["Command line arguments for the test binary"] - , "srcs": ["The sources of the test binary"] + , "srcs": + [ "The sources of the test binary" + , "" + , "The resulting test binary in an environment where it can assume" + , "that the environment variable TEST_TMPDIR points to a" + , "director that may be used exclusively by this test." + , "" + , "This running of the test is carried out by the implicit dependency" + , "on the target \"runner\". By setting this target in the target layer" + , "of this rues repository (instead of letting it default to the" + , "respective file), the C/C++ test environment can be modified globally." + ] , "private-hdrs": [ "Any additional header files that need to be present when compiling" , "the test binary." diff --git a/CC/test/runner b/CC/test/runner new file mode 100755 index 0000000..0647621 --- /dev/null +++ b/CC/test/runner @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import subprocess +import time + +time_start = time.time() +time_stop = 0 +result = "UNKNOWN" +stderr = "" +stdout = "" + + +def dump_results(): + with open("result", "w") as f: + f.write("%s\n" % (result, )) + with open("time-start", "w") as f: + f.write("%d\n" % (time_start, )) + with open("time-stop", "w") as f: + f.write("%d\n" % (time_stop, )) + with open("stdout", "w") as f: + f.write("%s\n" % (stdout, )) + with open("stderr", "w") as f: + f.write("%s\n" % (stderr, )) + + +dump_results() + +TEMP_DIR = os.path.realpath("scratch") +os.makedirs(TEMP_DIR, exist_ok=True) + +WORK_DIR = os.path.realpath("work") +os.makedirs(WORK_DIR, exist_ok=True) + +ENV = dict(os.environ, TEST_TMPDIR=TEMP_DIR) + +with open('test-launcher.json') as f: + test_launcher = json.load(f) + +with open('test-args.json') as f: + test_args = json.load(f) + +ret = subprocess.run(test_launcher + ["../test"] + test_args, + cwd=WORK_DIR, + env=ENV, + capture_output=True) + +time_stop = time.time() +result = "PASS" if ret.returncode == 0 else "FAIL" +stdout = ret.stdout.decode("utf-8") +stderr = ret.stderr.decode("utf-8") + +dump_results() + +if result != "PASS": exit(1) diff --git a/CC/test/test_runner.py b/CC/test/test_runner.py deleted file mode 100755 index 0647621..0000000 --- a/CC/test/test_runner.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import subprocess -import time - -time_start = time.time() -time_stop = 0 -result = "UNKNOWN" -stderr = "" -stdout = "" - - -def dump_results(): - with open("result", "w") as f: - f.write("%s\n" % (result, )) - with open("time-start", "w") as f: - f.write("%d\n" % (time_start, )) - with open("time-stop", "w") as f: - f.write("%d\n" % (time_stop, )) - with open("stdout", "w") as f: - f.write("%s\n" % (stdout, )) - with open("stderr", "w") as f: - f.write("%s\n" % (stderr, )) - - -dump_results() - -TEMP_DIR = os.path.realpath("scratch") -os.makedirs(TEMP_DIR, exist_ok=True) - -WORK_DIR = os.path.realpath("work") -os.makedirs(WORK_DIR, exist_ok=True) - -ENV = dict(os.environ, TEST_TMPDIR=TEMP_DIR) - -with open('test-launcher.json') as f: - test_launcher = json.load(f) - -with open('test-args.json') as f: - test_args = json.load(f) - -ret = subprocess.run(test_launcher + ["../test"] + test_args, - cwd=WORK_DIR, - env=ENV, - capture_output=True) - -time_stop = time.time() -result = "PASS" if ret.returncode == 0 else "FAIL" -stdout = ret.stdout.decode("utf-8") -stderr = ret.stderr.decode("utf-8") - -dump_results() - -if result != "PASS": exit(1) -- cgit v1.2.3