From d7d7f9d796bc73c8f8cb2dbb6e58039a663f9ef3 Mon Sep 17 00:00:00 2001 From: Klaus Aehlig Date: Wed, 2 Mar 2022 15:12:28 +0100 Subject: Extend shell-test rules to optionally detect flakyness --- shell/test/test_summary.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 shell/test/test_summary.py (limited to 'shell/test/test_summary.py') diff --git a/shell/test/test_summary.py b/shell/test/test_summary.py new file mode 100755 index 0000000..cba8d50 --- /dev/null +++ b/shell/test/test_summary.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import os +import time + +RESULTS = {} + +time_start = time.time() +time_stop = 0 + +for attempt in os.listdir("."): + if os.path.isdir(attempt): + with open(os.path.join(attempt, "result")) as f: + result = f.read().strip() + RESULTS[result] = RESULTS.get(result, []) + [int(attempt)] + try: + with open(os.path.join(attempt, "time-start")) as f: + time_start = min(time_start, float(f.read().strip())) + except: + pass + try: + with open(os.path.join(attempt, "time-stop")) as f: + time_stop = max(time_start, float(f.read().strip())) + except: + pass + +result = "UNKNOWN" +if set(RESULTS.keys()) <= set(["PASS", "FAIL"]): + if not RESULTS.get("FAIL"): + result = "PASS" + elif not RESULTS.get("PASS"): + result = "FAIL" + else: + result = "FLAKY" +with open("result", "w") as f: + f.write("%s\n" % (result,)) + +with open("time-start", "w") as f: + f.write("%d\n" % (time_start,)) +with open("time-stop", "w") as f: + f.write("%d\n" % (time_stop,)) + +with open("stdout", "w") as f: + f.write("Summary: %s\n\n" % (result,)) + f.write("PASS: %s\n" % (sorted(RESULTS.get("PASS", [])),)) + f.write("FAIL: %s\n" % (sorted(RESULTS.get("FAIL", [])),)) + RESULTS.pop("PASS", None) + RESULTS.pop("FAIL", None) + if RESULTS: + f.write("\nother results: %r\n" % (RESULTS,)) + +with open("stderr", "w") as f: + pass -- cgit v1.2.3