test remote runners: Add timeout for running execute and serve

The runners used in tests that rely on execution or serve
endpoints to exist can get stuck waiting for these to become
online if for some reason they cannot be set up. This commit fixes
this issue by setting a reasonable timeout, after which we fail
gracefully.

4 files changed, 34 insertions, 2 deletions

diff --git a/test/end-to-end/with_remote_test_runner.py b/test/end-to-end/with_remote_test_runner.py
index 31601272..079f627b 100755
--- a/test/end-to-end/with_remote_test_runner.py
+++ b/test/end-to-end/with_remote_test_runner.py
@@ -74,7 +74,7 @@ if not custom_remote:
         print(f"Warning: removing unexpected info file {REMOTE_INFO}")
         os.remove(REMOTE_INFO)
 
-    PATH=subprocess.run(
+    PATH = subprocess.run(
         ["env", "--", "sh", "-c", "echo -n $PATH"],
         stdout=subprocess.PIPE,
     ).stdout.decode('utf-8')
@@ -104,7 +104,15 @@ if not custom_remote:
         stderr=remotestderr,
     )
 
+    timeout: int = 30
     while not os.path.exists(REMOTE_INFO):
+        if timeout == 0:
+            result = "FAIL"
+            stdout = "Failed to start execution service"
+            remote_proc.terminate()
+            dump_results()
+            exit(1)
+        timeout -= 1
         time.sleep(1)
 
     with open(REMOTE_INFO) as f:
diff --git a/test/end-to-end/with_serve_test_runner.py b/test/end-to-end/with_serve_test_runner.py
index a71758fd..a7e7861b 100755
--- a/test/end-to-end/with_serve_test_runner.py
+++ b/test/end-to-end/with_serve_test_runner.py
@@ -253,7 +253,15 @@ serve_proc = subprocess.Popen(
     stderr=servestderr,
 )
 
+timeout: int = 30
 while not os.path.exists(SERVE_INFO):
+    if timeout == 0:
+        result = "FAIL"
+        stdout = "Failed to start serve service"
+        serve_proc.terminate()
+        dump_results()
+        exit(1)
+    timeout -= 1
     time.sleep(1)
 
 with open(SERVE_INFO) as f:
diff --git a/test/utils/remote_execution/test_runner.py b/test/utils/remote_execution/test_runner.py
index 8c22c99e..fb3e50ab 100755
--- a/test/utils/remote_execution/test_runner.py
+++ b/test/utils/remote_execution/test_runner.py
@@ -56,7 +56,7 @@ if os.path.exists(REMOTE_INFO):
     print(f"Warning: removing unexpected info file {REMOTE_INFO}")
     os.remove(REMOTE_INFO)
 
-PATH=subprocess.run(
+PATH = subprocess.run(
     ["env", "--", "sh", "-c", "echo -n $PATH"],
     stdout=subprocess.PIPE,
 ).stdout.decode('utf-8')
@@ -89,7 +89,15 @@ remote_proc = subprocess.Popen(
     stderr=remotestderr,
 )
 
+timeout: int = 30
 while not os.path.exists(REMOTE_INFO):
+    if timeout == 0:
+        result = "FAIL"
+        stdout = "Failed to start execution service"
+        remote_proc.terminate()
+        dump_results()
+        exit(1)
+    timeout -= 1
     time.sleep(1)
 
 with open(REMOTE_INFO) as f:
diff --git a/test/utils/serve_service/test_runner.py b/test/utils/serve_service/test_runner.py
index 01d5c0c8..3b6c4fdc 100755
--- a/test/utils/serve_service/test_runner.py
+++ b/test/utils/serve_service/test_runner.py
@@ -123,7 +123,15 @@ serve_proc = subprocess.Popen(
     stderr=servestderr,
 )
 
+timeout: int = 30
 while not os.path.exists(REMOTE_SERVE_INFO):
+    if timeout == 0:
+        result = "FAIL"
+        stdout = "Failed to start serve service"
+        serve_proc.terminate()
+        dump_results()
+        exit(1)
+    timeout -= 1
     time.sleep(1)
 
 with open(REMOTE_SERVE_INFO) as f: