Add ability for tests to mark that they should be skipped

Author: alexanderbock

runner/main.py

@@ -211,7 +211,7 @@ def setup_argparse():
     except Exception as e:
       print(f"Test '{path}' failed with error: {e}")
     else:
-      for file in result.files:
+      for file in (result.files if result is not None else []):
         if submit_images:
           submit_image(result, hardware, timestamp, file, runner_id, submit_url)
         else:
@@ -233,6 +233,8 @@ def setup_argparse():
       except Exception as e:
         print(f"Test '{file}' failed with error: {e}")
         continue
+      if result is None:
+        continue
       for img in result.files:
         if submit_images:
           submit_image(result, hardware, timestamp, img, runner_id, submit_url)
@@ -258,6 +260,8 @@ def setup_argparse():
       except Exception as e:
         print(f"Test '{path}' failed with error: {e}")
         continue
+      if result is None:
+        continue
       for file in result.files:
         if submit_images:
           submit_image(result, hardware, timestamp, file, runner_id, submit_url)

runner/testsuite/openspace.py

@@ -132,6 +132,11 @@ def run_single_test(test_path, executable) -> TestResult:
   print(f"Running test: {test_path}")
   test = Test(test_path)
 
+  # Skip the test if the test-creator asked for it
+  if test.skipTest:
+    print(f"  Skipping test {test_path}")
+    return None
+
   start_time = time.perf_counter()
   print(f"  Starting OpenSpace (Profile: {test.profile})")
   process = subprocess.Popen(

runner/testsuite/test.py

@@ -68,6 +68,8 @@ def __init__(self, path: str):
     if content["commands"] is None:
       raise Exception(f"Missing 'commands' in test {path}")
 
+    self.skipTest = content.get("skip_test", False)
+
     self.instructions = []
     for command in content["commands"]:
       try: