WIP: Add support for repeating tests with different seeds.

Author: LarsAsplund

vunit/ui/__init__.py

@@ -749,7 +749,8 @@ def _main(self, post_run):
         if self._args.compile:
             return self._main_compile_only()
 
-        all_ok = self._main_run(post_run)
+        all_ok = self._main_run(post_run, n_iterations=self._args.repeat + 1)
+
         return all_ok
 
     def _create_simulator_if(self):
@@ -770,7 +771,7 @@ def _create_simulator_if(self):
 
         return self._simulator_class.from_args(args=self._args, output_path=self._simulator_output_path)
 
-    def _main_run(self, post_run):
+    def _main_run(self, post_run, n_iterations):
         """
         Main with running tests
         """
@@ -779,30 +780,35 @@ def _main_run(self, post_run):
         self._compile(simulator_if)
         print()
 
-        start_time = ostools.get_time()
-        report = TestReport(printer=self._printer)
+        all_ok = True
+        for iteration in range(1, n_iterations + 1):
+            start_time = ostools.get_time()
+            report = TestReport(printer=self._printer)
 
-        try:
-            self._run_test(test_list, report)
-        except KeyboardInterrupt:
-            print()
-            LOGGER.debug("_main: Caught Ctrl-C shutting down")
-        finally:
-            del test_list
+            try:
+                self._run_test(test_list, report, iteration)
+            except KeyboardInterrupt:
+                print()
+                LOGGER.debug("_main: Caught Ctrl-C shutting down")
+            finally:
+                if (sys.exc_info()[0] is not None) or (iteration == n_iterations):
+                    del test_list
 
-        report.set_real_total_time(ostools.get_time() - start_time)
-        report.print_str()
+            report.set_real_total_time(ostools.get_time() - start_time)
+            report.print_str()
 
-        if post_run is not None:
-            post_run(results=Results(self._output_path, simulator_if, report))
+            if post_run is not None:
+                post_run(results=Results(self._output_path, simulator_if, report))
+
+            all_ok &= report.all_ok()
 
         del simulator_if
 
         if self._args.xunit_xml is not None:
             xml = report.to_junit_xml_str(self._args.xunit_xml_format)
             ostools.write_file(self._args.xunit_xml, xml)
 
-        return report.all_ok()
+        return all_ok
 
     def _main_list_only(self):
         """
@@ -938,7 +944,7 @@ def _get_testbench_files(self, simulator_if: Union[None, SimulatorInterface]):
             for file_name in tb_file_names
         ]
 
-    def _run_test(self, test_cases, report):
+    def _run_test(self, test_cases, report, iteration):
         """
         Run the test suites and return the report
         """
@@ -950,9 +956,14 @@ def _run_test(self, test_cases, report):
         else:
             verbosity = TestRunner.VERBOSITY_NORMAL
 
+        full_test_output_path = Path(self._output_path) / TEST_OUTPUT_PATH
+        if iteration > 1:
+            full_test_output_path /= f"rep_{iteration - 1}"
+        full_test_output_path = str(full_test_output_path)
+
         runner = TestRunner(
             report,
-            str(Path(self._output_path) / TEST_OUTPUT_PATH),
+            full_test_output_path,
             verbosity=verbosity,
             num_threads=self._args.num_threads,
             fail_fast=self._args.fail_fast,

vunit/vunit_cli.py

@@ -252,6 +252,13 @@ def _create_argument_parser(description=None, for_documentation=False):
         help="Base seed provided to the simulation. Must be 16 hex digits. Default seed is generated from system time.",
     )
 
+    parser.add_argument(
+        "-r",
+        "--repeat",
+        type=nonnegative_int,
+        default=0,
+        help="Number of times the tests are repeated with new seed values (unless --seed is set). Default: 0. ",
+    )
     SIMULATOR_FACTORY.add_arguments(parser)
 
     return parser