WIP: Added support for repeating seeds from previous test run.

Author: LarsAsplund

tests/acceptance/artificial/verilog/run.py

@@ -65,7 +65,8 @@ def post_check(output_path):
     module.add_config("cfg", post_check=post_check)
     module = ui.library("lib").module("tb_same_sim_from_python_all_pass")
     module.add_config("cfg", post_check=post_check, attributes=dict(run_all_in_same_sim=True))
-   
+
+
 configure_tb_with_parameter_config()
 configure_tb_same_sim_all_pass(vu)
 lib.module("tb_same_sim_from_python_some_fail").set_attribute("run_all_in_same_sim", True)

tests/acceptance/artificial/vhdl/reference_test_history/0

@@ -27,6 +27,7 @@
     FileLocation,
     Attribute,
     LegacyAttribute,
+    _get_historic_seed,
 )
 from vunit.configuration import AttributeException
 from vunit.ostools import write_file
@@ -483,7 +484,7 @@ def test_error_on_global_attributes_from_python_on_tests(self, tempdir):
 
         test_bench = TestBench(design_unit)
         test = test_bench.get_test_case("Test 1")
-        self.assertRaises(AttributeException, test.set_attribute, "run_all_in_same_sim", True)    
+        self.assertRaises(AttributeException, test.set_attribute, "run_all_in_same_sim", True)
 
     @with_tempdir
     def test_test_information(self, tempdir):
@@ -904,6 +905,19 @@ def assert_has_tests(self, test_list, tests):
                 self.assertEqual(test1.name, test2)
                 self.assertEqual(test1.test_names, [test2])
 
+    def test_seed_from_test_history(self):
+        test_history = dict(test_suite1=dict(test1=dict(seed="11"), test2=dict(seed="12")), test_suite2=dict())
+
+        self.assertEqual(_get_historic_seed(test_history, test_suite_name="test_suite1"), "11")
+        self.assertEqual(_get_historic_seed(test_history, test_suite_name="test_suite1", test_name="test2"), "12")
+
+    def test_no_seed_from_test_history(self):
+        test_history = dict(test_suite1=dict(test1=dict(seed="11"), test2=dict(seed="12")), test_suite2=dict())
+
+        self.assertIsNone(_get_historic_seed(test_history, test_suite_name="foo"))
+        self.assertIsNone(_get_historic_seed(test_history, test_suite_name="test_suite1", test_name="test3"))
+        self.assertIsNone(_get_historic_seed(test_history, test_suite_name="test_suite2"))
+
     @staticmethod
     def create_tests(test_bench):
         """

tests/acceptance/artificial/vhdl/reference_test_history/1

@@ -289,7 +289,13 @@ def test_junit_report_with_some_skipped_tests(self):
     def test_junit_report_with_testcase_classname(self):
         report = self._new_report()
         report.add_result(
-            "test", PASSED, time=1.0, output_file_name=self.output_file_name, test_suite_name="test", start_time=0
+            "test",
+            PASSED,
+            time=1.0,
+            output_file_name=self.output_file_name,
+            test_suite_name="test",
+            start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "lib.entity",
@@ -298,6 +304,7 @@ def test_junit_report_with_testcase_classname(self):
             output_file_name=self.output_file_name,
             test_suite_name="lib.entity",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "lib.entity.test",
@@ -306,6 +313,7 @@ def test_junit_report_with_testcase_classname(self):
             output_file_name=self.output_file_name,
             test_suite_name="lib.entity.test",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "lib.entity.config.test",
@@ -314,6 +322,7 @@ def test_junit_report_with_testcase_classname(self):
             output_file_name=self.output_file_name,
             test_suite_name="lib.entity.config.test",
             start_time=0,
+            seed="0123456789abcdef",
         )
         root = ElementTree.fromstring(report.to_junit_xml_str())
         names = set(
@@ -372,6 +381,7 @@ def _report_with_all_passed_tests(self, output_file_name=None):
             output_file_name=output_file_name,
             test_suite_name="passed_test0",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test1",
@@ -380,6 +390,7 @@ def _report_with_all_passed_tests(self, output_file_name=None):
             output_file_name=output_file_name,
             test_suite_name="passed_test0",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.set_expected_num_tests(2)
         return report
@@ -394,6 +405,7 @@ def _report_with_missing_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test0",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test1",
@@ -402,6 +414,7 @@ def _report_with_missing_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test0",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.set_expected_num_tests(3)
         return report
@@ -416,6 +429,7 @@ def _report_with_some_failed_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="failed_test0",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test",
@@ -424,6 +438,7 @@ def _report_with_some_failed_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "failed_test1",
@@ -432,6 +447,7 @@ def _report_with_some_failed_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="failed_test1",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.set_expected_num_tests(3)
         return report
@@ -446,6 +462,7 @@ def _report_with_some_skipped_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "skipped_test",
@@ -454,6 +471,7 @@ def _report_with_some_skipped_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="skipped_test",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "failed_test",
@@ -462,6 +480,7 @@ def _report_with_some_skipped_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="failed_test",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.set_expected_num_tests(3)
         return report
@@ -476,6 +495,7 @@ def _report_with_mixed_length_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test0",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test1",
@@ -484,6 +504,7 @@ def _report_with_mixed_length_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test1",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test2",
@@ -492,6 +513,7 @@ def _report_with_mixed_length_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test2",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test3",
@@ -500,6 +522,7 @@ def _report_with_mixed_length_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test3",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.set_expected_num_tests(4)
         return report
@@ -514,6 +537,7 @@ def _report_with_long_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test0",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test1",
@@ -522,6 +546,7 @@ def _report_with_long_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test1",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test2",
@@ -530,6 +555,7 @@ def _report_with_long_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test2",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test3",
@@ -538,6 +564,7 @@ def _report_with_long_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test3",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test4",
@@ -546,6 +573,7 @@ def _report_with_long_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test4",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.add_result(
             "passed_test5",
@@ -554,6 +582,7 @@ def _report_with_long_tests(self):
             output_file_name=self.output_file_name,
             test_suite_name="passed_test5",
             start_time=0,
+            seed="0123456789abcdef",
         )
         report.set_expected_num_tests(6)
         return report

tests/unit/test_test_bench.py

@@ -256,3 +256,6 @@ def run(self, output_path, read_output):
         self.output_path = output_path
         self.read_output = read_output
         return self.run_side_effect(output_path=output_path, read_output=read_output)
+
+    def get_seed(self):
+        return "0123456789abcdef"

tests/unit/test_test_report.py

@@ -10,10 +10,12 @@
 
 from pathlib import Path
 from unittest import TestCase
-from tests.common import create_tempdir
+from time import sleep
+from tests.common import with_tempdir, create_tempdir
 from vunit.test.suites import TestRun
 from vunit.test.report import PASSED, SKIPPED, FAILED
 from vunit.sim_if import SimulatorInterface
+from vunit.configuration import Configuration
 
 
 class TestTestSuites(TestCase):
@@ -105,7 +107,7 @@ def _read_test_results(self, expected, contents):
                 elaborate_only=False,
                 test_suite_name=None,
                 test_cases=expected,
-                seed="seed"
+                seed="seed",
             )
             results = run._read_test_results(file_name=file_name)  # pylint: disable=protected-access
             self.assertEqual(results, expected)
@@ -180,11 +182,80 @@ def func():
                 elaborate_only=False,
                 test_suite_name=None,
                 test_cases=expected,
-                seed="seed"
+                seed="seed",
             )
 
             results = run._read_test_results(file_name=file_name)  # pylint: disable=protected-access
             self.assertEqual(
                 run._check_results(results, sim_ok),  # pylint: disable=protected-access
                 (waschecked, expected),
             )
+
+    @with_tempdir
+    def test_get_external_seed(self, tempdir):
+        design_unit = Entity(file_name=str(Path(tempdir) / "file.vhd"))
+        config = Configuration(name="config", design_unit=design_unit, sim_options=dict(seed="config_seed"))
+        run = TestRun(
+            simulator_if=None,
+            config=config,
+            elaborate_only=False,
+            test_suite_name=None,
+            test_cases=None,
+            seed="external_seed",
+        )
+
+        self.assertEqual(run.get_seed(), "external_seed")
+
+    @with_tempdir
+    def test_get_seed_from_config(self, tempdir):
+        design_unit = Entity(file_name=str(Path(tempdir) / "file.vhd"))
+        config = Configuration(name="config", design_unit=design_unit, sim_options=dict(seed="config_seed"))
+        run = TestRun(
+            simulator_if=None,
+            config=config,
+            elaborate_only=False,
+            test_suite_name=None,
+            test_cases=None,
+            seed=None,
+        )
+
+        self.assertEqual(run.get_seed(), "config_seed")
+
+    @with_tempdir
+    def test_get_generated_seed(self, tempdir):
+        design_unit = Entity(file_name=str(Path(tempdir) / "file.vhd"))
+        config = Configuration(name="config", design_unit=design_unit)
+        run = TestRun(
+            simulator_if=None,
+            config=config,
+            elaborate_only=False,
+            test_suite_name=None,
+            test_cases=None,
+            seed=None,
+        )
+
+        # Seed is "random" but do not change for an instance
+        seed = run.get_seed()
+        self.assertEqual(run.get_seed(), seed)
+
+        run = TestRun(
+            simulator_if=None,
+            config=config,
+            elaborate_only=False,
+            test_suite_name=None,
+            test_cases=None,
+            seed=None,
+        )
+
+        # New random value for a new instance if we let system time ellapse
+        sleep(0.1)
+        self.assertNotEqual(run.get_seed(), seed)
+
+class Entity(object):  # pylint: disable=too-many-instance-attributes
+    """
+    Mock Entity
+    """
+
+    def __init__(self, file_name):
+        self.generic_names = []
+        self.original_file_name = file_name