Skip to content

Commit 7e210e5

Browse files
LarsAsplundLarsAsplund
committed
WIP: Added support for repeating seeds from previous test run.
1 parent 6d416fb commit 7e210e5

File tree

14 files changed

+173
-34
lines changed

14 files changed

+173
-34
lines changed

‎tests/acceptance/artificial/verilog/run.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ def post_check(output_path):
6565
module.add_config("cfg", post_check=post_check)
6666
module = ui.library("lib").module("tb_same_sim_from_python_all_pass")
6767
module.add_config("cfg", post_check=post_check, attributes=dict(run_all_in_same_sim=True))
68-
68+
69+
6970
configure_tb_with_parameter_config()
7071
configure_tb_same_sim_all_pass(vu)
7172
lib.module("tb_same_sim_from_python_some_fail").set_attribute("run_all_in_same_sim", True)

‎tests/acceptance/artificial/vhdl/reference_test_history/0‎

0 Bytes
Binary file not shown.

‎tests/acceptance/artificial/vhdl/reference_test_history/1‎

110 Bytes
Binary file not shown.

‎tests/unit/test_test_bench.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,7 @@ def test_error_on_global_attributes_from_python_on_tests(self, tempdir):
483483

484484
test_bench = TestBench(design_unit)
485485
test = test_bench.get_test_case("Test 1")
486-
self.assertRaises(AttributeException, test.set_attribute, "run_all_in_same_sim", True)
486+
self.assertRaises(AttributeException, test.set_attribute, "run_all_in_same_sim", True)
487487

488488
@with_tempdir
489489
def test_test_information(self, tempdir):

‎tests/unit/test_test_report.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,13 @@ def test_junit_report_with_some_skipped_tests(self):
289289
def test_junit_report_with_testcase_classname(self):
290290
report = self._new_report()
291291
report.add_result(
292-
"test", PASSED, time=1.0, output_file_name=self.output_file_name, test_suite_name="test", start_time=0
292+
"test",
293+
PASSED,
294+
time=1.0,
295+
output_file_name=self.output_file_name,
296+
test_suite_name="test",
297+
start_time=0,
298+
seed="0123456789abcdef",
293299
)
294300
report.add_result(
295301
"lib.entity",
@@ -298,6 +304,7 @@ def test_junit_report_with_testcase_classname(self):
298304
output_file_name=self.output_file_name,
299305
test_suite_name="lib.entity",
300306
start_time=0,
307+
seed="0123456789abcdef",
301308
)
302309
report.add_result(
303310
"lib.entity.test",
@@ -306,6 +313,7 @@ def test_junit_report_with_testcase_classname(self):
306313
output_file_name=self.output_file_name,
307314
test_suite_name="lib.entity.test",
308315
start_time=0,
316+
seed="0123456789abcdef",
309317
)
310318
report.add_result(
311319
"lib.entity.config.test",
@@ -314,6 +322,7 @@ def test_junit_report_with_testcase_classname(self):
314322
output_file_name=self.output_file_name,
315323
test_suite_name="lib.entity.config.test",
316324
start_time=0,
325+
seed="0123456789abcdef",
317326
)
318327
root = ElementTree.fromstring(report.to_junit_xml_str())
319328
names = set(
@@ -372,6 +381,7 @@ def _report_with_all_passed_tests(self, output_file_name=None):
372381
output_file_name=output_file_name,
373382
test_suite_name="passed_test0",
374383
start_time=0,
384+
seed="0123456789abcdef",
375385
)
376386
report.add_result(
377387
"passed_test1",
@@ -380,6 +390,7 @@ def _report_with_all_passed_tests(self, output_file_name=None):
380390
output_file_name=output_file_name,
381391
test_suite_name="passed_test0",
382392
start_time=0,
393+
seed="0123456789abcdef",
383394
)
384395
report.set_expected_num_tests(2)
385396
return report
@@ -394,6 +405,7 @@ def _report_with_missing_tests(self):
394405
output_file_name=self.output_file_name,
395406
test_suite_name="passed_test0",
396407
start_time=0,
408+
seed="0123456789abcdef",
397409
)
398410
report.add_result(
399411
"passed_test1",
@@ -402,6 +414,7 @@ def _report_with_missing_tests(self):
402414
output_file_name=self.output_file_name,
403415
test_suite_name="passed_test0",
404416
start_time=0,
417+
seed="0123456789abcdef",
405418
)
406419
report.set_expected_num_tests(3)
407420
return report
@@ -416,6 +429,7 @@ def _report_with_some_failed_tests(self):
416429
output_file_name=self.output_file_name,
417430
test_suite_name="failed_test0",
418431
start_time=0,
432+
seed="0123456789abcdef",
419433
)
420434
report.add_result(
421435
"passed_test",
@@ -424,6 +438,7 @@ def _report_with_some_failed_tests(self):
424438
output_file_name=self.output_file_name,
425439
test_suite_name="passed_test",
426440
start_time=0,
441+
seed="0123456789abcdef",
427442
)
428443
report.add_result(
429444
"failed_test1",
@@ -432,6 +447,7 @@ def _report_with_some_failed_tests(self):
432447
output_file_name=self.output_file_name,
433448
test_suite_name="failed_test1",
434449
start_time=0,
450+
seed="0123456789abcdef",
435451
)
436452
report.set_expected_num_tests(3)
437453
return report
@@ -446,6 +462,7 @@ def _report_with_some_skipped_tests(self):
446462
output_file_name=self.output_file_name,
447463
test_suite_name="passed_test",
448464
start_time=0,
465+
seed="0123456789abcdef",
449466
)
450467
report.add_result(
451468
"skipped_test",
@@ -454,6 +471,7 @@ def _report_with_some_skipped_tests(self):
454471
output_file_name=self.output_file_name,
455472
test_suite_name="skipped_test",
456473
start_time=0,
474+
seed="0123456789abcdef",
457475
)
458476
report.add_result(
459477
"failed_test",
@@ -462,6 +480,7 @@ def _report_with_some_skipped_tests(self):
462480
output_file_name=self.output_file_name,
463481
test_suite_name="failed_test",
464482
start_time=0,
483+
seed="0123456789abcdef",
465484
)
466485
report.set_expected_num_tests(3)
467486
return report
@@ -476,6 +495,7 @@ def _report_with_mixed_length_tests(self):
476495
output_file_name=self.output_file_name,
477496
test_suite_name="passed_test0",
478497
start_time=0,
498+
seed="0123456789abcdef",
479499
)
480500
report.add_result(
481501
"passed_test1",
@@ -484,6 +504,7 @@ def _report_with_mixed_length_tests(self):
484504
output_file_name=self.output_file_name,
485505
test_suite_name="passed_test1",
486506
start_time=0,
507+
seed="0123456789abcdef",
487508
)
488509
report.add_result(
489510
"passed_test2",
@@ -492,6 +513,7 @@ def _report_with_mixed_length_tests(self):
492513
output_file_name=self.output_file_name,
493514
test_suite_name="passed_test2",
494515
start_time=0,
516+
seed="0123456789abcdef",
495517
)
496518
report.add_result(
497519
"passed_test3",
@@ -500,6 +522,7 @@ def _report_with_mixed_length_tests(self):
500522
output_file_name=self.output_file_name,
501523
test_suite_name="passed_test3",
502524
start_time=0,
525+
seed="0123456789abcdef",
503526
)
504527
report.set_expected_num_tests(4)
505528
return report
@@ -514,6 +537,7 @@ def _report_with_long_tests(self):
514537
output_file_name=self.output_file_name,
515538
test_suite_name="passed_test0",
516539
start_time=0,
540+
seed="0123456789abcdef",
517541
)
518542
report.add_result(
519543
"passed_test1",
@@ -522,6 +546,7 @@ def _report_with_long_tests(self):
522546
output_file_name=self.output_file_name,
523547
test_suite_name="passed_test1",
524548
start_time=0,
549+
seed="0123456789abcdef",
525550
)
526551
report.add_result(
527552
"passed_test2",
@@ -530,6 +555,7 @@ def _report_with_long_tests(self):
530555
output_file_name=self.output_file_name,
531556
test_suite_name="passed_test2",
532557
start_time=0,
558+
seed="0123456789abcdef",
533559
)
534560
report.add_result(
535561
"passed_test3",
@@ -538,6 +564,7 @@ def _report_with_long_tests(self):
538564
output_file_name=self.output_file_name,
539565
test_suite_name="passed_test3",
540566
start_time=0,
567+
seed="0123456789abcdef",
541568
)
542569
report.add_result(
543570
"passed_test4",
@@ -546,6 +573,7 @@ def _report_with_long_tests(self):
546573
output_file_name=self.output_file_name,
547574
test_suite_name="passed_test4",
548575
start_time=0,
576+
seed="0123456789abcdef",
549577
)
550578
report.add_result(
551579
"passed_test5",
@@ -554,6 +582,7 @@ def _report_with_long_tests(self):
554582
output_file_name=self.output_file_name,
555583
test_suite_name="passed_test5",
556584
start_time=0,
585+
seed="0123456789abcdef",
557586
)
558587
report.set_expected_num_tests(6)
559588
return report

‎tests/unit/test_test_runner.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,3 +256,6 @@ def run(self, output_path, read_output):
256256
self.output_path = output_path
257257
self.read_output = read_output
258258
return self.run_side_effect(output_path=output_path, read_output=read_output)
259+
260+
def get_seed(self):
261+
return "0123456789abcdef"

‎tests/unit/test_test_suites.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ def _read_test_results(self, expected, contents):
105105
elaborate_only=False,
106106
test_suite_name=None,
107107
test_cases=expected,
108-
seed="seed"
108+
seed="seed",
109109
)
110110
results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access
111111
self.assertEqual(results, expected)
@@ -180,7 +180,7 @@ def func():
180180
elaborate_only=False,
181181
test_suite_name=None,
182182
test_cases=expected,
183-
seed="seed"
183+
seed="seed",
184184
)
185185

186186
results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access

‎vunit/test/bench.py‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,25 @@ def get_default_config(self):
7474
raise RuntimeError(f"Test bench {self.library_name!s}.{self.name!s} has individually configured tests")
7575
return self._configs[DEFAULT_NAME]
7676

77+
@staticmethod
78+
def _get_historic_seed(test_history, test_suite_name, test_name=None):
79+
"""Return seed from test history or None if no history exists."""
80+
if test_suite_name not in test_history:
81+
return None
82+
83+
# If there are multiple tests running in the same simulation, they all have the same seed so we take the
84+
# first test.
85+
if not test_name:
86+
if not test_history[test_suite_name]: # This is a precaution. Should never happen.
87+
return None
88+
89+
test_name = next(iter(test_history[test_suite_name]))
90+
91+
if test_name not in test_history[test_suite_name]:
92+
return None
93+
94+
return test_history[test_suite_name][test_name]["seed"]
95+
7796
def create_tests(self, simulator_if, seed, elaborate_only, test_list=None):
7897
"""
7998
Create all test cases from this test bench
@@ -84,28 +103,45 @@ def create_tests(self, simulator_if, seed, elaborate_only, test_list=None):
84103
if test_list is None:
85104
test_list = TestList()
86105

106+
key = b"test_history"
107+
if not self._database or key not in self._database:
108+
test_history = {}
109+
else:
110+
test_history = self._database[key]
111+
87112
if self._individual_tests:
88113
for test_case in self._test_cases:
89-
test_case.create_tests(simulator_if, seed, elaborate_only, test_list)
114+
test_case.create_tests(simulator_if, seed, elaborate_only, test_list, test_history)
90115
elif self._implicit_test:
91116
for config in self._get_configurations_to_run():
117+
actual_seed = seed
118+
if seed == "repeat":
119+
# Test suite name and test name are the same for implicit tests
120+
test_suite_name = test_name = IndependentSimTestCase.get_name(self._implicit_test, config)
121+
actual_seed = self._get_historic_seed(test_history, test_suite_name, test_name)
122+
92123
test_list.add_test(
93124
IndependentSimTestCase(
94125
test=self._implicit_test,
95126
config=config,
96127
simulator_if=simulator_if,
97-
seed=seed,
128+
seed=actual_seed,
98129
elaborate_only=elaborate_only,
99130
)
100131
)
101132
else:
102133
for config in self._get_configurations_to_run():
134+
actual_seed = seed
135+
if seed == "repeat":
136+
test_suite_name = SameSimTestSuite.get_name(config)
137+
actual_seed = self._get_historic_seed(test_history, test_suite_name)
138+
103139
test_list.add_suite(
104140
SameSimTestSuite(
105141
tests=[test.test for test in self._test_cases],
106142
config=config,
107143
simulator_if=simulator_if,
108-
seed=seed,
144+
seed=actual_seed,
109145
elaborate_only=elaborate_only,
110146
)
111147
)
@@ -414,17 +450,33 @@ def _get_configurations_to_run(self):
414450
del configs[DEFAULT_NAME]
415451
return configs.values()
416452

417-
def create_tests(self, simulator_if, seed, elaborate_only, test_list=None):
453+
def create_tests(
454+
self, simulator_if, seed, elaborate_only, test_list=None, test_history=None
455+
): # pylint: disable=too-many-positional-arguments
418456
"""
419457
Create all tests from this test case which may be several depending on the number of configurations
420458
"""
459+
test_history = test_history if test_history else {}
460+
421461
for config in self._get_configurations_to_run():
462+
actual_seed = seed
463+
if seed == "repeat":
464+
# Test suite name and test name are the same for individual tests
465+
test_suite_name = test_name = IndependentSimTestCase.get_name(self._test, config)
466+
467+
if test_suite_name not in test_history:
468+
actual_seed = None
469+
elif test_name not in test_history[test_suite_name]:
470+
actual_seed = None
471+
else:
472+
actual_seed = test_history[test_suite_name][test_name]["seed"]
473+
422474
test_list.add_test(
423475
IndependentSimTestCase(
424476
test=self._test,
425477
config=config,
426478
simulator_if=simulator_if,
427-
seed=seed,
479+
seed=actual_seed,
428480
elaborate_only=elaborate_only,
429481
)
430482
)

‎vunit/test/list.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,3 +104,6 @@ def run(self, *args, **kwargs):
104104
"""
105105
test_ok = self._test_case.run(*args, **kwargs)
106106
return {self._test_case.name: PASSED if test_ok else FAILED}
107+
108+
def get_seed(self):
109+
return self._test_case.get_seed()

‎vunit/test/report.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ class TestResult(object):
275275
"""
276276

277277
def __init__(
278-
self, name, status, time, output_file_name, *, test_suite_name, start_time
278+
self, name, status, time, output_file_name, *, test_suite_name, start_time, seed
279279
): # pylint: disable=too-many-arguments
280280
assert status in (PASSED, FAILED, SKIPPED)
281281
self.name = name
@@ -284,6 +284,7 @@ def __init__(
284284
self._output_file_name = output_file_name
285285
self.test_suite_name = test_suite_name
286286
self.start_time = start_time
287+
self.seed = seed
287288

288289
@property
289290
def output(self):

0 commit comments

Comments
 (0)