Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/news.d/1113.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added support for repeating seeds from previous test run by setting the ``--seed`` option to ``repeat``.
2 changes: 2 additions & 0 deletions docs/run/user_guide.rst
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,8 @@ To reproduce the failing test setup and verify a bug fix, the failing seed can b
.. raw:: html
:file: img/seed_option.html

``--seed`` can also be set to ``repeat``, in which case the values from the previous test run will be repeated.

Running A VUnit Testbench Standalone
------------------------------------

Expand Down
Binary file modified tests/acceptance/artificial/vhdl/reference_test_history/0
Binary file not shown.
Binary file modified tests/acceptance/artificial/vhdl/reference_test_history/1
Binary file not shown.
10 changes: 10 additions & 0 deletions tests/acceptance/artificial/vhdl/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,15 @@ def pre_config(output_path):
for idx, (extra_time, fail) in enumerate([(5, False), (2, True)], 1):
tb.add_config(name=f"test_{idx}", pre_config=make_pre_config(idx, extra_time, fail))

def configure_tb_seed(ui):
tb = ui.library("lib").test_bench("tb_seed")

if args.seed == "0123456789abcdef":
tb.test("test_1").set_generic("expected_seed", "2e373913e5ad677d")
tb.test("test_2").set_generic("expected_seed", "2e373913e5ad677d")
elif args.seed == "repeat":
tb.test("test_1").set_generic("expected_seed", "ffa08cd9489aad14")
tb.test("test_2").set_generic("expected_seed", "9a292b3679afd081")

configure_tb_with_generic_config()
configure_tb_same_sim_all_pass(vu)
Expand All @@ -172,6 +181,7 @@ def pre_config(output_path):
configure_tb_with_vhdl_configuration(vu)
configure_tb_no_fail_on_warning(vu)
configure_tb_test_prio(vu)
configure_tb_seed(vu)
lib.entity("tb_no_generic_override").set_generic("g_val", False)
lib.entity("tb_ieee_warning").test("pass").set_sim_option("disable_ieee_warnings", True)
lib.entity("tb_other_file_tests").scan_tests_from_file(str(root / "other_file_tests.vhd"))
Expand Down
47 changes: 47 additions & 0 deletions tests/acceptance/artificial/vhdl/tb_seed.vhd
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
-- This Source Code Form is subject to the terms of the Mozilla Public
-- License, v. 2.0. If a copy of the MPL was not distributed with this file,
-- You can obtain one at http://mozilla.org/MPL/2.0/.
--
-- Copyright (c) 2014-2025, Lars Asplund [email protected]

-- This attribute should be ignored when VHDL assert stop level is used
-- vunit: fail_on_warning

library vunit_lib;
context vunit_lib.vunit_context;

entity tb_seed is
generic (
runner_cfg : string;
expected_seed : string := ""
);
end entity;

architecture tb of tb_seed is
constant seed : string := get_seed(runner_cfg);
begin
test_runner : process
begin
test_runner_setup(runner, runner_cfg);
show(display_handler, pass);

while test_suite loop
if expected_seed = "" then
-- Not expecting the seed derived from --seed=0123456789abcdef
check(seed /= "2e373913e5ad677d");
else
check_equal(seed, expected_seed);
end if;

if run("test_1") then
-- Not expecting the seed derived from --seed=repeat
check_implication(expected_seed = "", seed /= "ffa08cd9489aad14");
elsif run("test_2") then
-- Not expecting the seed derived from --seed=repeat
check_implication(expected_seed = "", seed /= "9a292b3679afd081");
end if;
end loop;

test_runner_cleanup(runner);
end process;
end architecture;
22 changes: 22 additions & 0 deletions tests/acceptance/test_artificial.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,26 @@ def test_artificial_changed(self):
],
)

def test_hardcoded_seed(self):
self.check(self.artificial_run_vhdl, args=["lib.tb_seed*", "--seed=0123456789AbCdEf"])
check_report(
self.report_file,
[
("passed", "lib.tb_seed.test_1"),
("passed", "lib.tb_seed.test_2"),
],
)

def test_repeated_seed(self):
self.check(self.artificial_run_vhdl, args=["lib.tb_seed*", "--seed=repeat"])
check_report(
self.report_file,
[
("passed", "lib.tb_seed.test_1"),
("passed", "lib.tb_seed.test_2"),
],
)

def _test_artificial(self, args=None):
"""
Utility function to run and check the result of all test benches
Expand Down Expand Up @@ -289,4 +309,6 @@ def test_exit_0_flag(self):
("passed", "lib.tb_test_prio_1.test_4"),
("passed", "lib.tb_test_prio_2.test_1"),
("failed", "lib.tb_test_prio_2.test_2"),
("passed", "lib.tb_seed.test_1"),
("passed", "lib.tb_seed.test_2"),
)
16 changes: 15 additions & 1 deletion tests/unit/test_test_bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
FileLocation,
Attribute,
LegacyAttribute,
_get_historic_seed,
)
from vunit.configuration import AttributeException
from vunit.ostools import write_file
Expand Down Expand Up @@ -483,7 +484,7 @@ def test_error_on_global_attributes_from_python_on_tests(self, tempdir):

test_bench = TestBench(design_unit)
test = test_bench.get_test_case("Test 1")
self.assertRaises(AttributeException, test.set_attribute, "run_all_in_same_sim", True)
self.assertRaises(AttributeException, test.set_attribute, "run_all_in_same_sim", True)

@with_tempdir
def test_test_information(self, tempdir):
Expand Down Expand Up @@ -904,6 +905,19 @@ def assert_has_tests(self, test_list, tests):
self.assertEqual(test1.name, test2)
self.assertEqual(test1.test_names, [test2])

def test_seed_from_test_history(self):
test_history = dict(test_suite1=dict(test1=dict(seed="11"), test2=dict(seed="12")), test_suite2=dict())

self.assertEqual(_get_historic_seed(test_history, test_suite_name="test_suite1"), "11")
self.assertEqual(_get_historic_seed(test_history, test_suite_name="test_suite1", test_name="test2"), "12")

def test_no_seed_from_test_history(self):
test_history = dict(test_suite1=dict(test1=dict(seed="11"), test2=dict(seed="12")), test_suite2=dict())

self.assertIsNone(_get_historic_seed(test_history, test_suite_name="foo"))
self.assertIsNone(_get_historic_seed(test_history, test_suite_name="test_suite1", test_name="test3"))
self.assertIsNone(_get_historic_seed(test_history, test_suite_name="test_suite2"))

@staticmethod
def create_tests(test_bench):
"""
Expand Down
31 changes: 30 additions & 1 deletion tests/unit/test_test_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,13 @@ def test_junit_report_with_some_skipped_tests(self):
def test_junit_report_with_testcase_classname(self):
report = self._new_report()
report.add_result(
"test", PASSED, time=1.0, output_file_name=self.output_file_name, test_suite_name="test", start_time=0
"test",
PASSED,
time=1.0,
output_file_name=self.output_file_name,
test_suite_name="test",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"lib.entity",
Expand All @@ -298,6 +304,7 @@ def test_junit_report_with_testcase_classname(self):
output_file_name=self.output_file_name,
test_suite_name="lib.entity",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"lib.entity.test",
Expand All @@ -306,6 +313,7 @@ def test_junit_report_with_testcase_classname(self):
output_file_name=self.output_file_name,
test_suite_name="lib.entity.test",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"lib.entity.config.test",
Expand All @@ -314,6 +322,7 @@ def test_junit_report_with_testcase_classname(self):
output_file_name=self.output_file_name,
test_suite_name="lib.entity.config.test",
start_time=0,
seed="0123456789abcdef",
)
root = ElementTree.fromstring(report.to_junit_xml_str())
names = set(
Expand Down Expand Up @@ -372,6 +381,7 @@ def _report_with_all_passed_tests(self, output_file_name=None):
output_file_name=output_file_name,
test_suite_name="passed_test0",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test1",
Expand All @@ -380,6 +390,7 @@ def _report_with_all_passed_tests(self, output_file_name=None):
output_file_name=output_file_name,
test_suite_name="passed_test0",
start_time=0,
seed="0123456789abcdef",
)
report.set_expected_num_tests(2)
return report
Expand All @@ -394,6 +405,7 @@ def _report_with_missing_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test0",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test1",
Expand All @@ -402,6 +414,7 @@ def _report_with_missing_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test0",
start_time=0,
seed="0123456789abcdef",
)
report.set_expected_num_tests(3)
return report
Expand All @@ -416,6 +429,7 @@ def _report_with_some_failed_tests(self):
output_file_name=self.output_file_name,
test_suite_name="failed_test0",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test",
Expand All @@ -424,6 +438,7 @@ def _report_with_some_failed_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"failed_test1",
Expand All @@ -432,6 +447,7 @@ def _report_with_some_failed_tests(self):
output_file_name=self.output_file_name,
test_suite_name="failed_test1",
start_time=0,
seed="0123456789abcdef",
)
report.set_expected_num_tests(3)
return report
Expand All @@ -446,6 +462,7 @@ def _report_with_some_skipped_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"skipped_test",
Expand All @@ -454,6 +471,7 @@ def _report_with_some_skipped_tests(self):
output_file_name=self.output_file_name,
test_suite_name="skipped_test",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"failed_test",
Expand All @@ -462,6 +480,7 @@ def _report_with_some_skipped_tests(self):
output_file_name=self.output_file_name,
test_suite_name="failed_test",
start_time=0,
seed="0123456789abcdef",
)
report.set_expected_num_tests(3)
return report
Expand All @@ -476,6 +495,7 @@ def _report_with_mixed_length_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test0",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test1",
Expand All @@ -484,6 +504,7 @@ def _report_with_mixed_length_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test1",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test2",
Expand All @@ -492,6 +513,7 @@ def _report_with_mixed_length_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test2",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test3",
Expand All @@ -500,6 +522,7 @@ def _report_with_mixed_length_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test3",
start_time=0,
seed="0123456789abcdef",
)
report.set_expected_num_tests(4)
return report
Expand All @@ -514,6 +537,7 @@ def _report_with_long_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test0",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test1",
Expand All @@ -522,6 +546,7 @@ def _report_with_long_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test1",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test2",
Expand All @@ -530,6 +555,7 @@ def _report_with_long_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test2",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test3",
Expand All @@ -538,6 +564,7 @@ def _report_with_long_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test3",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test4",
Expand All @@ -546,6 +573,7 @@ def _report_with_long_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test4",
start_time=0,
seed="0123456789abcdef",
)
report.add_result(
"passed_test5",
Expand All @@ -554,6 +582,7 @@ def _report_with_long_tests(self):
output_file_name=self.output_file_name,
test_suite_name="passed_test5",
start_time=0,
seed="0123456789abcdef",
)
report.set_expected_num_tests(6)
return report
Expand Down
3 changes: 3 additions & 0 deletions tests/unit/test_test_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,3 +256,6 @@ def run(self, output_path, read_output):
self.output_path = output_path
self.read_output = read_output
return self.run_side_effect(output_path=output_path, read_output=read_output)

def get_seed(self):
return "0123456789abcdef"
Loading