Skip to content

Commit 61951cb

Browse files
LarsAsplundLarsAsplund
committed
WIP: Dynamic scheduling
1 parent e1c563a commit 61951cb

File tree

4 files changed

+474
-103
lines changed

4 files changed

+474
-103
lines changed

‎tests/unit/test_test_scheduler.py‎

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
# This Source Code Form is subject to the terms of the Mozilla Public
2+
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
3+
# You can obtain one at http://mozilla.org/MPL/2.0/.
4+
#
5+
# Copyright (c) 2014-2025, Lars Asplund [email protected]
6+
7+
"""
8+
Test the test scheduler
9+
"""
10+
11+
import unittest
12+
from unittest import mock
13+
from vunit.test.runner import TestScheduler
14+
15+
16+
class TestTestScheduler(unittest.TestCase):
17+
"""
18+
Test the test scheduler
19+
"""
20+
21+
def setUp(self):
22+
self._test_suite_history = dict()
23+
24+
@staticmethod
25+
def _create_test_suite(name, test_names, file_name):
26+
"""Helper method to create a mock test suite."""
27+
28+
test_suite = mock.MagicMock()
29+
test_suite.name = name
30+
test_suite.test_names = test_names
31+
test_suite.file_name = file_name
32+
return test_suite
33+
34+
def _add_test_suite_history(self, test_suite_name, test_name, status, start_time, total_time):
35+
"""Helper method to create test suite histories."""
36+
37+
if test_suite_name not in self._test_suite_history:
38+
self._test_suite_history[test_suite_name] = dict()
39+
40+
history = dict()
41+
history["total_time"] = total_time
42+
history["passed"] = status == "passed"
43+
history["skipped"] = status == "skipped"
44+
history["failed"] = status == "failed"
45+
history["start_time"] = start_time
46+
self._test_suite_history[test_suite_name][test_name] = history
47+
48+
def test_that_new_test_suites_go_to_set_2(self):
49+
latest_dependency_updates = {}
50+
test_suites = [self._create_test_suite("lib1.tb1.test1", ["lib1.tb1.test1"], "file1")]
51+
self._add_test_suite_history("lib1.tb1.test2", "lib1.tb1.test2", "passed", 0, 10)
52+
53+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
54+
result = test_scheduler._create_test_suite_sets(test_suites)
55+
56+
for set_idx in range(5):
57+
if set_idx == 2:
58+
self.assertIsNone(result[set_idx]["total_exec_time"])
59+
self.assertEqual(len(result[set_idx]["test_suites"]), 1)
60+
self.assertEqual(result[set_idx]["test_suites"][0]["test_suite"].name, "lib1.tb1.test1")
61+
self.assertIsNone(result[set_idx]["test_suites"][0]["exec_time"])
62+
else:
63+
self.assertEqual(result[set_idx]["total_exec_time"], 0)
64+
self.assertEqual(len(result[set_idx]["test_suites"]), 0)
65+
66+
def test_that_skipped_test_suites_go_to_set_2(self):
67+
latest_dependency_updates = dict(file1=0)
68+
test_suites = [self._create_test_suite("lib1.tb1.test1", ["lib1.tb1.test1"], "file1")]
69+
self._add_test_suite_history("lib1.tb1.test1", "lib1.tb1.test1", "skipped", 0, 10)
70+
71+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
72+
result = test_scheduler._create_test_suite_sets(test_suites)
73+
74+
for set_idx in range(5):
75+
if set_idx == 2:
76+
self.assertIsNone(result[set_idx]["total_exec_time"])
77+
self.assertEqual(len(result[set_idx]["test_suites"]), 1)
78+
self.assertEqual(result[set_idx]["test_suites"][0]["test_suite"].name, "lib1.tb1.test1")
79+
else:
80+
self.assertEqual(result[set_idx]["total_exec_time"], 0)
81+
self.assertEqual(len(result[set_idx]["test_suites"]), 0)
82+
83+
def test_that_failed_test_suites_wo_updates_go_to_set_0(self):
84+
latest_dependency_updates = dict(file1=0)
85+
test_suites = [self._create_test_suite("lib1.tb1.test1", ["lib1.tb1.test1"], "file1")]
86+
self._add_test_suite_history("lib1.tb1.test1", "lib1.tb1.test1", "failed", 1, 10)
87+
88+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
89+
result = test_scheduler._create_test_suite_sets(test_suites)
90+
91+
for set_idx in range(5):
92+
if set_idx == 0:
93+
self.assertEqual(result[set_idx]["total_exec_time"], 10)
94+
self.assertEqual(len(result[set_idx]["test_suites"]), 1)
95+
self.assertEqual(result[set_idx]["test_suites"][0]["test_suite"].name, "lib1.tb1.test1")
96+
else:
97+
if set_idx == 2:
98+
self.assertIsNone(result[set_idx]["total_exec_time"])
99+
else:
100+
self.assertEqual(result[set_idx]["total_exec_time"], 0)
101+
self.assertEqual(len(result[set_idx]["test_suites"]), 0)
102+
103+
def test_that_failed_test_suites_w_updates_go_to_set_1(self):
104+
latest_dependency_updates = dict(file1=2)
105+
test_suites = [self._create_test_suite("lib1.tb1.test1", ["lib1.tb1.test1"], "file1")]
106+
self._add_test_suite_history("lib1.tb1.test1", "lib1.tb1.test1", "failed", 1, 10)
107+
108+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
109+
result = test_scheduler._create_test_suite_sets(test_suites)
110+
111+
for set_idx in range(5):
112+
if set_idx == 1:
113+
self.assertEqual(result[set_idx]["total_exec_time"], 10)
114+
self.assertEqual(len(result[set_idx]["test_suites"]), 1)
115+
self.assertEqual(result[set_idx]["test_suites"][0]["test_suite"].name, "lib1.tb1.test1")
116+
else:
117+
if set_idx == 2:
118+
self.assertIsNone(result[set_idx]["total_exec_time"])
119+
else:
120+
self.assertEqual(result[set_idx]["total_exec_time"], 0)
121+
self.assertEqual(len(result[set_idx]["test_suites"]), 0)
122+
123+
def test_that_passed_test_suites_wo_updates_go_to_set_4(self):
124+
latest_dependency_updates = dict(file1=0)
125+
test_suites = [self._create_test_suite("lib1.tb1.test1", ["lib1.tb1.test1"], "file1")]
126+
self._add_test_suite_history("lib1.tb1.test1", "lib1.tb1.test1", "passed", 1, 10)
127+
128+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
129+
result = test_scheduler._create_test_suite_sets(test_suites)
130+
131+
for set_idx in range(5):
132+
if set_idx == 4:
133+
self.assertEqual(result[set_idx]["total_exec_time"], 10)
134+
self.assertEqual(len(result[set_idx]["test_suites"]), 1)
135+
self.assertEqual(result[set_idx]["test_suites"][0]["test_suite"].name, "lib1.tb1.test1")
136+
else:
137+
if set_idx == 2:
138+
self.assertIsNone(result[set_idx]["total_exec_time"])
139+
else:
140+
self.assertEqual(result[set_idx]["total_exec_time"], 0)
141+
self.assertEqual(len(result[set_idx]["test_suites"]), 0)
142+
143+
def test_that_passed_test_suites_w_updates_go_to_set_3(self):
144+
latest_dependency_updates = dict(file1=2)
145+
test_suites = [self._create_test_suite("lib1.tb1.test1", ["lib1.tb1.test1"], "file1")]
146+
self._add_test_suite_history("lib1.tb1.test1", "lib1.tb1.test1", "passed", 1, 10)
147+
148+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
149+
result = test_scheduler._create_test_suite_sets(test_suites)
150+
151+
for set_idx in range(5):
152+
if set_idx == 3:
153+
self.assertEqual(result[set_idx]["total_exec_time"], 10)
154+
self.assertEqual(len(result[set_idx]["test_suites"]), 1)
155+
self.assertEqual(result[set_idx]["test_suites"][0]["test_suite"].name, "lib1.tb1.test1")
156+
else:
157+
if set_idx == 2:
158+
self.assertIsNone(result[set_idx]["total_exec_time"])
159+
else:
160+
self.assertEqual(result[set_idx]["total_exec_time"], 0)
161+
self.assertEqual(len(result[set_idx]["test_suites"]), 0)
162+
163+
def test_that_highest_priority_set_wins(self):
164+
latest_dependency_updates = dict(file1=2)
165+
test_suites = [
166+
self._create_test_suite(
167+
"lib1.tb1",
168+
["lib1.tb1.test1", "lib1.tb1.test2", "lib1.tb1.test3", "lib1.tb1.test4", "lib1.tb1.test5"],
169+
"file1",
170+
)
171+
]
172+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test1", "failed", 3, 10)
173+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test2", "failed", 1, 11)
174+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test3", "skipped", 1, None)
175+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test4", "passed", 1, 13)
176+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test5", "passed", 3, 14)
177+
178+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
179+
result = test_scheduler._create_test_suite_sets(test_suites)
180+
181+
for set_idx in range(5):
182+
if set_idx == 0:
183+
self.assertEqual(result[set_idx]["total_exec_time"], 48)
184+
self.assertEqual(len(result[set_idx]["test_suites"]), 1)
185+
self.assertEqual(result[set_idx]["test_suites"][0]["test_suite"].name, "lib1.tb1")
186+
else:
187+
if set_idx == 2:
188+
self.assertIsNone(result[set_idx]["total_exec_time"])
189+
else:
190+
self.assertEqual(result[set_idx]["total_exec_time"], 0)
191+
self.assertEqual(len(result[set_idx]["test_suites"]), 0)
192+
193+
def test_that_sets_are_ordered_by_execution_time(self):
194+
latest_dependency_updates = dict(file1=2, file2=1, file3=2.5)
195+
test_suites = [
196+
self._create_test_suite(
197+
"lib1.tb1",
198+
["lib1.tb1.test1", "lib1.tb1.test2", "lib1.tb1.test3", "lib1.tb1.test4", "lib1.tb1.test5"],
199+
"file1",
200+
),
201+
self._create_test_suite("lib1.tb2", ["lib1.tb2.test1"], "file2"),
202+
self._create_test_suite("lib1.tb3", ["lib1.tb3.test1"], "file3"),
203+
]
204+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test1", "failed", 3, 10)
205+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test2", "failed", 1, 11)
206+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test3", "skipped", 1, None)
207+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test4", "passed", 1, 13)
208+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test5", "passed", 3, 14)
209+
self._add_test_suite_history("lib1.tb2", "lib1.tb2.test1", "failed", 3, 9)
210+
self._add_test_suite_history("lib1.tb3", "lib1.tb3.test1", "failed", 3, 8)
211+
212+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
213+
result = test_scheduler._create_test_suite_sets(test_suites)
214+
215+
for set_idx in range(5):
216+
if set_idx == 0:
217+
self.assertEqual(result[set_idx]["total_exec_time"], 65)
218+
self.assertEqual(len(result[set_idx]["test_suites"]), 3)
219+
self.assertEqual(result[set_idx]["test_suites"][0]["test_suite"].name, "lib1.tb3")
220+
self.assertEqual(result[set_idx]["test_suites"][1]["test_suite"].name, "lib1.tb2")
221+
self.assertEqual(result[set_idx]["test_suites"][2]["test_suite"].name, "lib1.tb1")
222+
else:
223+
if set_idx == 2:
224+
self.assertIsNone(result[set_idx]["total_exec_time"])
225+
else:
226+
self.assertEqual(result[set_idx]["total_exec_time"], 0)
227+
self.assertEqual(len(result[set_idx]["test_suites"]), 0)
228+
229+
def test_that_next_test_suite_is_picked_in_time_order_for_single_thread(self):
230+
latest_dependency_updates = dict(file1=2, file2=1, file3=2.5, file4=1, file5=1)
231+
test_suites = [
232+
self._create_test_suite("lib1.tb1", ["lib1.tb1.test1"], "file1"),
233+
self._create_test_suite("lib1.tb2", ["lib1.tb2.test1"], "file2"),
234+
self._create_test_suite("lib1.tb3", ["lib1.tb3.test1"], "file3"),
235+
self._create_test_suite("lib1.tb4", ["lib1.tb4.test1"], "file4"),
236+
self._create_test_suite("lib1.tb5", ["lib1.tb5.test1"], "file5"),
237+
]
238+
239+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test1", "failed", 3, 10)
240+
self._add_test_suite_history("lib1.tb2", "lib1.tb2.test1", "failed", 3, 9)
241+
self._add_test_suite_history("lib1.tb4", "lib1.tb4.test1", "passed", 3, 9)
242+
243+
test_scheduler = TestScheduler(test_suites, 1, latest_dependency_updates, self._test_suite_history)
244+
245+
with mock.patch("time.time", lambda: 1000):
246+
self.assertEqual(test_scheduler.next(thread_id=0).name, "lib1.tb2")
247+
self.assertEqual(test_scheduler.next(thread_id=0).name, "lib1.tb1")
248+
self.assertEqual(test_scheduler.next(thread_id=0).name, "lib1.tb3")
249+
self.assertEqual(test_scheduler.next(thread_id=0).name, "lib1.tb5")
250+
self.assertEqual(test_scheduler.next(thread_id=0).name, "lib1.tb4")
251+
self.assertRaises(StopIteration, test_scheduler.next, thread_id=0)
252+
253+
def test_that_next_test_suite_is_picked_considering_load_balancing(self):
254+
latest_dependency_updates = dict(file1=0, file2=0, file3=0, file4=0, file5=0)
255+
test_suites = [
256+
self._create_test_suite("lib1.tb1", ["lib1.tb1.test1"], "file1"),
257+
self._create_test_suite("lib1.tb2", ["lib1.tb2.test1"], "file2"),
258+
self._create_test_suite("lib1.tb3", ["lib1.tb3.test1"], "file3"),
259+
self._create_test_suite("lib1.tb4", ["lib1.tb4.test1"], "file4"),
260+
self._create_test_suite("lib1.tb5", ["lib1.tb5.test1"], "file5"),
261+
]
262+
263+
self._add_test_suite_history("lib1.tb1", "lib1.tb1.test1", "passed", 3, 2)
264+
self._add_test_suite_history("lib1.tb2", "lib1.tb2.test1", "passed", 3, 3.5)
265+
self._add_test_suite_history("lib1.tb3", "lib1.tb3.test1", "passed", 3, 3)
266+
self._add_test_suite_history("lib1.tb4", "lib1.tb4.test1", "passed", 3, 9.5)
267+
self._add_test_suite_history("lib1.tb5", "lib1.tb5.test1", "passed", 3, 12)
268+
269+
test_scheduler = TestScheduler(test_suites, 3, latest_dependency_updates, self._test_suite_history)
270+
271+
with mock.patch("time.time", lambda: 1000):
272+
self.assertEqual(test_scheduler.next(thread_id=0).name, "lib1.tb5")
273+
self.assertEqual(test_scheduler.next(thread_id=1).name, "lib1.tb1")
274+
self.assertEqual(test_scheduler.next(thread_id=2).name, "lib1.tb4")
275+
276+
with mock.patch("time.time", lambda: 1002):
277+
test_scheduler.test_done(thread_id=1)
278+
self.assertEqual(test_scheduler.next(thread_id=1).name, "lib1.tb3")
279+
280+
with mock.patch("time.time", lambda: 1005):
281+
test_scheduler.test_done(thread_id=1)
282+
self.assertEqual(test_scheduler.next(thread_id=1).name, "lib1.tb2")
283+
284+
with mock.patch("time.time", lambda: 1008.5):
285+
test_scheduler.test_done(thread_id=1)
286+
self.assertRaises(StopIteration, test_scheduler.next, thread_id=1)
287+
288+
with mock.patch("time.time", lambda: 1009.5):
289+
test_scheduler.test_done(thread_id=2)
290+
self.assertRaises(StopIteration, test_scheduler.next, thread_id=2)
291+
292+
self.assertFalse(test_scheduler.is_finished())
293+
294+
with mock.patch("time.time", lambda: 1012):
295+
test_scheduler.test_done(thread_id=0)
296+
self.assertRaises(StopIteration, test_scheduler.next, thread_id=0)
297+
298+
self.assertTrue(test_scheduler.is_finished())

‎vunit/test/list.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,6 @@ def __iter__(self):
6464
def __len__(self):
6565
return len(self._test_suites)
6666

67-
def __getitem__(self, idx):
68-
return self._test_suites[idx]
69-
7067

7168
class TestSuiteWrapper(object):
7269
"""

0 commit comments

Comments
 (0)