WIP: Updated test priority.

Author: LarsAsplund

vunit/dependency_graph.py

@@ -126,6 +126,18 @@ def get_direct_dependencies(self, node: T) -> Set[T]:
         """
         return self._backward.get(node, set())
 
+    def get_direct_dependents(self, node: T) -> Set[T]:
+        """
+        Get the direct dependents of node.
+        """
+        return self._forward.get(node, set())
+
+    def get_independents(self) -> Set[T]:
+        """
+        Get nodes without dependencies.
+        """
+        return {node for node in self._nodes if not self._backward.get(node, False)}
+
 
 class CircularDependencyException(Exception):
     """

vunit/project.py

@@ -473,7 +473,7 @@ def _handle_circular_dependency(exception):
             " ->\n".join(source_file.name for source_file in exception.path),
         )
 
-    def _get_compile_timestamps(self, files):
+    def get_compile_timestamps(self, files):
         """
         Return a dictionary of mapping file to the timestamp when it
         was compiled or None if it was not compiled
@@ -509,7 +509,7 @@ def _get_files_to_recompile(self, files, dependency_graph, incremental):
         param: files: a list of type SourceFile
         param: dependency_graph: The DependencyGraph object to be used
         """
-        timestamps = self._get_compile_timestamps(files)
+        timestamps = self.get_compile_timestamps(files)
         result_list = []
         for source_file in files:
             if (not incremental) or self._needs_recompile(dependency_graph, source_file, timestamps):

vunit/test/list.py

@@ -35,6 +35,9 @@ def keep_matches(self, test_filter):
         """
         self._test_suites = [test for test in self._test_suites if test.keep_matches(test_filter)]
 
+    def sort(self, key):
+        self._test_suites.sort(key=key)
+
     @property
     def num_tests(self):
         """

vunit/test/report.py

@@ -241,6 +241,9 @@ def to_junit_xml_str(self, xunit_xml_format="jenkins"):
         xml = ElementTree.tostring(root, encoding="unicode")
         return xml
 
+    def __iter__(self):
+        return iter(self._test_results.values())
+
 
 class TestStatus(object):
     """
@@ -271,12 +274,14 @@ class TestResult(object):
     Represents the result of a single test case
     """
 
-    def __init__(self, name, status, time, output_file_name):
+    def __init__(self, name, status, time, output_file_name, test_suite_name, start_time):
         assert status in (PASSED, FAILED, SKIPPED)
         self.name = name
         self._status = status
         self.time = time
         self._output_file_name = output_file_name
+        self.test_suite_name = test_suite_name
+        self.start_time = start_time
 
     @property
     def output(self):

vunit/test/runner.py

@@ -327,7 +327,7 @@ def _add_results(
 
         for test_name in test_suite.test_names:
             status = results[test_name]
-            self._report.add_result(test_name, status, time_per_test, output_file_name)
+            self._report.add_result(test_name, status, time_per_test, output_file_name, test_suite.name, start_time)
             self._report.print_latest_status(total_tests=num_tests)
         print()
 

vunit/ui/__init__.py

@@ -159,16 +159,17 @@ def __init__(
 
         self._create_output_path(args.clean)
 
-        database = self._create_database()
+        self._database = self._create_database()
         self._project = Project(
-            database=database,
+            database=self._database,
             depend_on_package_body=simulator_class.package_users_depend_on_bodies,
         )
 
-        self._test_bench_list = TestBenchList(database=database)
+        self._test_bench_list = TestBenchList(database=self._database)
 
         self._builtins = Builtins(self, self._vhdl_standard, simulator_class)
 
+        self._dependency_graph = None
         self._include_in_test_pattern: Optional[List[Union[str, Path]]] = []
         self._exclude_from_test_pattern: Optional[List[Union[str, Path]]] = []
 
@@ -773,6 +774,7 @@ def _update_test_filter(self, include_dependencies=None, exclude_dependencies=No
 
         project = self._project
         project_source_files = project.get_source_files_in_order()
+        self._dependency_graph = project.create_dependency_graph(True)
 
         def get_dependent_files(dependencies):
             "Return all project files dependent on project files matching any of the dependencies."
@@ -792,8 +794,7 @@ def get_dependent_files(dependencies):
                         dependency_files.add(source_file)
 
             # Get dependent files, non-testbench files included
-            dependency_graph = project.create_dependency_graph(True)
-            dependent_files = set(project.get_affected_files(dependency_files, dependency_graph.get_dependent))
+            dependent_files = set(project.get_affected_files(dependency_files, self._dependency_graph.get_dependent))
 
             return dependent_files
 
@@ -832,13 +833,154 @@ def _create_simulator_if(self):
 
         return self._simulator_class.from_args(args=self._args, output_path=self._simulator_output_path)
 
+    def _get_test_suite_history(self):
+        key = "test_suite_history".encode()
+        if key in self._database:
+            return self._database[key]
+        else:
+            return {}
+
+    def _get_latest_dependency_updates(self):
+        """
+        Return the timestamp for the latest updated dependency of each file.
+        """
+        if not self._dependency_graph:
+            self._dependency_graph = self._project.create_dependency_graph(True)
+
+        # Start with leaf nodes in the dependency graph which has no dependencies
+        # other then to themselves
+        timestamps = {}
+        source_files = self._dependency_graph.get_independents()
+
+        # Move towards the graph roots and record the maximum of the timestamps for each file and
+        # its dependencies
+        while source_files:
+            source_file_timestamps = self._project.get_compile_timestamps(source_files)
+            updated_timestamps = {}
+            for source_file, timestamp in source_file_timestamps.items():
+                dependency_timestamps = self._project.get_compile_timestamps(
+                    self._dependency_graph.get_direct_dependencies(source_file)
+                )
+                dependency_timestamps = [timestamp for timestamp in dependency_timestamps.values() if timestamp]
+                if dependency_timestamps:
+                    updated_timestamps[source_file.name] = max(timestamp, *dependency_timestamps)
+                elif timestamp is None:
+                    # If there is no timestamp, we set a timestamp well into the future to make
+                    # sure that all dependent tests consider it to be updated after the last test run.
+                    updated_timestamps[source_file.name] = 1e10
+                else:
+                    updated_timestamps[source_file.name] = timestamp
+
+            timestamps = {**timestamps, **updated_timestamps}
+            source_files = set.union(
+                *(self._dependency_graph.get_direct_dependents(source_file) for source_file in source_files)
+            )
+
+        return timestamps
+
+    def _sort_test_list(self, test_list, simulator_if):
+        """
+        Sort the test list.
+        """
+        latest_dependency_updates = self._get_latest_dependency_updates()
+        test_suite_history = self._get_test_suite_history()
+
+        # Prune removed tests from history
+        full_test_list = self._test_bench_list.create_tests(simulator_if, self._args.seed, self._args.elaborate)
+        full_test_list = {test_suite.name: test_suite for test_suite in full_test_list._test_suites}
+
+        pruned_test_suite_history = {
+            test_suite_name: test_suite_data
+            for test_suite_name, test_suite_data in test_suite_history.items()
+            if test_suite_name in full_test_list
+        }
+        for test_suite_name, test_suite_data in pruned_test_suite_history.items():
+            pruned_test_suite_data = {
+                test_name: test_data
+                for test_name, test_data in test_suite_data.items()
+                if test_name in full_test_list[test_suite_name].test_names
+            }
+            pruned_test_suite_history[test_suite_name] = pruned_test_suite_data
+
+        test_suite_history = pruned_test_suite_history
+
+        def test_priority(test_suite):
+            # Priority is set to fail fast:
+            # 1. Tests that failed before and for which no updates have been made. Expected to fail again. Shortest   #    test first (STF).
+            # 2. Tests that failed before but updates have been made that can change that. STF.
+            # 3. Tests without a history. New tests are more likely to fail and should be run early. Without
+            #    any execution time order, there is no further sorting within this group
+            # 4. Tests that passed before but depends on updates. There is a risk that we've introduced new bugs. STF.
+            # 5. Tests that passed before and for which there are no updates. They are expected to pass again.
+            #    Longest test first to optimize completion time when running with multiple threads
+            #
+            # Each test is given a priority number where the integer part is according to the priority above and
+            # the decimal part seperates the tests within the group based on execution time.
+
+            test_suite_data = test_suite_history.get(test_suite.name, False)
+            if not test_suite_data:
+                return 3
+
+            highest_priority = None
+            for test_name in test_suite.test_names:
+                test_data = test_suite_data.get(test_name, False)
+                if not test_data:
+                    priority = 3
+                else:
+                    updated_dependency = latest_dependency_updates[test_suite.file_name] > test_data["start_time"]
+                    if test_data["failed"]:
+                        if not updated_dependency:
+                            priority = 1 + test_data["total_time"] / 1e9
+                        else:
+                            priority = 2 + test_data["total_time"] / 1e9
+                    elif test_data["skipped"]:
+                        priority = 3
+                    elif updated_dependency:
+                        priority = 4 + test_data["total_time"] / 1e9
+                    else:
+                        priority = 6 - test_data["total_time"] / 1e9
+
+                highest_priority = priority if not highest_priority else min(highest_priority, priority)
+
+            return highest_priority
+
+        test_list.sort(test_priority)
+
+    def _update_test_suite_history(self, report):
+        # TODO: It is a problem if a test suite with several test cases (run_all_in_same_sim) is run partially.
+        # Then previous history for the test cases not run is overwritten
+        # Need to to know the test cases in the suite or we may save data for a deleted test case "forever"
+        test_suite_data = {}
+        all_tests = self._test_bench_list.get_test_benches()
+        for test_result in report:
+            if not test_result.test_suite_name in test_suite_data:
+                test_suite_data[test_result.test_suite_name] = dict()
+
+            if not test_result.name in test_suite_data[test_result.test_suite_name]:
+                test_suite_data[test_result.test_suite_name][test_result.name] = dict()
+
+            test_suite_data[test_result.test_suite_name][test_result.name]["total_time"] = test_result.time
+            test_suite_data[test_result.test_suite_name][test_result.name]["passed"] = test_result.passed
+            test_suite_data[test_result.test_suite_name][test_result.name]["skipped"] = test_result.skipped
+            test_suite_data[test_result.test_suite_name][test_result.name]["failed"] = test_result.failed
+            test_suite_data[test_result.test_suite_name][test_result.name]["start_time"] = test_result.start_time
+
+        test_suite_history = self._get_test_suite_history()
+
+        for test_suite_name, data in test_suite_data.items():
+            test_suite_history[test_suite_name] = data
+
+        self._database["test_suite_history".encode()] = test_suite_history
+
     def _main_run(self, post_run):
         """
         Main with running tests
         """
         simulator_if = self._create_simulator_if()
         test_list = self._create_tests(simulator_if)
         self._compile(simulator_if)
+        # Sort after compilation has determined changed files
+        self._sort_test_list(test_list, simulator_if)
         print()
 
         start_time = ostools.get_time()
@@ -853,6 +995,7 @@ def _main_run(self, post_run):
             del test_list
 
         report.set_real_total_time(ostools.get_time() - start_time)
+        self._update_test_suite_history(report)
         report.print_str()
 
         if post_run is not None: