Enforce limiting frameworks to a maximum number of test mutations

> The number of test mutations will be limited to 10.
#8420

Author: p8

Diff for: toolset/benchmark/benchmarker.py

@@ -18,7 +18,7 @@
 
 from colorama import Fore
 import numbers
-
+from collections import defaultdict
 
 class Benchmarker:
     def __init__(self, config):
@@ -36,6 +36,8 @@ def __init__(self, config):
             self.tests.reverse()
         self.results = Results(self)
         self.docker_helper = DockerHelper(self)
+        self.max_framework_test_count = 10
+        self.framework_test_count = defaultdict(lambda: 0)
 
         self.last_test = False
 
@@ -138,6 +140,19 @@ def __run_test(self, test, benchmark_log):
                 prefix=log_prefix,
                 file=benchmark_log)
 
+        # Each framework has maximum number of tests
+        self.framework_test_count[test.framework] += 1
+        if self.framework_test_count[test.framework] > self.max_framework_test_count:
+            message = "Framework {framework} has run the maximum of {max} tests. Skipping {name}.".format(
+                name=test.name, framework=test.framework, max=self.max_framework_test_count)
+            self.results.write_intermediate(test.name, message)
+            self.results.upload()
+            return self.__exit_test(
+                success=False,
+                message=message,
+                prefix=log_prefix,
+                file=benchmark_log)
+
         database_container = None
         try:
             # Start database container