diff --git a/doc/en/introduction.rst b/doc/en/introduction.rst index 5cc7fe8c1..42f55b0b3 100644 --- a/doc/en/introduction.rst +++ b/doc/en/introduction.rst @@ -346,7 +346,8 @@ Command line "": { | "execution_time": 199.99, | "setup_time": 39.99, - | "teardown_time": 39.99, // optional + | "teardown_time": 39.99, + | "testcase_count": 10 // optional }, } diff --git a/doc/en/pools.rst b/doc/en/pools.rst index 643357c53..d5455930d 100644 --- a/doc/en/pools.rst +++ b/doc/en/pools.rst @@ -390,7 +390,8 @@ enabled by providing runtime data like following via ``--runtime-data`` command "": { "execution_time": 199.99, "setup_time": 39.99, - "teardown_time": 0 + "teardown_time": 0, + "testcase_count": 10 }, ...... } diff --git a/doc/newsfragments/3294_changed.testcase_count_adjusted_runtime.rst b/doc/newsfragments/3294_changed.testcase_count_adjusted_runtime.rst new file mode 100644 index 000000000..9a4200e18 --- /dev/null +++ b/doc/newsfragments/3294_changed.testcase_count_adjusted_runtime.rst @@ -0,0 +1 @@ +Runtime data JSON file specified through ``--runtime-data`` now accepts an optional ``testcase_count`` field, which should hold the total number of testcases from the previous run. If ``testcase_count`` is specified, estimated MultiTest execution time will be adjusted by the ratio of current run's testcase count to that of previous run, which should bring more accurate auto-partitioning and improved testing resource utilization. \ No newline at end of file diff --git a/examples/ExecutionPools/AutoPart/runtime_data.txt b/examples/ExecutionPools/AutoPart/runtime_data.txt index 4d3703844..8b795bbf0 100755 --- a/examples/ExecutionPools/AutoPart/runtime_data.txt +++ b/examples/ExecutionPools/AutoPart/runtime_data.txt @@ -2,6 +2,7 @@ "Proj1-suite": { "execution_time": 199.99, "setup_time": 5, - "teardown_time": 0 + "teardown_time": 0, + "testcase_count": 8 } } diff --git a/testplan/base.py b/testplan/base.py index bf8b0ce4b..6009ba552 100644 --- a/testplan/base.py +++ b/testplan/base.py @@ -174,7 +174,7 @@ def __init__( parse_cmdline: bool = True, parser: Type[TestplanParser] = TestplanParser, interactive_port: Optional[int] = None, - abort_signals: Optional[List[int]] = None, + abort_signals: Optional[List[signal.Signals]] = None, logger_level: int = logger.USER_INFO, file_log_level: int = logger.DEBUG, runpath: Union[str, Callable] = path.default_runpath, @@ -197,8 +197,8 @@ def __init__( browse: bool = False, ui_port: Optional[int] = None, web_server_startup_timeout: int = defaults.WEB_SERVER_TIMEOUT, - test_filter: Type[BaseFilter] = filtering.Filter(), - test_sorter: Type[BaseSorter] = ordering.NoopSorter(), + test_filter: BaseFilter = filtering.Filter(), + test_sorter: BaseSorter = ordering.NoopSorter(), test_lister: Optional[MetadataBasedLister] = None, test_lister_output: Optional[os.PathLike] = None, verbose: bool = False, diff --git a/testplan/common/entity/base.py b/testplan/common/entity/base.py index 50e5cf8ab..45c1e0673 100644 --- a/testplan/common/entity/base.py +++ b/testplan/common/entity/base.py @@ -906,7 +906,7 @@ class RunnableResult: def __init__(self): self.step_results = OrderedDict() - self.run = False + self.run: Union[bool, Exception] = False def __repr__(self): return f"{self.__class__.__name__}[{vars(self)}]" diff --git a/testplan/common/utils/logger.py b/testplan/common/utils/logger.py index cb678119b..2895cc7a5 100644 --- a/testplan/common/utils/logger.py +++ b/testplan/common/utils/logger.py @@ -106,7 +106,7 @@ def _custom_log(self, level, msg, *args, **kwargs): self._log(level, msg, args, **kwargs) -def _initial_setup(): +def _initial_setup() -> tuple[TestplanLogger, logging.StreamHandler]: """ Perform initial setup for the logger. Creates and adds a handler to log to stdout with default level USER_INFO. diff --git a/testplan/parser.py b/testplan/parser.py index 6cab50770..5ec617380 100644 --- a/testplan/parser.py +++ b/testplan/parser.py @@ -202,6 +202,7 @@ def generate_parser(self) -> HelpParser: "execution_time": 199.99, "setup_time": 39.99, "teardown_time": 39.99, + "testcase_count": 10 }, ...... }""", @@ -599,7 +600,8 @@ def _read_text_file(file: str) -> List[str]: str: { "execution_time": schema.Or(int, float), "setup_time": schema.Or(int, float), - schema.Optional("teardown_time"): schema.Or(int, float), + "teardown_time": schema.Or(int, float), + schema.Optional("testcase_count"): int, } } ) diff --git a/testplan/runnable/base.py b/testplan/runnable/base.py index 709e680c3..1db8b6f30 100644 --- a/testplan/runnable/base.py +++ b/testplan/runnable/base.py @@ -86,6 +86,9 @@ TestTask = Union[Test, Task, Callable] +MULTITEST_EXEC_TIME_ADJUST_FACTOR_LB = 0.25 + + @dataclass class TaskInformation: target: TestTask @@ -913,8 +916,13 @@ def auto_part(self, tasks: List[Task]) -> List[Task]: discovered: List[TaskInformation] = [ _detach_task_info(task) for task in tasks ] - auto_part_runtime_limit = self._calculate_part_runtime(discovered) + runtime_data = self.cfg.runtime_data or {} + self._adjust_runtime_data(discovered, runtime_data) + # here we replace the original runtime data with adjusted values + # and "previous" testcase count with current run count + self.cfg.set_local("runtime_data", runtime_data) + auto_part_runtime_limit = self._calculate_part_runtime(discovered) for task_info in discovered: partitioned.extend( self._calculate_parts_and_weights( @@ -924,7 +932,49 @@ def auto_part(self, tasks: List[Task]) -> List[Task]: return [_attach_task_info(task_info) for task_info in partitioned] - def _calculate_part_runtime(self, discovered: List[TaskInformation]): + def _adjust_runtime_data( + self, discovered: List[TaskInformation], runtime_data: dict + ): + """ + Adjust the runtime data to ensure that all discovered tasks have their + runtime data available. If a task's UID is not found in the runtime data, + it will be added with default values. + """ + for task_info in discovered: + uid = task_info.uid + time_info = runtime_data.get(uid, None) + if time_info and isinstance( + task_info.materialized_test, MultiTest + ): + if prev_case_count := time_info.get("testcase_count", 0): + # XXX: cache dry_run result somewhere? + # NOTE: get_metadata won't work here since filters not applied + if ( + curr_case_count + := task_info.materialized_test.dry_run().report.counter[ + "total" + ] + ): + # XXX: lb defined, ub? + adjusted_exec_time = time_info["execution_time"] * max( + curr_case_count / prev_case_count, + MULTITEST_EXEC_TIME_ADJUST_FACTOR_LB, + ) + self.logger.user_info( + "%s: adjust estimated total execution time %.2f -> %.2f " + "(prev total testcase number: %d, curr total testcase number: %d)", + uid, + time_info["execution_time"], + adjusted_exec_time, + prev_case_count, + curr_case_count, + ) + time_info["execution_time"] = adjusted_exec_time + time_info["testcase_count"] = curr_case_count + + def _calculate_part_runtime( + self, discovered: List[TaskInformation] + ) -> float: if self.cfg.auto_part_runtime_limit != "auto": return self.cfg.auto_part_runtime_limit @@ -991,7 +1041,7 @@ def _calculate_parts_and_weights( ) # @task_target(multitest_parts=...) uid = task_info.uid runtime_data: dict = self.cfg.runtime_data or {} - time_info = runtime_data.get(uid, None) + time_info: Optional[dict] = runtime_data.get(uid, None) partitioned: List[TaskInformation] = [] @@ -1025,7 +1075,7 @@ def _calculate_parts_and_weights( - time_info["teardown_time"] {time_info["teardown_time"]} ) ) - """ +""" try: num_of_parts = math.ceil( time_info["execution_time"] @@ -1063,7 +1113,7 @@ def _calculate_parts_and_weights( + time_info["teardown_time"] ) if time_info - else auto_part_runtime_limit + else int(auto_part_runtime_limit) ) self.logger.user_info( "%s: parts=%d, weight=%d", diff --git a/testplan/runnable/interactive/reloader.py b/testplan/runnable/interactive/reloader.py index 769d2c8c6..fcb12af9b 100644 --- a/testplan/runnable/interactive/reloader.py +++ b/testplan/runnable/interactive/reloader.py @@ -14,15 +14,19 @@ from testplan.common.utils import path as path_utils from testplan.common.utils import logger from testplan.common.utils import strings +from testplan.common.utils.logger import TESTPLAN_LOGGER from testplan.common.utils.package import import_tmp_module from testplan.testing.multitest import suite, MultiTest -def _patched_find_module(name, path=None): +def _patched_find_module(name, path=None, logger=None): """ modified from <3.13 branches, to handle namespace packages """ + if logger is None: + logger = TESTPLAN_LOGGER + # Old imp constants: _SEARCH_ERROR = 0 _PY_SOURCE = 1 @@ -64,6 +68,7 @@ def _patched_find_module(name, path=None): ): # ModuleFinder.find_module is designed to only return one package dir, # while namespace packages can have multiple + logger.debug("Skipping namespace package %s under %s", name, path) return None, None, ("", "", _NAMESPACE_IGNORED) if spec.loader is importlib.machinery.BuiltinImporter: @@ -150,7 +155,7 @@ def __init__(self, extra_deps=None, scheduled_modules=None): ) = self._build_dependencies() # Last recorded reload time for watched modules. - self._last_reload_time = {} # type: Dict[str, float] + self._last_reload_time: dict[str, float] = {} self._init_time = time.time() def reload(self, tests, rebuild_dependencies=False): @@ -663,7 +668,7 @@ def find_module(self, name, path, parent=None): path = self.path - return _patched_find_module(name, path) + return _patched_find_module(name, path, logger=self.logger) class _ModuleNode: diff --git a/testplan/testing/multitest/base.py b/testplan/testing/multitest/base.py index 9e7398f32..45c389038 100644 --- a/testplan/testing/multitest/base.py +++ b/testplan/testing/multitest/base.py @@ -1,7 +1,7 @@ """MultiTest test execution framework.""" import collections.abc -import concurrent +import concurrent.futures import functools import itertools import warnings diff --git a/tests/functional/testplan/runners/pools/test_auto_part.py b/tests/functional/testplan/runners/pools/test_auto_part.py index b5df23429..ab1adac95 100755 --- a/tests/functional/testplan/runners/pools/test_auto_part.py +++ b/tests/functional/testplan/runners/pools/test_auto_part.py @@ -1,5 +1,7 @@ import os +import math import tempfile +from itertools import count from pathlib import Path import pytest @@ -379,3 +381,65 @@ def test_auto_plan_runtime_target(): assert task.weight == 900 mockplan.run() assert pool.size == 6 + + +@pytest.mark.parametrize( + "tc, adjusted_exec_t, exp_weight, exp_parts", + [ + (10, 60, 35, 2), + (5, 120, 35, 4), + (20, 30, 35, 1), + (8, 75, 30, 3), + (100, 15, 20, 1), + ], + ids=count(0), +) +def test_multitest_weight_adjusted_by_relative_testcase_count( + tc, adjusted_exec_t, exp_weight, exp_parts +): + with tempfile.TemporaryDirectory() as runpath: + mockplan = TestplanMock( + "plan", + runpath=runpath, + merge_scheduled_parts=True, + auto_part_runtime_limit=40, + plan_runtime_target=80, + runtime_data={ + "Proj1-suite": { + "execution_time": 60, + "setup_time": 5, + "teardown_time": 0, + "testcase_count": tc, + } + }, + ) + pool = ProcessPool(name="MyPool", size="auto") + mockplan.add_resource(pool) + current_folder = Path(__file__).resolve().parent + assert pool.cfg.runtime_data == { + "Proj1-suite": { + "execution_time": 60, + "setup_time": 5, + "teardown_time": 0, + "testcase_count": tc, + } + } + # curr tc: 10 + mockplan.schedule_all( + path=current_folder / "discover_tasks", + name_pattern=r".*auto_parts_tasks\.py$", + resource="MyPool", + ) + assert pool.cfg.runtime_data == { + "Proj1-suite": { + "execution_time": adjusted_exec_t, + "setup_time": 5, + "teardown_time": 0, + "testcase_count": 10, + } + } + assert len(pool.added_items) == exp_parts + for task in pool.added_items.values(): + assert task.weight == exp_weight + mockplan.run() + assert pool.size == math.ceil(exp_weight * exp_parts / 80)