diff --git a/README.rst b/README.rst index bdd32828..952b85b0 100644 --- a/README.rst +++ b/README.rst @@ -67,6 +67,20 @@ source code control and committed before you apply a mutant! If during the installation you get an error for the `libcst` dependency mentioning the lack of a rust compiler on your system, it is because your architecture does not have a prebuilt binary for `libcst` and it requires both `rustc` and `cargo` from the [rust toolchain](https://www.rust-lang.org/tools/install) to be built. This is known for at least the `x86_64-darwin` architecture. +Wildcards for testing mutants +----------------------------- + +Unix filename pattern matching style on mutants is supported. Example: + +.. code-block:: console + + mutmut run "my_module*" + mutmut run "my_module.my_function*" + +In the `browse` TUI you can press `f` to retest a function, and `m` to retest +an entire module. + + Configuration ------------- @@ -76,7 +90,7 @@ In `setup.cfg` in the root of your project you can configure mutmut if you need [mutmut] paths_to_mutate=src/ - tests_dir=tests/ + pytest_add_cli_args_test_selection=tests/ If you use `pyproject.toml`, you must specify the paths as array in a `tool.mutmut` section: @@ -84,27 +98,13 @@ If you use `pyproject.toml`, you must specify the paths as array in a `tool.mutm [tool.mutmut] paths_to_mutate = [ "src/" ] - tests_dir = [ "tests/" ] + pytest_add_cli_args_test_selection= [ "tests/" ] See below for more options for configuring mutmut. -Wildcards for testing mutants ------------------------------ - -Unix filename pattern matching style on mutants is supported. Example: - -.. code-block:: console - - mutmut run "my_module*" - mutmut run "my_module.my_function*" - -In the `browse` TUI you can press `f` to retest a function, and `m` to retest -an entire module. - - "also copy" files ------------------ +~~~~~~~~~~~~~~~~~ To run the full test suite some files are often needed above the tests and the source. You can configure to copy extra files that you need by adding @@ -118,7 +118,7 @@ directories and files to `also_copy` in your `setup.cfg`: Limit stack depth ------------------ +~~~~~~~~~~~~~~~~~ In big code bases some functions are called incidentally by huge swaths of the codebase, but you really don't want tests that hit those executions to count @@ -142,7 +142,7 @@ caught. Exclude files from mutation ---------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can exclude files from mutation in `setup.cfg`: @@ -153,7 +153,7 @@ You can exclude files from mutation in `setup.cfg`: Enable coverage.py filtering of lines to mutate ------------------------------------------------ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, mutmut will mutate only functions that are called. But, if you would like a finer grained (line-level) check for coverage, mutmut can use coverage.py to do that. @@ -168,7 +168,7 @@ If you only want to mutate lines that are called (according to coverage.py), you Enable debug output (increase verbosity) ----------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, mutmut "swallows" all the test output etc. so that you get a nice clean output. @@ -182,7 +182,7 @@ to failing tests. Whitelisting ------------- +~~~~~~~~~~~~ You can mark lines like this: @@ -198,6 +198,27 @@ whitelist lines are: to continue, but it's slower. +Modifying pytest arguments +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can add and override pytest arguments: + +.. code-block:: python + + # for CLI args that select or deselect tests, use `pytest_add_cli_args_test_selection` + pytest_add_cli_args_test_selection = ["-m", "not fail", "-k=test_include"] + + # for other CLI args, use `pytest_add_cli_args` + pytest_add_cli_args = ["-p", "no:some_plugin"] # disable a plugin + pytest_add_cli_args = ["-o", "xfail_strict=False"] # overrides xfail_strict from your normal config + + # if you want to ignore the normal pytest configuration + # you can specify a diferent pytest ini file to be used + pytest_add_cli_args = ["-c", "mutmut_pytest.ini"] + also_copy = ["mutmut_pytest.ini"] + + + Example mutations ----------------- diff --git a/e2e_projects/config/pyproject.toml b/e2e_projects/config/pyproject.toml index 5a994b54..10e196cc 100644 --- a/e2e_projects/config/pyproject.toml +++ b/e2e_projects/config/pyproject.toml @@ -29,3 +29,11 @@ do_not_mutate = [ "*ignore*" ] also_copy = [ "data" ] max_stack_depth=8 # Includes frames by mutmut, see https://github.com/boxed/mutmut/issues/378 tests_dir = [ "tests/main/" ] +# verify that we can override options with pytest_add_cli_args +pytest_add_cli_args = ["-o", "xfail_strict=False"] +# verify test exclusion (-m 'not fail') and test inclusion (-k=test_include) +pytest_add_cli_args_test_selection = [ "-m", "not fail", "-k=test_include"] + +[tool.pytest.ini_options] +xfail_strict = true +markers = [ "fail: tests that should be ignored with mutmut" ] \ No newline at end of file diff --git a/e2e_projects/config/tests/ignored/test_ignored.py b/e2e_projects/config/tests/ignored/test_ignored.py index db57d7b4..8a5f51ee 100644 --- a/e2e_projects/config/tests/ignored/test_ignored.py +++ b/e2e_projects/config/tests/ignored/test_ignored.py @@ -1,4 +1,5 @@ from config_pkg.math import func_with_no_tests -def test_func_with_no_tests(): +# ignored, because tests_dir specifies only the main directory +def test_include_func_with_no_tests(): assert func_with_no_tests() == 420 diff --git a/e2e_projects/config/tests/main/test_main.py b/e2e_projects/config/tests/main/test_main.py index ef066d0b..c632e4bc 100644 --- a/e2e_projects/config/tests/main/test_main.py +++ b/e2e_projects/config/tests/main/test_main.py @@ -1,26 +1,41 @@ import json +import pytest from pathlib import Path from config_pkg import hello from config_pkg.math import add, call_depth_two from config_pkg.ignore_me import this_function_shall_NOT_be_mutated -def test_hello(): +def test_include_hello(): assert hello() == "Hello from config!" -def test_add(): +def test_include_add(): assert add(1, 0) == 1 -def test_non_mutated_function(): +def test_include_non_mutated_function(): assert this_function_shall_NOT_be_mutated() == 3 -def test_max_stack_depth(): +def test_include_max_stack_depth(): # This test should only cover functions up to some depth # For more context, see https://github.com/boxed/mutmut/issues/378 assert call_depth_two() == 2 -def test_data_exists(): +def test_include_data_exists(): path = (Path("data") / "data.json").resolve() assert path.exists() with open(path) as f: data = json.load(f) - assert data['comment'] == 'this should be copied to the mutants folder' \ No newline at end of file + assert data['comment'] == 'this should be copied to the mutants folder' + +# ignored, because it does not match -k 'test_include' +def test_should_be_ignored(): + assert 'This test should be ignored' == 1234 + +@pytest.mark.xfail +def test_include_xfail_that_does_not_fail(): + # verify that we can override the xfail=strict from the pytest settings + assert 1 == 1 + +# ignored, because of -m 'not fail' +@pytest.mark.fail +def test_include_that_should_be_ignored(): + assert 'This test should be ignored' == 1234 \ No newline at end of file diff --git a/mutmut/__main__.py b/mutmut/__main__.py index 875bba31..86020678 100644 --- a/mutmut/__main__.py +++ b/mutmut/__main__.py @@ -420,13 +420,22 @@ def new_tests(self): class PytestRunner(TestRunner): + def __init__(self): + self._pytest_add_cli_args: List[str] = mutmut.config.pytest_add_cli_args + self._pytest_add_cli_args_test_selection: List[str] = mutmut.config.pytest_add_cli_args_test_selection + + # tests_dir is a special case of a test selection option, + # so also use pytest_add_cli_args_test_selection for the implementation + self._pytest_add_cli_args_test_selection += mutmut.config.tests_dir + + # noinspection PyMethodMayBeStatic def execute_pytest(self, params: list[str], **kwargs): import pytest - params += ['--rootdir=.'] + params = ['--rootdir=.'] + params + self._pytest_add_cli_args if mutmut.config.debug: params = ['-vv'] + params - print('python -m pytest ', ' '.join(params)) + print('python -m pytest ', ' '.join([f'"{param}"' for param in params])) exit_code = int(pytest.main(params, **kwargs)) if mutmut.config.debug: print(' exit code', exit_code) @@ -457,9 +466,7 @@ def pytest_runtest_makereport(self, item, call): if tests: pytest_args += list(tests) else: - tests_dir = mutmut.config.tests_dir - if tests_dir: - pytest_args += tests_dir + pytest_args += self._pytest_add_cli_args_test_selection with change_cwd('mutants'): return int(self.execute_pytest(pytest_args, plugins=[stats_collector])) @@ -468,38 +475,39 @@ def run_tests(self, *, mutant_name, tests): if tests: pytest_args += list(tests) else: - tests_dir = mutmut.config.tests_dir - if tests_dir: - pytest_args += tests_dir + pytest_args += self._pytest_add_cli_args_test_selection with change_cwd('mutants'): return int(self.execute_pytest(pytest_args)) def run_forced_fail(self): - pytest_args = ['-x', '-q'] - tests_dir = mutmut.config.tests_dir - if tests_dir: - pytest_args += tests_dir + pytest_args = ['-x', '-q'] + self._pytest_add_cli_args_test_selection with change_cwd('mutants'): return int(self.execute_pytest(pytest_args)) def list_all_tests(self): class TestsCollector: + def __init__(self): + self.collected_nodeids = set() + self.deselected_nodeids = set() + def pytest_collection_modifyitems(self, items): - self.nodeids = {item.nodeid for item in items} + self.collected_nodeids |= {item.nodeid for item in items} + + def pytest_deselected(self, items): + self.deselected_nodeids |= {item.nodeid for item in items} collector = TestsCollector() tests_dir = mutmut.config.tests_dir - pytest_args = ['-x', '-q', '--collect-only'] - if tests_dir: - pytest_args += tests_dir + pytest_args = ['-x', '-q', '--collect-only'] + self._pytest_add_cli_args_test_selection with change_cwd('mutants'): exit_code = int(self.execute_pytest(pytest_args, plugins=[collector])) if exit_code != 0: raise CollectTestsFailedException() - return ListAllTestsResult(ids=collector.nodeids) + selected_nodeids = collector.collected_nodeids - collector.deselected_nodeids + return ListAllTestsResult(ids=selected_nodeids) class HammettRunner(TestRunner): @@ -701,8 +709,10 @@ class Config: max_stack_depth: int debug: bool paths_to_mutate: List[Path] - tests_dir: List[str] = None - mutate_only_covered_lines: bool = False + pytest_add_cli_args: List[str] + pytest_add_cli_args_test_selection: List[str] + tests_dir: List[str] + mutate_only_covered_lines: bool def should_ignore_for_mutation(self, path): if not str(path).endswith('.py'): @@ -739,7 +749,7 @@ def s(key, default): config_parser = ConfigParser() config_parser.read('setup.cfg') - def s(key, default): + def s(key: str, default): try: result = config_parser.get('mutmut', key) except (NoOptionError, NoSectionError): @@ -784,6 +794,8 @@ def load_config(): for y in s('paths_to_mutate', []) ] or guess_paths_to_mutate(), tests_dir=s('tests_dir', []), + pytest_add_cli_args=s('pytest_add_cli_args', []), + pytest_add_cli_args_test_selection=s('pytest_add_cli_args_test_selection', []), )