Add pytest_add_cli_args and pytest_add_cli_args_test_selection configs (#440)

* Add pytest_add_cli_args and pytest_add_cli_args_test_selection configs

* Only list selected tests

Necessary if we run `mutmut run` twice with some pytest_add_cli_args_test_selection.

* Document pytest_add_cli_args_test_selection instead of tests_dir

Author: Otto-AA

README.rst

@@ -67,6 +67,20 @@ source code control and committed before you apply a mutant!
 If during the installation you get an error for the `libcst` dependency mentioning the lack of a rust compiler on your system, it is because your architecture does not have a prebuilt binary for `libcst` and it requires both `rustc` and `cargo` from the [rust toolchain](https://www.rust-lang.org/tools/install) to be built. This is known for at least the `x86_64-darwin` architecture.
 
 
+Wildcards for testing mutants
+-----------------------------
+
+Unix filename pattern matching style on mutants is supported. Example:
+
+.. code-block:: console
+
+    mutmut run "my_module*"
+    mutmut run "my_module.my_function*"
+
+In the `browse` TUI you can press `f` to retest a function, and `m` to retest
+an entire module.
+
+
 Configuration
 -------------
 
@@ -76,35 +90,21 @@ In `setup.cfg` in the root of your project you can configure mutmut if you need
 
     [mutmut]
     paths_to_mutate=src/
-    tests_dir=tests/
+    pytest_add_cli_args_test_selection=tests/
 
 If you use `pyproject.toml`, you must specify the paths as array in a `tool.mutmut` section:
 
 .. code-block:: toml
 
     [tool.mutmut]
     paths_to_mutate = [ "src/" ]
-    tests_dir = [ "tests/" ]
+    pytest_add_cli_args_test_selection= [ "tests/" ]
 
 See below for more options for configuring mutmut.
 
 
-Wildcards for testing mutants
------------------------------
-
-Unix filename pattern matching style on mutants is supported. Example:
-
-.. code-block:: console
-
-    mutmut run "my_module*"
-    mutmut run "my_module.my_function*"
-
-In the `browse` TUI you can press `f` to retest a function, and `m` to retest
-an entire module.
-
-
 "also copy" files
------------------
+~~~~~~~~~~~~~~~~~
 
 To run the full test suite some files are often needed above the tests and the
 source. You can configure to copy extra files that you need by adding
@@ -118,7 +118,7 @@ directories and files to `also_copy` in your `setup.cfg`:
 
 
 Limit stack depth
------------------
+~~~~~~~~~~~~~~~~~
 
 In big code bases some functions are called incidentally by huge swaths of the
 codebase, but you really don't want tests that hit those executions to count
@@ -142,7 +142,7 @@ caught.
 
 
 Exclude files from mutation
----------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can exclude files from mutation in `setup.cfg`:
 
@@ -153,7 +153,7 @@ You can exclude files from mutation in `setup.cfg`:
 
 
 Enable coverage.py filtering of lines to mutate
------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 By default, mutmut will mutate only functions that are called. But, if you would like a finer grained (line-level)
 check for coverage, mutmut can use coverage.py to do that.
@@ -168,7 +168,7 @@ If you only want to mutate lines that are called (according to coverage.py), you
 
 
 Enable debug output (increase verbosity)
-----------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 By default, mutmut "swallows" all the test output etc. so that you get a nice clean output.
 
@@ -182,7 +182,7 @@ to failing tests.
 
 
 Whitelisting
-------------
+~~~~~~~~~~~~
 
 You can mark lines like this:
 
@@ -198,6 +198,27 @@ whitelist lines are:
   to continue, but it's slower.
 
 
+Modifying pytest arguments
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can add and override pytest arguments:
+
+.. code-block:: python
+
+    # for CLI args that select or deselect tests, use `pytest_add_cli_args_test_selection`
+    pytest_add_cli_args_test_selection = ["-m", "not fail", "-k=test_include"]
+
+    # for other CLI args, use `pytest_add_cli_args`
+    pytest_add_cli_args = ["-p", "no:some_plugin"] # disable a plugin
+    pytest_add_cli_args = ["-o", "xfail_strict=False"] # overrides xfail_strict from your normal config
+
+    # if you want to ignore the normal pytest configuration
+    # you can specify a diferent pytest ini file to be used
+    pytest_add_cli_args = ["-c", "mutmut_pytest.ini"]
+    also_copy = ["mutmut_pytest.ini"]
+
+
+
 Example mutations
 -----------------
 

e2e_projects/config/pyproject.toml

@@ -29,3 +29,11 @@ do_not_mutate = [ "*ignore*" ]
 also_copy = [ "data" ]
 max_stack_depth=8 # Includes frames by mutmut, see https://github.com/boxed/mutmut/issues/378
 tests_dir = [ "tests/main/" ]
+# verify that we can override options with pytest_add_cli_args
+pytest_add_cli_args = ["-o", "xfail_strict=False"]
+# verify test exclusion (-m 'not fail') and test inclusion (-k=test_include)
+pytest_add_cli_args_test_selection = [ "-m", "not fail", "-k=test_include"]
+
+[tool.pytest.ini_options]
+xfail_strict = true
+markers = [ "fail: tests that should be ignored with mutmut" ]

e2e_projects/config/tests/ignored/test_ignored.py

@@ -1,4 +1,5 @@
 from config_pkg.math import func_with_no_tests
 
-def test_func_with_no_tests():
+# ignored, because tests_dir specifies only the main directory
+def test_include_func_with_no_tests():
     assert func_with_no_tests() == 420

e2e_projects/config/tests/main/test_main.py

@@ -1,26 +1,41 @@
 import json
+import pytest
 from pathlib import Path
 from config_pkg import hello
 from config_pkg.math import add, call_depth_two
 from config_pkg.ignore_me import this_function_shall_NOT_be_mutated
 
-def test_hello():
+def test_include_hello():
     assert hello() == "Hello from config!"
 
-def test_add():
+def test_include_add():
     assert add(1, 0) == 1
 
-def test_non_mutated_function():
+def test_include_non_mutated_function():
     assert this_function_shall_NOT_be_mutated() == 3
 
-def test_max_stack_depth():
+def test_include_max_stack_depth():
     # This test should only cover functions up to some depth
     # For more context, see https://github.com/boxed/mutmut/issues/378
     assert call_depth_two() == 2
 
-def test_data_exists():
+def test_include_data_exists():
     path = (Path("data") / "data.json").resolve()
     assert path.exists()
     with open(path) as f:
         data = json.load(f)
-        assert data['comment'] == 'this should be copied to the mutants folder'
+        assert data['comment'] == 'this should be copied to the mutants folder'
+
+# ignored, because it does not match -k 'test_include' 
+def test_should_be_ignored():
+    assert 'This test should be ignored' == 1234
+
+@pytest.mark.xfail
+def test_include_xfail_that_does_not_fail():
+    # verify that we can override the xfail=strict from the pytest settings
+    assert 1 == 1
+
+# ignored, because of -m 'not fail'
+@pytest.mark.fail
+def test_include_that_should_be_ignored():
+    assert 'This test should be ignored' == 1234

mutmut/__main__.py

@@ -420,13 +420,22 @@ def new_tests(self):
 
 
 class PytestRunner(TestRunner):
+    def __init__(self):
+        self._pytest_add_cli_args: List[str] = mutmut.config.pytest_add_cli_args
+        self._pytest_add_cli_args_test_selection: List[str] = mutmut.config.pytest_add_cli_args_test_selection
+
+        # tests_dir is a special case of a test selection option,
+        # so also use pytest_add_cli_args_test_selection for the implementation
+        self._pytest_add_cli_args_test_selection += mutmut.config.tests_dir
+
+
     # noinspection PyMethodMayBeStatic
     def execute_pytest(self, params: list[str], **kwargs):
         import pytest
-        params += ['--rootdir=.']
+        params = ['--rootdir=.'] + params + self._pytest_add_cli_args
         if mutmut.config.debug:
             params = ['-vv'] + params
-            print('python -m pytest ', ' '.join(params))
+            print('python -m pytest ', ' '.join([f'"{param}"' for param in params]))
         exit_code = int(pytest.main(params, **kwargs))
         if mutmut.config.debug:
             print('    exit code', exit_code)
@@ -457,9 +466,7 @@ def pytest_runtest_makereport(self, item, call):
         if tests:
             pytest_args += list(tests)
         else:
-            tests_dir = mutmut.config.tests_dir
-            if tests_dir:
-                pytest_args += tests_dir
+            pytest_args += self._pytest_add_cli_args_test_selection
         with change_cwd('mutants'):
             return int(self.execute_pytest(pytest_args, plugins=[stats_collector]))
 
@@ -468,38 +475,39 @@ def run_tests(self, *, mutant_name, tests):
         if tests:
             pytest_args += list(tests)
         else:
-            tests_dir = mutmut.config.tests_dir
-            if tests_dir:
-                pytest_args += tests_dir
+            pytest_args += self._pytest_add_cli_args_test_selection
         with change_cwd('mutants'):
             return int(self.execute_pytest(pytest_args))
 
     def run_forced_fail(self):
-        pytest_args = ['-x', '-q']
-        tests_dir = mutmut.config.tests_dir
-        if tests_dir:
-            pytest_args += tests_dir
+        pytest_args = ['-x', '-q'] + self._pytest_add_cli_args_test_selection
         with change_cwd('mutants'):
             return int(self.execute_pytest(pytest_args))
 
     def list_all_tests(self):
         class TestsCollector:
+            def __init__(self):
+                self.collected_nodeids = set()
+                self.deselected_nodeids = set()
+
             def pytest_collection_modifyitems(self, items):
-                self.nodeids = {item.nodeid for item in items}
+                self.collected_nodeids |= {item.nodeid for item in items}
+
+            def pytest_deselected(self, items):
+                self.deselected_nodeids |= {item.nodeid for item in items}
 
         collector = TestsCollector()
 
         tests_dir = mutmut.config.tests_dir
-        pytest_args = ['-x', '-q', '--collect-only']
-        if tests_dir:
-            pytest_args += tests_dir
+        pytest_args = ['-x', '-q', '--collect-only'] + self._pytest_add_cli_args_test_selection
 
         with change_cwd('mutants'):
             exit_code = int(self.execute_pytest(pytest_args, plugins=[collector]))
             if exit_code != 0:
                 raise CollectTestsFailedException()
 
-        return ListAllTestsResult(ids=collector.nodeids)
+        selected_nodeids = collector.collected_nodeids - collector.deselected_nodeids
+        return ListAllTestsResult(ids=selected_nodeids)
 
 
 class HammettRunner(TestRunner):
@@ -701,8 +709,10 @@ class Config:
     max_stack_depth: int
     debug: bool
     paths_to_mutate: List[Path]
-    tests_dir: List[str] = None
-    mutate_only_covered_lines: bool = False
+    pytest_add_cli_args: List[str]
+    pytest_add_cli_args_test_selection: List[str]
+    tests_dir: List[str]
+    mutate_only_covered_lines: bool
 
     def should_ignore_for_mutation(self, path):
         if not str(path).endswith('.py'):
@@ -739,7 +749,7 @@ def s(key, default):
     config_parser = ConfigParser()
     config_parser.read('setup.cfg')
 
-    def s(key, default):
+    def s(key: str, default):
         try:
             result = config_parser.get('mutmut', key)
         except (NoOptionError, NoSectionError):
@@ -784,6 +794,8 @@ def load_config():
             for y in s('paths_to_mutate', [])
         ] or guess_paths_to_mutate(),
         tests_dir=s('tests_dir', []),
+        pytest_add_cli_args=s('pytest_add_cli_args', []),
+        pytest_add_cli_args_test_selection=s('pytest_add_cli_args_test_selection', []),
     )