Deprecate passing iterator argvalues to parametrize

changelog/13409.deprecation.rst

@@ -0,0 +1,8 @@
+Using non-:class:`~collections.abc.Collection` iterables (such as generators, iterators, or custom iterable objects) for the ``argvalues`` parameter in :ref:`@pytest.mark.parametrize <pytest.mark.parametrize ref>` and :meth:`metafunc.parametrize <pytest.Metafunc.parametrize>` is now deprecated.
+
+These iterables get exhausted after the first iteration,
+leading to tests getting unexpectedly skipped in cases such as running :func:`pytest.main()` multiple times,
+using class-level parametrize decorators,
+or collecting tests multiple times.
+
+See :ref:`parametrize-iterators` for details and suggestions.

doc/en/deprecations.rst

@@ -15,6 +15,67 @@ Below is a complete list of all pytest features which are considered deprecated.
 :class:`~pytest.PytestWarning` or subclasses, which can be filtered using :ref:`standard warning filters <warnings>`.
 
 
+.. _parametrize-iterators:
+
+Non-Collection iterables in ``@pytest.mark.parametrize``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. deprecated:: 9.1
+
+Using non-:class:`~collections.abc.Collection` iterables (such as generators, iterators, or custom iterable objects)
+for the ``argvalues`` parameter in :ref:`@pytest.mark.parametrize <pytest.mark.parametrize ref>`
+and :meth:`metafunc.parametrize <pytest.Metafunc.parametrize>` is deprecated.
+
+These iterables get exhausted after the first iteration, leads to tests getting unexpectedly skipped in cases such as:
+
+* Running :func`pytest.main()` multiple times in the same process
+* Using class-level parametrize decorators where the same mark is applied to multiple test methods
+* Collecting tests multiple times
+
+Example of problematic code:
+
+.. code-block:: python
+
+    import pytest
+
+
+    def data_generator():
+        yield 1
+        yield 2
+
+
+    @pytest.mark.parametrize("n", data_generator())
+    class Test:
+        def test_1(self, n):
+            pass
+
+        # test_2 will be skipped because data_generator() is exhausted.
+        def test_2(self, n):
+            pass
+
+You can fix it by convert generators and iterators to lists or tuples:
+
+.. code-block:: python
+
+    import pytest
+
+
+    def data_generator():
+        yield 1
+        yield 2
+
+
+    @pytest.mark.parametrize("n", list(data_generator()))
+    class Test:
+        def test_1(self, n):
+            pass
+
+        def test_2(self, n):
+            pass
+
+Note that :class:`range` objects are ``Collection`` and are not affected by this deprecation.
+
+
 .. _monkeypatch-fixup-namespace-packages:
 
 ``monkeypatch.syspath_prepend`` with legacy namespace packages

pyproject.toml

@@ -532,6 +532,7 @@ warn_unreachable = true
 warn_unused_configs = true
 no_implicit_reexport = true
 warn_unused_ignores = true
+enable_error_code = [ "deprecated" ]
 
 [tool.pyright]
 include = [

src/_pytest/compat.py

@@ -15,6 +15,7 @@
 from typing import Any
 from typing import Final
 from typing import NoReturn
+from typing import TYPE_CHECKING
 
 import py
 
@@ -311,3 +312,17 @@ def running_on_ci() -> bool:
     # Only enable CI mode if one of these env variables is defined and non-empty.
     env_vars = ["CI", "BUILD_NUMBER"]
     return any(os.environ.get(var) for var in env_vars)
+
+
+if sys.version_info >= (3, 13):
+    from warnings import deprecated as deprecated
+else:
+    if TYPE_CHECKING:
+        from typing_extensions import deprecated as deprecated
+    else:
+
+        def deprecated(msg, /, *, category=None, stacklevel=1):
+            def decorator(func):
+                return func
+
+            return decorator

src/_pytest/deprecated.py

@@ -75,6 +75,13 @@
     "See https://docs.pytest.org/en/stable/deprecations.html#monkeypatch-fixup-namespace-packages"
 )
 
+PARAMETRIZE_NON_COLLECTION_ITERABLE = UnformattedWarning(
+    PytestRemovedIn10Warning,
+    "The {param_name} parameter in parametrize uses a non-Collection iterable of type '{type_name}'.\n"
+    "Please convert to a list or tuple.\n"
+    "See https://docs.pytest.org/en/stable/deprecations.html#parametrize-iterators",
+)
+
 # You want to make some `__init__` or function "private".
 #
 #   def my_private_function(some, args):

src/_pytest/logging.py

@@ -513,7 +513,7 @@ def _force_enable_logging(
 
         if isinstance(level, str):
             # Try to translate the level string to an int for `logging.disable()`
-            level = logging.getLevelName(level)
+            level = logging.getLevelName(level)  # type: ignore[deprecated]
 
         if not isinstance(level, int):
             # The level provided was not valid, so just un-disable all logging.

src/_pytest/mark/structures.py

@@ -21,11 +21,13 @@
 import warnings
 
 from .._code import getfslineno
+from ..compat import deprecated
 from ..compat import NOTSET
 from ..compat import NotSetType
 from _pytest.config import Config
 from _pytest.deprecated import check_ispytest
 from _pytest.deprecated import MARKED_FIXTURE
+from _pytest.deprecated import PARAMETRIZE_NON_COLLECTION_ITERABLE
 from _pytest.outcomes import fail
 from _pytest.raises import AbstractRaises
 from _pytest.scope import _ScopeName
@@ -193,6 +195,15 @@ def _for_parametrize(
         config: Config,
         nodeid: str,
     ) -> tuple[Sequence[str], list[ParameterSet]]:
+        if not isinstance(argvalues, Collection):
+            warnings.warn(
+                PARAMETRIZE_NON_COLLECTION_ITERABLE.format(
+                    param_name="argvalues",
+                    type_name=type(argvalues).__name__,
+                ),
+                stacklevel=3,
+            )
+
         argnames, force_tuple = cls._parse_parametrize_args(argnames, argvalues)
         parameters = cls._parse_parametrize_parameters(argvalues, force_tuple)
         del argvalues
@@ -508,7 +519,25 @@ def __call__(
         ) -> MarkDecorator: ...
 
     class _ParametrizeMarkDecorator(MarkDecorator):
-        def __call__(  # type: ignore[override]
+        @overload  # type: ignore[override,no-overload-impl]
+        def __call__(
+            self,
+            argnames: str | Sequence[str],
+            argvalues: Collection[ParameterSet | Sequence[object] | object],
+            *,
+            indirect: bool | Sequence[str] = ...,
+            ids: Iterable[None | str | float | int | bool]
+            | Callable[[Any], object | None]
+            | None = ...,
+            scope: _ScopeName | None = ...,
+        ) -> MarkDecorator: ...
+
+        @overload
+        @deprecated(
+            "Passing a non-Collection iterable to the 'argvalues' parameter of @pytest.mark.parametrize is deprecated. "
+            "Convert argvalues to a list or tuple.",
+        )
+        def __call__(
             self,
             argnames: str | Sequence[str],
             argvalues: Iterable[ParameterSet | Sequence[object] | object],

src/_pytest/python.py

@@ -7,6 +7,7 @@
 from collections import Counter
 from collections import defaultdict
 from collections.abc import Callable
+from collections.abc import Collection
 from collections.abc import Generator
 from collections.abc import Iterable
 from collections.abc import Iterator
@@ -28,6 +29,7 @@
 from typing import final
 from typing import Literal
 from typing import NoReturn
+from typing import overload
 from typing import TYPE_CHECKING
 import warnings
 
@@ -41,6 +43,7 @@
 from _pytest._code.code import Traceback
 from _pytest._io.saferepr import saferepr
 from _pytest.compat import ascii_escaped
+from _pytest.compat import deprecated
 from _pytest.compat import get_default_arg_names
 from _pytest.compat import get_real_func
 from _pytest.compat import getimfunc
@@ -1209,6 +1212,34 @@ def __init__(
 
         self._params_directness: dict[str, Literal["indirect", "direct"]] = {}
 
+    @overload
+    def parametrize(
+        self,
+        argnames: str | Sequence[str],
+        argvalues: Collection[ParameterSet | Sequence[object] | object],
+        indirect: bool | Sequence[str] = False,
+        ids: Iterable[object | None] | Callable[[Any], object | None] | None = None,
+        scope: _ScopeName | None = None,
+        *,
+        _param_mark: Mark | None = None,
+    ) -> None: ...
+
+    @overload
+    @deprecated(
+        "Passing a non-Collection iterable to the 'argvalues' parameter of metafunc.parametrize is deprecated. "
+        "Convert argvalues to a list or tuple.",
+    )
+    def parametrize(
+        self,
+        argnames: str | Sequence[str],
+        argvalues: Iterable[ParameterSet | Sequence[object] | object],
+        indirect: bool | Sequence[str] = False,
+        ids: Iterable[object | None] | Callable[[Any], object | None] | None = None,
+        scope: _ScopeName | None = None,
+        *,
+        _param_mark: Mark | None = None,
+    ) -> None: ...
+
     def parametrize(
         self,
         argnames: str | Sequence[str],
@@ -1247,6 +1278,12 @@ def parametrize(
             N-tuples, where each tuple-element specifies a value for its
             respective argname.
 
+            .. versionchanged:: 9.1
+
+                Passing a non-:class:`~collections.abc.Collection` iterable
+                (such as a generator or iterator) is deprecated. See
+                :ref:`parametrize-iterators` for details.
+
         :param indirect:
             A list of arguments' names (subset of argnames) or a boolean.
             If True the list contains all names from the argnames. Each

testing/_py/test_local.py

@@ -734,12 +734,8 @@ def test_dump(self, tmpdir, bin):
     def test_setmtime(self):
         import tempfile
 
-        try:
-            fd, name = tempfile.mkstemp()
-            os.close(fd)
-        except AttributeError:
-            name = tempfile.mktemp()
-            open(name, "w").close()
+        fd, name = tempfile.mkstemp()
+        os.close(fd)
         try:
             # Do not use _pytest.timing here, as we do not want time mocking to affect this test.
             mtime = int(time.time()) - 100

testing/python/metafunc.py

@@ -93,7 +93,7 @@ def func(x, y):
             metafunc.parametrize("y", [5, 6])
 
         with pytest.raises(TypeError, match=r"^ids must be a callable or an iterable$"):
-            metafunc.parametrize("y", [5, 6], ids=42)  # type: ignore[arg-type]
+            metafunc.parametrize("y", [5, 6], ids=42)  # type: ignore[call-overload]
 
     def test_parametrize_error_iterator(self) -> None:
         def func(x):
@@ -134,7 +134,7 @@ def func(x):
             fail.Exception,
             match=r"parametrize\(\) call in func got an unexpected scope value 'doggy'",
         ):
-            metafunc.parametrize("x", [1], scope="doggy")  # type: ignore[arg-type]
+            metafunc.parametrize("x", [1], scope="doggy")  # type: ignore[call-overload]
 
     def test_parametrize_request_name(self, pytester: Pytester) -> None:
         """Show proper error  when 'request' is used as a parameter name in parametrize (#6183)"""
@@ -813,7 +813,7 @@ def func(x, y):
             fail.Exception,
             match="In func: expected Sequence or boolean for indirect, got dict",
         ):
-            metafunc.parametrize("x, y", [("a", "b")], indirect={})  # type: ignore[arg-type]
+            metafunc.parametrize("x, y", [("a", "b")], indirect={})  # type: ignore[call-overload]
 
     def test_parametrize_indirect_list_functional(self, pytester: Pytester) -> None:
         """
@@ -1123,6 +1123,24 @@ def test_3(self, arg, arg2):
         """
         )
 
+    def test_parametrize_iterator_deprecation(self) -> None:
+        """Test that using iterators for argvalues raises a deprecation warning."""
+
+        def func(x):
+            pass
+
+        def data_generator():
+            yield 1
+            yield 2
+
+        metafunc = self.Metafunc(func)
+
+        with pytest.warns(
+            pytest.PytestRemovedIn10Warning,
+            match=r"The argvalues parameter.*uses a non-Collection iterable",
+        ):
+            metafunc.parametrize("x", data_generator())
+
 
 class TestMetafuncFunctional:
     def test_attributes(self, pytester: Pytester) -> None:
@@ -1682,6 +1700,62 @@ def test_3(self, fixture):
             ]
         )
 
+    def test_parametrize_generator_multiple_runs(self, pytester: Pytester) -> None:
+        """Test that generators in parametrize work with multiple pytest.main() (deprecated)."""
+        pytester.makepyfile(
+            """
+            import pytest
+
+            def data_generator():
+                yield 1
+                yield 2
+
+            @pytest.mark.parametrize("bar", data_generator())
+            def test_foo(bar):
+                pass
+
+            if __name__ == '__main__':
+                args = ["-q", "--collect-only"]
+                pytest.main(args)  # First run - should work with warning
+                pytest.main(args)  # Second run - should also work with warning
+            """
+        )
+        result = pytester.runpytest("-W", "default")
+        # Should see the deprecation warnings.
+        result.stdout.fnmatch_lines(
+            [
+                "*PytestRemovedIn10Warning: The argvalues parameter*",
+            ]
+        )
+
+    def test_parametrize_iterator_class_multiple_tests(
+        self, pytester: Pytester
+    ) -> None:
+        """Test that iterators in parametrize on a class get exhausted (deprecated)."""
+        pytester.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.parametrize("n", iter(range(2)))
+            class Test:
+                def test_1(self, n):
+                    pass
+
+                def test_2(self, n):
+                    pass
+            """
+        )
+        result = pytester.runpytest("-v", "-W", "default")
+        # Iterator gets exhausted after first test, second test gets no parameters.
+        # This is deprecated.
+        result.assert_outcomes(passed=2, skipped=1)
+        # Should see the deprecation warnings.
+        result.stdout.fnmatch_lines(
+            [
+                "*PytestRemovedIn10Warning: The argvalues parameter*",
+            ]
+        )
+
 
 class TestMetafuncFunctionalAuto:
     """Tests related to automatically find out the correct scope for