Run tests in parallel with pytest-xdist (#9945)

Author: hugovk

.ci/build.sh

@@ -2,6 +2,5 @@
 
 set -e
 
-python3 -m coverage erase
 make clean
 make install-coverage

.ci/requirements-mypy.txt

@@ -12,4 +12,5 @@ pytest
 types-atheris
 types-defusedxml
 types-olefile
+types-psutil
 types-setuptools

.ci/test.cmd

@@ -1,3 +1,3 @@
 python.exe -c "from PIL import Image"
 IF ERRORLEVEL 1 EXIT /B
-python.exe -bb -m pytest -vv -x -W always --cov PIL --cov Tests --cov-report term --cov-report xml Tests
+python.exe -bb -m pytest -vv -x -W always --numprocesses=logical --dist=worksteal --cov PIL --cov Tests --cov-report term --cov-report xml Tests

.ci/test.sh

@@ -4,4 +4,4 @@ set -e
 
 python3 -c "from PIL import Image"
 
-python3 -bb -m pytest -vv -x -W always --cov PIL --cov Tests --cov-report term --cov-report xml Tests $REVERSE
+python3 -bb -m pytest -vv -x -W always --numprocesses=logical --dist=worksteal --cov PIL --cov Tests --cov-report term --cov-report xml Tests

.github/workflows/test-mingw.yml

@@ -65,6 +65,7 @@ jobs:
               mingw-w64-x86_64-python-numpy \
               mingw-w64-x86_64-python-olefile \
               mingw-w64-x86_64-python-pip \
+              mingw-w64-x86_64-python-psutil \
               mingw-w64-x86_64-python-pytest \
               mingw-w64-x86_64-python-pytest-cov \
               mingw-w64-x86_64-python-pytest-timeout \
@@ -73,7 +74,7 @@ jobs:
           pushd depends && ./install_extra_test_images.sh && popd
 
       - name: Build Pillow
-        run: CFLAGS="-coverage" python3 -m pip install .
+        run: CFLAGS="-coverage" python3 -m pip install .[tests]
 
       - name: Test Pillow
         run: |

.github/workflows/test.yml

@@ -46,7 +46,7 @@ jobs:
           "3.11",
         ]
         include:
-        - { python-version: "3.13", PYTHONOPTIMIZE: 1, REVERSE: "--reverse" }
+        - { python-version: "3.13", PYTHONOPTIMIZE: 1 }
         - { python-version: "3.12", PYTHONOPTIMIZE: 2 }
         # Intel
         - { os: "macos-26-intel", python-version: "3.11" }
@@ -125,9 +125,6 @@ jobs:
 
     - name: Test
       run: |
-        if [ $REVERSE ]; then
-          python3 -m pip install pytest-reverse
-        fi
         if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then
           xvfb-run -s '-screen 0 1024x768x24' sway&
           export WAYLAND_DISPLAY=wayland-1
@@ -137,7 +134,6 @@ jobs:
         fi
       env:
         PYTHONOPTIMIZE: ${{ matrix.PYTHONOPTIMIZE }}
-        REVERSE: ${{ matrix.REVERSE }}
 
     - name: Prepare to upload errors
       if: failure()

Makefile

@@ -62,7 +62,7 @@ install:
 
 .PHONY: install-coverage
 install-coverage:
-	CFLAGS="-coverage -Werror=implicit-function-declaration" python3 -m pip -v install .
+	CFLAGS="-coverage -Werror=implicit-function-declaration" python3 -m pip -v install .[tests]
 	python3 selftest.py
 
 .PHONY: debug
@@ -98,8 +98,7 @@ test:
 .PHONY: test-p
 test-p:
 	python3 -c "import xdist" > /dev/null 2>&1 || python3 -m pip install pytest-xdist
-	python3 -m pytest -qq -n auto
-
+	python3 -m pytest -qq --numprocesses=logical --dist=worksteal
 
 .PHONY: valgrind
 valgrind:

Tests/helper.py

@@ -22,8 +22,15 @@
 if TYPE_CHECKING:
     from collections.abc import Callable, Sequence
     from pathlib import Path
+    from types import ModuleType
     from typing import Any
 
+psutil: ModuleType | None
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
 logger = logging.getLogger(__name__)
 
 uploader = None
@@ -211,33 +218,33 @@ def is_pypy() -> bool:
     return sys.implementation.name == "pypy"
 
 
-@pytest.mark.skipif(sys.platform.startswith("win32"), reason="Requires Unix or macOS")
+@pytest.mark.skipif(psutil is None, reason="psutil not installed")
+@pytest.mark.skipif(
+    sys.platform.startswith("win32"),
+    reason="Leak limits are not calibrated for Windows",
+)
 # Per https://stackoverflow.com/a/29007723/51685, due to JIT compilation,
 # RSS utilization is known to grow in PyPy.
 @pytest.mark.skipif(is_pypy(), reason="max RSS utilization is not stable on PyPy")
 class PillowLeakTestCase:
-    # requires unix/macOS
     iterations = 100  # count
     mem_limit = 512  # k
 
     def _get_mem_usage(self) -> float:
         """
-        Gets the RUSAGE memory usage, returns in K. Encapsulates the difference
-        between macOS and Linux rss reporting
+        Gets the resident set size currently used by this process.
 
         :returns: memory usage in kilobytes
         """
 
-        from resource import RUSAGE_SELF, getrusage
-
-        mem = getrusage(RUSAGE_SELF).ru_maxrss
-        # man 2 getrusage:
-        #     ru_maxrss
-        # This is the maximum resident set size utilized
-        # in bytes on macOS, in kilobytes on Linux
-        return mem / 1024 if sys.platform == "darwin" else mem
+        assert psutil is not None
+        return psutil.Process().memory_info().rss / 1024
 
     def _test_leak(self, core: Callable[[], None]) -> None:
+        # Warm up so allocator arenas, caches, etc. are allocated,
+        # before taking the baseline measurement.
+        core()
+
         start_mem = self._get_mem_usage()
         for cycle in range(self.iterations):
             core()

Tests/test_font_leaks.py

@@ -23,9 +23,8 @@ def _test_font(self, font: ImageFont.FreeTypeFont | ImageFont.ImageFont) -> None
 
 
 class TestTTypeFontLeak(TestFontLeak):
-    # fails at iteration 3 in main
-    iterations = 10
-    mem_limit = 4096  # k
+    iterations = 30
+    mem_limit = 32768  # k
 
     @skip_unless_feature("freetype2")
     def test_leak(self) -> None:
@@ -34,9 +33,8 @@ def test_leak(self) -> None:
 
 
 class TestDefaultFontLeak(TestFontLeak):
-    # fails at iteration 37 in main
-    iterations = 100
-    mem_limit = 1024  # k
+    iterations = 1000
+    mem_limit = 16384  # k
 
     def test_leak(self, monkeypatch: pytest.MonkeyPatch) -> None:
         if features.check_module("freetype2"):

pyproject.toml

@@ -68,6 +68,9 @@ optional-dependencies.tests = [
   "markdown2",
   "olefile",
   "packaging",
+  # Only used by the leak tests, which only run on Linux and macOS.
+  # psutil does not support iOS at all, and ships no wheels for Android.
+  "psutil; sys_platform=='linux' or sys_platform=='darwin'",
   "pytest",
   "pytest-cov",
   "pytest-timeout",