optimize test suite (#1893)

zzstoatzz · claude · web-flow · commit 04831b75c280 · 2025-09-22T20:07:04.000-04:00
Co-authored-by: Claude &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -48,7 +48,7 @@ jobs:
         run: uv sync --frozen
 
       - name: Run tests (excluding integration and client_process)
-        run: uv run pytest --inline-snapshot=disable -v tests -m "not integration and not client_process"
+        run: uv run pytest --inline-snapshot=disable -v tests -m "not integration and not client_process" --numprocesses auto --maxprocesses 4 --dist worksteal
 
       - name: Run client process tests separately
         run: uv run pytest --inline-snapshot=disable -v tests -m "client_process" -x
@@ -74,7 +74,7 @@ jobs:
 
       - name: Run integration tests
         # use longer per-test timeout than the default 3s
-        run: uv run pytest -v tests -m "integration" --timeout=15
+        run: uv run pytest -v tests -m "integration" --timeout=15 --numprocesses auto --maxprocesses 2 --dist worksteal
         env:
           FASTMCP_GITHUB_TOKEN: ${{ secrets.FASTMCP_GITHUB_TOKEN }}
           FASTMCP_TEST_AUTH_GITHUB_CLIENT_ID: ${{ secrets.FASTMCP_TEST_AUTH_GITHUB_CLIENT_ID }}
diff --git a/src/fastmcp/utilities/tests.py b/src/fastmcp/utilities/tests.py
@@ -109,18 +109,20 @@ def run_server_in_process(
     proc.start()
 
     # Wait for server to be running
-    max_attempts = 10
+    max_attempts = 30
     attempt = 0
     while attempt < max_attempts and proc.is_alive():
         try:
             with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                 s.connect((host, port))
                 break
         except ConnectionRefusedError:
-            if attempt < 3:
-                time.sleep(0.01)
-            else:
+            if attempt < 5:
+                time.sleep(0.05)
+            elif attempt < 15:
                 time.sleep(0.1)
+            else:
+                time.sleep(0.2)
             attempt += 1
     else:
         raise RuntimeError(f"Server failed to start after {max_attempts} attempts")
diff --git a/tests/client/auth/test_oauth_client.py b/tests/client/auth/test_oauth_client.py
@@ -39,7 +39,7 @@ def run_server(host: str, port: int, **kwargs) -> None:
     fastmcp_server(f"http://{host}:{port}").run(host=host, port=port, **kwargs)
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def streamable_http_server() -> Generator[str, None, None]:
     with run_server_in_process(run_server, transport="http") as url:
         yield f"{url}/mcp"
diff --git a/tests/client/test_openapi_experimental.py b/tests/client/test_openapi_experimental.py
@@ -55,19 +55,19 @@ def run_proxy_server(host: str, port: int, shttp_url: str, **kwargs) -> None:
     app.run(host=host, port=port, **kwargs)
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def shttp_server() -> Generator[str, None, None]:
     with run_server_in_process(run_server, transport="http") as url:
         yield f"{url}/mcp"
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def sse_server() -> Generator[str, None, None]:
     with run_server_in_process(run_server, transport="sse") as url:
         yield f"{url}/sse"
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def proxy_server(shttp_server: str) -> Generator[str, None, None]:
     with run_server_in_process(
         run_proxy_server,
diff --git a/tests/client/test_openapi_legacy.py b/tests/client/test_openapi_legacy.py
@@ -52,19 +52,19 @@ def run_proxy_server(host: str, port: int, shttp_url: str, **kwargs) -> None:
     app.run(host=host, port=port, **kwargs)
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def shttp_server() -> Generator[str, None, None]:
     with run_server_in_process(run_server, transport="http") as url:
         yield f"{url}/mcp"
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def sse_server() -> Generator[str, None, None]:
     with run_server_in_process(run_server, transport="sse") as url:
         yield f"{url}/sse"
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def proxy_server(shttp_server: str) -> Generator[str, None, None]:
     with run_server_in_process(
         run_proxy_server,
diff --git a/tests/client/test_sse.py b/tests/client/test_sse.py
@@ -67,7 +67,7 @@ def run_server(host: str, port: int, **kwargs) -> None:
     fastmcp_server().run(host=host, port=port, **kwargs)
 
 
-@pytest.fixture(autouse=True, scope="module")
+@pytest.fixture(autouse=True)
 def sse_server() -> Generator[str, None, None]:
     with run_server_in_process(run_server, transport="sse") as url:
         yield f"{url}/sse"
@@ -161,6 +161,6 @@ async def test_timeout_client_timeout_does_not_override_tool_call_timeout_if_low
         """
         async with Client(
             transport=SSETransport(sse_server),
-            timeout=0.01,
+            timeout=0.1,
         ) as client:
-            await client.call_tool("sleep", {"seconds": 0.1}, timeout=2)
+            await client.call_tool("sleep", {"seconds": 0.01}, timeout=2)
diff --git a/tests/client/test_streamable_http.py b/tests/client/test_streamable_http.py
@@ -225,9 +225,9 @@ async def test_timeout(self, streamable_http_server: str):
         with pytest.raises(McpError, match="Timed out"):
             async with Client(
                 transport=StreamableHttpTransport(streamable_http_server),
-                timeout=0.1,
+                timeout=0.02,
             ) as client:
-                await client.call_tool("sleep", {"seconds": 0.2})
+                await client.call_tool("sleep", {"seconds": 0.05})
 
     async def test_timeout_tool_call(self, streamable_http_server: str):
         async with Client(
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,3 +1,4 @@
+import socket
 from collections.abc import Callable
 from typing import Any
 
@@ -22,3 +23,37 @@ def import_rich_rule():
 
 def get_fn_name(fn: Callable[..., Any]) -> str:
     return fn.__name__  # ty: ignore[unresolved-attribute]
+
+
+@pytest.fixture
+def worker_id(request):
+    """Get the xdist worker ID, or 'master' if not using xdist."""
+    return getattr(request.config, "workerinput", {}).get("workerid", "master")
+
+
+@pytest.fixture
+def free_port():
+    """Get a free port for the test to use."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        s.listen(1)
+        port = s.getsockname()[1]
+    return port
+
+
+@pytest.fixture
+def free_port_factory(worker_id):
+    """Factory to get free ports that tracks used ports per test session."""
+    used_ports = set()
+
+    def get_port():
+        while True:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.bind(("127.0.0.1", 0))
+                s.listen(1)
+                port = s.getsockname()[1]
+                if port not in used_ports:
+                    used_ports.add(port)
+                    return port
+
+    return get_port
diff --git a/tests/contrib/test_bulk_tool_caller.py b/tests/contrib/test_bulk_tool_caller.py
@@ -67,7 +67,7 @@ def no_return_tool_result_factory(arg1: str) -> CallToolRequestResult:
     )
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def live_server_with_tool() -> FastMCP:
     """Fixture to create a FastMCP server instance with the echo_tool registered."""
     server = FastMCP()
diff --git a/tests/server/auth/providers/test_descope.py b/tests/server/auth/providers/test_descope.py
@@ -134,7 +134,7 @@ def add(a: int, b: int) -> int:
     mcp.run(host=host, port=port, transport="http")
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def mcp_server_url() -> Generator[str]:
     with run_server_in_process(run_mcp_server) as url:
         yield f"{url}/mcp"
diff --git a/tests/server/auth/providers/test_workos.py b/tests/server/auth/providers/test_workos.py
@@ -172,7 +172,7 @@ def add(a: int, b: int) -> int:
     mcp.run(host=host, port=port, transport="http")
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def mcp_server_url() -> Generator[str]:
     with run_server_in_process(run_mcp_server) as url:
         yield f"{url}/mcp"
diff --git a/tests/server/auth/test_jwt_provider.py b/tests/server/auth/test_jwt_provider.py
@@ -132,7 +132,7 @@ def add(a: int, b: int) -> int:
     mcp.run(host=host, port=port, **run_kwargs or {})
 
 
-@pytest.fixture(scope="module")
+@pytest.fixture
 def mcp_server_url(rsa_key_pair: RSAKeyPair) -> Generator[str]:
     with run_server_in_process(
         run_mcp_server,
diff --git a/tests/server/auth/test_oauth_proxy.py b/tests/server/auth/test_oauth_proxy.py
@@ -44,7 +44,7 @@ class MockOAuthProvider:
     - Network calls to external services
     """
 
-    def __init__(self, port: int = 9999):
+    def __init__(self, port: int = 0):
         self.port = port
         self.base_url = f"http://localhost:{port}"
         self.app = None
@@ -229,23 +229,34 @@ async def handle_revoke(self, request):
 
     async def start(self):
         """Start the mock OAuth server."""
+        import socket
+
         from uvicorn import Config, Server
 
         self.app = self.create_app()
+
+        # If port is 0, find an available port
+        if self.port == 0:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.bind(("127.0.0.1", 0))
+                s.listen(1)
+                self.port = s.getsockname()[1]
+
+        self.base_url = f"http://localhost:{self.port}"
         config = Config(self.app, host="localhost", port=self.port, log_level="error")
         self.server = Server(config)
 
         # Start server in background
         asyncio.create_task(self.server.serve())
 
         # Wait for server to be ready
-        await asyncio.sleep(0.5)
+        await asyncio.sleep(0.05)
 
     async def stop(self):
         """Stop the mock OAuth server."""
         if self.server:
             self.server.should_exit = True
-            await asyncio.sleep(0.1)
+            await asyncio.sleep(0.01)
 
     def reset(self):
         """Reset all state for next test."""
@@ -308,7 +319,7 @@ def oauth_proxy(jwt_verifier):
 @pytest.fixture
 async def mock_oauth_provider():
     """Create and start a mock OAuth provider."""
-    provider = MockOAuthProvider(port=9999)
+    provider = MockOAuthProvider()
     await provider.start()
     yield provider
     await provider.stop()
diff --git a/tests/server/http/test_http_dependencies.py b/tests/server/http/test_http_dependencies.py
@@ -42,13 +42,13 @@ def run_server(host: str, port: int, **kwargs) -> None:
     fastmcp_server().run(host=host, port=port, **kwargs)
 
 
-@pytest.fixture(autouse=True, scope="module")
+@pytest.fixture(autouse=True)
 def shttp_server() -> Generator[str, None, None]:
     with run_server_in_process(run_server, transport="http") as url:
         yield f"{url}/mcp"
 
 
-@pytest.fixture(autouse=True, scope="module")
+@pytest.fixture(autouse=True)
 def sse_server() -> Generator[str, None, None]:
     with run_server_in_process(run_server, transport="sse") as url:
         yield f"{url}/sse"
diff --git a/tests/server/middleware/test_timing.py b/tests/server/middleware/test_timing.py
@@ -155,15 +155,15 @@ def instant_task() -> str:
 
     @mcp.tool
     def short_task() -> str:
-        """A task that takes 0.1 seconds."""
-        time.sleep(0.1)
-        return "Done after 0.1s"
+        """A task that takes 0.01 seconds."""
+        time.sleep(0.01)
+        return "Done after 0.01 seconds"
 
     @mcp.tool
     def medium_task() -> str:
-        """A task that takes 0.15 seconds."""
-        time.sleep(0.15)
-        return "Done after 0.15s"
+        """A task that takes 0.02 seconds."""
+        time.sleep(0.02)
+        return "Done after 0.02 seconds"
 
     @mcp.tool
     def failing_task() -> str:
@@ -173,14 +173,14 @@ def failing_task() -> str:
     @mcp.resource("timer://test")
     def test_resource() -> str:
         """A resource that takes time to read."""
-        time.sleep(0.05)
-        return "Resource content after 0.05s"
+        time.sleep(0.005)
+        return "Resource content after 0.005 seconds"
 
     @mcp.prompt
     def test_prompt() -> str:
         """A prompt that takes time to generate."""
-        time.sleep(0.08)
-        return "Prompt content after 0.08s"
+        time.sleep(0.008)
+        return "Prompt content after 0.008 seconds"
 
     return mcp
 
diff --git a/tests/test_mcp_config.py b/tests/test_mcp_config.py
@@ -345,12 +345,12 @@ def pid() -> int:
     with pytest.raises(psutil.NoSuchProcess):
         while True:
             psutil.Process(pid_1)
-            await asyncio.sleep(0.1)
+            await asyncio.sleep(0.01)
 
     with pytest.raises(psutil.NoSuchProcess):
         while True:
             psutil.Process(pid_2)
-            await asyncio.sleep(0.1)
+            await asyncio.sleep(0.01)
 
 
 @pytest.mark.skipif(

Original file line number	Diff line number	Diff line change
`@@ -67,7 +67,7 @@ def no_return_tool_result_factory(arg1: str) -> CallToolRequestResult:`
`67`	`67`	`)`
`68`	`68`
`69`	`69`
`70`		`-@pytest.fixture(scope="module")`
	`70`	`+@pytest.fixture`
`71`	`71`	`def live_server_with_tool() -> FastMCP:`
`72`	`72`	`"""Fixture to create a FastMCP server instance with the echo_tool registered."""`
`73`	`73`	`server = FastMCP()`