Merge pull request #1 from timrid/speedup-tests

Call mypy manually and speedup tests

Author: timrid

.github/workflows/python-package.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   build:
 
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
@@ -41,3 +41,5 @@ jobs:
     - name: Test with pytest
       run: |
         uv run pytest -v
+      env:
+        FORCE_COLOR: "1"

.vscode/settings.json

@@ -4,5 +4,7 @@
         "tests"
     ],
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
+    "python.testing.pytestEnabled": true,
+    "python-envs.defaultEnvManager": "ms-python.python:venv",
+    "python-envs.pythonProjects": []
 }

pyproject.toml

@@ -35,7 +35,7 @@ dynamic = ["version"]
 [dependency-groups]
 dev = [
     "pytest",
-    "mypy==1.15.0",
+    "mypy==1.19.1",
 ]
 
 [tool.setuptools_scm]

tests/conftest.py

@@ -1,25 +1,69 @@
 # Note that this conftest file exists at the package level as it is needed to configure
 # the JVM for docstrings at the package level.
-import os
-import pathlib
+import logging
+import tempfile
+import textwrap
+from collections.abc import Generator
+from pathlib import Path
 
 import pytest
 
+import chaquopy_stubgen
 
-pytest_plugins = [
-    "mypy.test.data",
-]
-os.environ["MYPY_TEST_PREFIX"] = str(pathlib.Path(__file__).parent / "stubtest")
+logger = logging.getLogger(__name__)
 
-ANDROID_JAR = pathlib.Path(__file__).parent / "android-35.jar"
 
+ANDROID_JAR = Path(__file__).parent / "android-35.jar"
 
-@pytest.fixture(autouse=True, scope="session")
+
+@pytest.fixture(scope="session")
 def jvm():
     import jpype  # type: ignore
 
-    if not jpype.isJVMStarted():
-        jpype.startJVM(None, classpath=[ANDROID_JAR], convertStrings=True)  # type: ignore
+    jpype.startJVM(None, classpath=[ANDROID_JAR], convertStrings=True)  # type: ignore
     import jpype.imports  # type: ignore
 
-    yield jpype
+    try:
+        yield jpype
+    finally:
+        jpype.shutdownJVM()
+
+
+@pytest.fixture(scope="session")
+def stub_dir(jvm) -> Generator[Path, None, None]:
+    # logging.basicConfig(level="DEBUG")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir = Path(tmpdir)
+        import java  # type: ignore
+
+        logger.debug(f"Generating stubs in {tmpdir}...")
+
+        chaquopy_stubgen.generate_java_stubs(
+            [java],  # type: ignore
+            output_dir=tmpdir,
+        )
+
+        # Rename "java-stubs" to "java"
+        (tmpdir / "java-stubs").rename(tmpdir / "java")
+
+        yield tmpdir
+
+
+@pytest.fixture(scope="session")
+def mypy_project_dir(stub_dir: Path) -> Generator[Path, None, None]:
+    with tempfile.TemporaryDirectory() as tmp:
+        project_dir = Path(tmp)
+        with project_dir.joinpath("pyproject.toml").open("w") as f:
+            f.write(
+                textwrap.dedent(
+                    f"""\
+                    [tool.mypy]
+                    mypy_path = "{stub_dir.absolute()}"
+
+                    [[tool.mypy.overrides]]
+                    module = "java.*"
+                    ignore_errors = true
+                    """
+                )
+            )
+        yield project_dir

tests/mypy_helper.py

@@ -0,0 +1,159 @@
+import io
+import json
+import logging
+import os
+import re
+import tokenize
+from collections.abc import Generator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Literal, Optional, TypedDict
+
+import mypy.api
+
+logger = logging.getLogger(__name__)
+
+
+class MypyMessage(TypedDict):
+    file: str
+    line: int
+    column: int
+    message: str
+    hint: Optional[str]
+    code: Optional[str]
+    severity: Literal["error", "note", "warning"]
+
+
+@contextmanager
+def change_dir(path: Path) -> Generator[None, None, None]:
+    """Temporary change the current working directory."""
+    old = Path.cwd()
+    try:
+        os.chdir(path)
+        yield
+    finally:
+        os.chdir(old)
+
+
+def _parse_mypy_jsonl_output(mypy_stdout: str) -> list[MypyMessage]:
+    """Parse mypy's JSONL output into a list of MypyMessage objects."""
+    messages = []
+    for line in mypy_stdout.strip().split("\n"):
+        if line:  # Skip empty lines
+            messages.append(json.loads(line))
+    return messages
+
+
+def run_mypy(
+    project_dir: Path,
+    test_code: str,
+) -> str:
+    """Run mypy on the given test code and return the mypy stdout."""
+    testfile_name = "testfile.py"
+    with project_dir.joinpath(testfile_name).open("w") as f:
+        f.write(test_code)
+
+    with change_dir(project_dir):
+        result = mypy.api.run([testfile_name, "--output", "json"])
+
+    mypy_stdout, mypy_stderr, mypy_returncode = result
+    logger.debug(f"mypy stdout: {mypy_stdout}")
+    logger.debug(f"mypy stderr: {mypy_stderr}")
+    logger.debug(f"mypy returncode: {mypy_returncode}")
+
+    return mypy_stdout
+
+
+def run_and_assert_mypy(
+    project_dir: Path,
+    test_code: str,
+    expected_output: dict[str, str] | str,
+):
+    """
+    Run mypy on the given test code and assert that the output matches the expected output.
+
+    The expected_output can be either:
+    - A dictionary mapping markers like "*1" to expected mypy messages. In this case,
+      the function will parse the mypy JSON output and compare the messages for each marked line.
+    - A raw string containing the expected mypy output. In this case, the function will compare
+      the raw mypy stdout with the expected string.
+    """
+    mypy_stdout = run_mypy(project_dir, test_code)
+    if isinstance(expected_output, dict):
+        mypy_output = _parse_mypy_jsonl_output(mypy_stdout)
+        assert_mypy_json_output(test_code, mypy_output, expected_output)
+    else:
+        # some errors like syntax errors are not returned as json message. In
+        # this case, we just check the raw stdout for the expected error message.
+        assert expected_output.strip() == mypy_stdout.strip()
+
+
+def _format_mypy_msg(msg: MypyMessage) -> str:
+    """Format a mypy message into a human-readable string, including hints if present."""
+    formatted_msg = f"{msg['severity']}: {msg['message']}"
+    if hints := msg.get("hint"):
+        for hint in hints.splitlines():
+            formatted_msg += f"\nhint: {hint}"
+    return formatted_msg
+
+
+def _parse_marked_lines_with_tokenize(code: str) -> dict[str, int]:
+    """Parse the code to find all lines with markers like *1, *2, etc. in comments."""
+    marked_lines = {}
+
+    # Tokenize the code
+    tokens = tokenize.generate_tokens(io.StringIO(code).readline)
+
+    for token in tokens:
+        if token.type == tokenize.COMMENT:
+            # token.string contains the comment including the #
+            match = re.search(r"#\s*\*(\d+)", token.string)
+            if match:
+                marker = f"*{match.group(1)}"
+                marked_lines[marker] = token.start[0]  # Line number
+
+    return marked_lines
+
+
+def assert_mypy_json_output(
+    code: str,
+    mypy_output: list[MypyMessage],
+    expected_output: dict[str, str],
+):
+    """Assert that the mypy output matches the expected output for the given code."""
+
+    # Parse the code to find the marked lines
+    marked_lines: dict[str, int] = _parse_marked_lines_with_tokenize(code)
+
+    # Validate that all expected markers are present in the code
+    missing_markers = set(expected_output.keys()) - set(marked_lines.keys())
+    if missing_markers:
+        assert False, f"Expected markers not found in code: {', '.join(sorted(missing_markers))}"
+
+    # Group messages by line number
+    result_by_line: dict[int, list[MypyMessage]] = {}
+    for msg in mypy_output:
+        line = msg["line"]
+        if line not in result_by_line:
+            result_by_line[line] = []
+        result_by_line[line].append(msg)
+
+    # Compare the expected output with the actual mypy messages for each marked line
+    for marker, line in marked_lines.items():
+        expected_msgs = expected_output[marker]
+        mypy_msgs: list[MypyMessage] | None = result_by_line.pop(line)
+        assert mypy_msgs is not None and len(mypy_msgs) > 0
+
+        if isinstance(expected_msgs, str):
+            expected_msgs = [expected_msgs]
+
+        for expected_msg, msg in zip(expected_msgs, mypy_msgs, strict=True):
+            actual_msg = _format_mypy_msg(msg)
+            assert expected_msg.strip() == actual_msg.strip()
+
+    if len(result_by_line) > 0:
+        unexpected_msgs = []
+        for line, msgs in result_by_line.items():
+            for msg in msgs:
+                unexpected_msgs.append(f"line {line}: {_format_mypy_msg(msg)}")
+        assert False, "Unexpected mypy messages:\n" + "\n".join(unexpected_msgs)