Migrate to ruff, improve typing

.pre-commit-config.yaml

@@ -1,21 +1,7 @@
-default_language_version:
-    python: python3
 repos:
-  - hooks:
-      - id: black
-    repo: https://github.com/ambv/black
-    rev: 24.8.0
-  - hooks:
-      - id: isort
-    repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
-  - hooks:
-      - id: flake8
-        additional_dependencies:
-          - flake8-bugbear
-          - flake8-comprehensions
-          - flake8-debugger
-          - flake8-docstrings
-          - flake8-string-format
-    repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.9.6
+  hooks:
+    - id: ruff
+      args: [ --fix ]
+    - id: ruff-format

pyproject.toml

@@ -43,9 +43,16 @@ version = {attr = "scrapy_spider_metadata.__version__"}
 [tool.setuptools.package-data]
 scrapy_spider_metadata = ["py.typed"]
 
-[tool.isort]
-profile = "black"
-multi_line_output = 3
+[tool.bumpversion]
+current_version = "0.2.0"
+commit = true
+tag = true
+tag_name = "{new_version}"
+
+[[tool.bumpversion.files]]
+filename = "scrapy_spider_metadata/__init__.py"
+search = '__version__ = "{current_version}"'
+replace = '__version__ = "{new_version}"'
 
 [tool.mypy]
 check_untyped_defs = true
@@ -55,10 +62,124 @@ module = [
     "tests.*",
 ]
 # Allow test functions to be untyped
-disallow_untyped_defs = false
+allow_untyped_defs = true
 
-[[tool.mypy.overrides]]
-module = [
-    "scrapy.*",
+[tool.ruff.lint]
+extend-select = [
+    # flake8-bugbear
+    "B",
+    # flake8-comprehensions
+    "C4",
+    # pydocstyle
+    "D",
+    # flake8-future-annotations
+    "FA",
+    # flynt
+    "FLY",
+    # refurb
+    "FURB",
+    # isort
+    "I",
+    # flake8-implicit-str-concat
+    "ISC",
+    # flake8-logging
+    "LOG",
+    # Perflint
+    "PERF",
+    # pygrep-hooks
+    "PGH",
+    # flake8-pie
+    "PIE",
+    # pylint
+    "PL",
+    # flake8-pytest-style
+    "PT",
+    # flake8-use-pathlib
+    "PTH",
+    # flake8-pyi
+    "PYI",
+    # flake8-quotes
+    "Q",
+    # flake8-return
+    "RET",
+    # flake8-raise
+    "RSE",
+    # Ruff-specific rules
+    "RUF",
+    # flake8-bandit
+    "S",
+    # flake8-simplify
+    "SIM",
+    # flake8-slots
+    "SLOT",
+    # flake8-debugger
+    "T10",
+    # flake8-type-checking
+    "TC",
+    # pyupgrade
+    "UP",
+    # pycodestyle warnings
+    "W",
+    # flake8-2020
+    "YTT",
+]
+ignore = [
+    # Missing docstring in public module
+    "D100",
+    # Missing docstring in public class
+    "D101",
+    # Missing docstring in public method
+    "D102",
+    # Missing docstring in public function
+    "D103",
+    # Missing docstring in public package
+    "D104",
+    # Missing docstring in magic method
+    "D105",
+    # Missing docstring in __init__
+    "D107",
+    # One-line docstring should fit on one line with quotes
+    "D200",
+    # No blank lines allowed after function docstring
+    "D202",
+    # 1 blank line required between summary line and description
+    "D205",
+    # Multi-line docstring closing quotes should be on a separate line
+    "D209",
+    # First line should end with a period
+    "D400",
+    # First line should be in imperative mood; try rephrasing
+    "D401",
+    # First line should not be the function's "signature"
+    "D402",
+    # Too many return statements
+    "PLR0911",
+    # Too many branches
+    "PLR0912",
+    # Too many arguments in function definition
+    "PLR0913",
+    # Too many statements
+    "PLR0915",
+    # Magic value used in comparison
+    "PLR2004",
+    # String contains ambiguous {}.
+    "RUF001",
+    # Docstring contains ambiguous {}.
+    "RUF002",
+    # Comment contains ambiguous {}.
+    "RUF003",
+    # Mutable class attributes should be annotated with `typing.ClassVar`
+    "RUF012",
+    # Use of `assert` detected
+    "S101",
 ]
-ignore_missing_imports = true
+
+[tool.ruff.lint.flake8-type-checking]
+runtime-evaluated-base-classes = ["pydantic.BaseModel"]
+
+[tool.ruff.lint.per-file-ignores]
+# `from __future__ import annotations` breaks Pydantic 1.x
+"tests/test_params.py" = ["FA100"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"

scrapy_spider_metadata/__init__.py

@@ -2,3 +2,8 @@
 
 from ._metadata import get_spider_metadata
 from ._params import Args
+
+__all__ = [
+    "Args",
+    "get_spider_metadata",
+]

scrapy_spider_metadata/_metadata.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, Type
+from typing import Any
 
 from scrapy import Spider
 
@@ -8,8 +8,8 @@
 
 
 def get_spider_metadata(
-    spider_cls: Type[Spider], *, normalize: bool = False
-) -> Dict[str, Any]:
+    spider_cls: type[Spider], *, normalize: bool = False
+) -> dict[str, Any]:
     """Return the metadata for the spider class.
 
     Return a copy of the ``metadata`` dict. If the spider class defines

scrapy_spider_metadata/_params.py

@@ -1,5 +1,5 @@
 from logging import getLogger
-from typing import Any, Dict, Generic, TypeVar
+from typing import Any, Generic, TypeVar
 
 from pydantic import BaseModel, ValidationError
 
@@ -15,7 +15,7 @@ class Args(Generic[ParamSpecT]):
     specification <define-params>`.
     """
 
-    def __init__(self, *args, **kwargs) -> None:
+    def __init__(self, *args: Any, **kwargs: Any):
         param_model = get_generic_param(self.__class__, Args)
         #: :ref:`Spider arguments <spiderargs>` parsed according to the
         #: :ref:`spider parameter specification <define-params>`.
@@ -31,7 +31,7 @@ def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 
     @classmethod
-    def get_param_schema(cls, normalize: bool = False) -> Dict[Any, Any]:
+    def get_param_schema(cls, normalize: bool = False) -> dict[Any, Any]:
         """Return a :class:`dict` with the :ref:`parameter definition
         <define-params>` as `JSON Schema`_.
 

scrapy_spider_metadata/_utils.py

@@ -1,11 +1,11 @@
+from __future__ import annotations
+
 import copy
 from collections import deque
-from typing import Any, Dict, Optional, Tuple, TypeVar, Union, get_args
+from typing import Any, TypeVar, cast, get_args
 
 
-def get_generic_param(
-    cls: type, expected: Union[type, Tuple[type, ...]]
-) -> Optional[type]:
+def get_generic_param(cls: type, expected: type | tuple[type, ...]) -> type | None:
     """Search the base classes recursively breadth-first for a generic class and return its param.
 
     Returns the param of the first found class that is a subclass of ``expected``.
@@ -20,15 +20,15 @@ def get_generic_param(
             if origin and issubclass(origin, expected):
                 result = get_args(base)[0]
                 if not isinstance(result, TypeVar):
-                    return result
+                    return cast(type, result)
             queue.append(base)
     return None
 
 
-def _normalize_param(key, value, defs, /):
-    def get_def(ref: str) -> Dict[str, Any]:
+def _normalize_param(key: str, value: dict[str, Any], defs: dict[str, Any], /) -> None:
+    def get_def(ref: str) -> dict[str, Any]:
         def_id = ref.rsplit("/", maxsplit=1)[1]
-        return defs[def_id]
+        return cast(dict[str, Any], defs[def_id])
 
     extra = value.pop("json_schema_extra", None)
     if extra:
@@ -69,8 +69,8 @@ def get_def(ref: str) -> Dict[str, Any]:
         value["title"] = key.title().replace("_", " ")
 
 
-def normalize_param_schema(schema, /):
-    params = schema.get("properties", None)
+def normalize_param_schema(schema: dict[str, Any], /) -> None:
+    params = schema.get("properties")
     if not params:
         return
     defs = schema.pop("$defs", None)

tests/__init__.py

@@ -1,6 +1,17 @@
+from __future__ import annotations
+
+from typing import Any, TypeVar, cast
+
+from scrapy import Spider
 from scrapy.utils.test import get_crawler
 
+_SpiderT = TypeVar("_SpiderT", bound=Spider)
+
 
-def get_spider(spidercls, settings=None, kwargs=None):
+def get_spider(
+    spidercls: type[_SpiderT],
+    settings: dict[str, Any] | None = None,
+    kwargs: dict[str, Any] | None = None,
+) -> _SpiderT:
     crawler = get_crawler(spidercls, settings or {})
-    return crawler._create_spider(spidercls.name, **(kwargs or {}))
+    return cast(_SpiderT, crawler._create_spider(spidercls.name, **(kwargs or {})))