Split ruff backend into separate backends for formatting and linting (#20545)

Closes #20525.

This PR splits Ruff formatting out into a separate backend from Ruff
linting/fixing, and registers the new backend as an experimental one.
For backwards compatibility, we retain the existing lint/fix backend at
`pants.backend.experimental.python.lint.ruff` and move the formatting
backend out to `pants.backend.experimental.python.lint.ruff_fmt`.

I tried to keep the changes as minimal as possible and not touch too
many files / move too many things around.

This will be a breaking change for anyone who has adopted the Ruff
formatter in 2.20 pre-release, but that should be okay as the feature
hasn't been released yet.

The naming of the new backend is a bit confusing, but since all other
formatter backends are published under the `lint` package I tried to
keep the convention consistent.

---------

Co-authored-by: Andreas Stenius <[email protected]>
Co-authored-by: Jacob Floyd <[email protected]>

Author: 3 people

docs/docs/python/overview/linters-and-formatters.mdx

@@ -26,19 +26,20 @@ Pants does several things to speed up running formatters and linters:
 
 Linter/formatter support is implemented in separate [backends](../../using-pants/key-concepts/backends.mdx) so that they are easy to opt in to individually:
 
-| Backend                                       | Tool                                                                                                                       |
-| :-------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- |
-| `pants.backend.python.lint.bandit`            | [Bandit](https://bandit.readthedocs.io/en/latest/): security linter                                                        |
-| `pants.backend.python.lint.black`             | [Black](https://black.readthedocs.io/en/stable/): code formatter                                                           |
-| `pants.backend.python.lint.docformatter`      | [Docformatter](https://pypi.org/project/docformatter/): docstring formatter                                                |
-| `pants.backend.python.lint.flake8`            | [Flake8](https://flake8.pycqa.org/en/latest/): style and bug linter                                                        |
-| `pants.backend.python.lint.isort`             | [isort](https://readthedocs.org/projects/isort/): import statement formatter                                               |
-| `pants.backend.python.lint.pydocstyle`        | [Pydocstyle](https://pypi.org/project/pydocstyle/): docstring linter                                                       |
-| `pants.backend.python.lint.pylint`            | [Pylint](https://pylint.pycqa.org/): style and bug linter                                                                  |
-| `pants.backend.python.lint.yapf`              | [Yapf](https://github.com/google/yapf): code formatter                                                                     |
-| `pants.backend.python.lint.autoflake`         | [Autoflake](https://github.com/myint/autoflake): remove unused imports                                                     |
-| `pants.backend.python.lint.pyupgrade`         | [Pyupgrade](https://github.com/asottile/pyupgrade): automatically update code to use modern Python idioms like `f-strings` |
-| `pants.backend.experimental.python.lint.ruff` | [Ruff](https://beta.ruff.rs/docs/): an extremely fast Python linter, written in Rust.                                      |
+| Backend                                              | Tool                                                                                                                       |
+| :--------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- |
+| `pants.backend.python.lint.bandit`                   | [Bandit](https://bandit.readthedocs.io/en/latest/): security linter                                                        |
+| `pants.backend.python.lint.black`                    | [Black](https://black.readthedocs.io/en/stable/): code formatter                                                           |
+| `pants.backend.python.lint.docformatter`             | [Docformatter](https://pypi.org/project/docformatter/): docstring formatter                                                |
+| `pants.backend.python.lint.flake8`                   | [Flake8](https://flake8.pycqa.org/en/latest/): style and bug linter                                                        |
+| `pants.backend.python.lint.isort`                    | [isort](https://readthedocs.org/projects/isort/): import statement formatter                                               |
+| `pants.backend.python.lint.pydocstyle`               | [Pydocstyle](https://pypi.org/project/pydocstyle/): docstring linter                                                       |
+| `pants.backend.python.lint.pylint`                   | [Pylint](https://pylint.pycqa.org/): style and bug linter                                                                  |
+| `pants.backend.python.lint.yapf`                     | [Yapf](https://github.com/google/yapf): code formatter                                                                     |
+| `pants.backend.python.lint.autoflake`                | [Autoflake](https://github.com/myint/autoflake): remove unused imports                                                     |
+| `pants.backend.python.lint.pyupgrade`                | [Pyupgrade](https://github.com/asottile/pyupgrade): automatically update code to use modern Python idioms like `f-strings` |
+| `pants.backend.experimental.python.lint.ruff.check`  | [Ruff (for linting)](https://docs.astral.sh/ruff/linter/): an extremely fast Python linter, written in Rust.               |
+| `pants.backend.experimental.python.lint.ruff.format` | [Ruff (for formatting)](https://docs.astral.sh/ruff/formatter/): an extremely fast Python code formatter, written in Rust. |
 
 To enable, add the appropriate backends in `pants.toml`:
 

docs/docs/using-pants/key-concepts/backends.mdx

@@ -98,7 +98,8 @@ The list of all backends (both stable and experimental) is also available via `p
 | `pants.backend.experimental.python.framework.django`               | Enables better support for projects using Django: [https://www.djangoproject.com](https://www.djangoproject.com)                                      |                                                                                                       |
 | `pants.backend.experimental.python.framework.stevedore`            | Enables better support for projects using stevedore: [https://docs.openstack.org/stevedore/](https://docs.openstack.org/stevedore/)                   |                                                                                                       |
 | `pants.backend.experimental.python.lint.add_trailing_comma`        | Enables add-trailing-comma, a Python code formatter: [https://github.com/asottile/add-trailing-comma](https://github.com/asottile/add-trailing-comma) | [`add-trailing-comma`](../../../reference/subsystems/add-trailing-comma.mdx)                          |
-| `pants.backend.experimental.python.lint.ruff`                      | Enables Ruff, an extremely fast Python linter: [https://beta.ruff.rs/docs/](https://beta.ruff.rs/docs/)                                               | [Linters and formatters](../../python/overview/linters-and-formatters.mdx)                            |
+| `pants.backend.experimental.python.lint.ruff.check`                | Enables Ruff (for `lint`), an extremely fast Python linter: [https://docs.astral.sh/ruff/linter/](https://docs.astral.sh/ruff/linter/)                | [Linters and formatters](../../python/overview/linters-and-formatters.mdx)                            |
+| `pants.backend.experimental.python.lint.ruff.format`               | Enables Ruff (for `fmt`), an extremely fast Python code formatter: [https://docs.astral.sh/ruff/formatter/](https://docs.astral.sh/ruff/formatter/)   | [Linters and formatters](../../python/overview/linters-and-formatters.mdx)                            |
 | `pants.backend.experimental.python.packaging.pyoxidizer`           | Enables `pyoxidizer_binary` target.                                                                                                                   | [PyOxidizer](../../python/integrations/pyoxidizer.mdx)                                                |
 | `pants.backend.experimental.python.typecheck.pyright`              | Enables Pyright, a Python type checker: [https://github.com/microsoft/pyright](https://github.com/microsoft/pyright)                                  |                                                                                                       |
 | `pants.backend.experimental.python.typecheck.pytype`               | Enables Pytype, a Python type checker: [https://google.github.io/pytype/](https://google.github.io/pytype/)                                           |                                                                                                       |

src/python/pants/backend/BUILD

@@ -36,7 +36,8 @@ __dependents_rules__(
             "[/build_files/fmt/yapf/register.py]",
             "[/python/lint/black/rules.py]",
             "[/python/lint/black/subsystem.py]",
-            "[/python/lint/ruff/rules.py]",
+            "[/python/lint/ruff/fmt_rules.py]",
+            "[/python/lint/ruff/check_rules.py]",
             "[/python/lint/ruff/subsystem.py]",
             "[/python/lint/yapf/rules.py]",
             "[/python/lint/yapf/subsystem.py]",

src/python/pants/backend/build_files/fmt/ruff/integration_test.py

@@ -7,7 +7,7 @@
 
 from pants.backend.build_files.fmt.ruff.register import RuffRequest
 from pants.backend.build_files.fmt.ruff.register import rules as ruff_build_rules
-from pants.backend.python.lint.ruff.rules import rules as ruff_fmt_rules
+from pants.backend.python.lint.ruff.check_rules import rules as ruff_fmt_rules
 from pants.backend.python.lint.ruff.subsystem import Ruff
 from pants.backend.python.lint.ruff.subsystem import rules as ruff_subsystem_rules
 from pants.backend.python.target_types import PythonSourcesGeneratorTarget

src/python/pants/backend/build_files/fmt/ruff/register.py

@@ -3,7 +3,7 @@
 
 from pants.backend.build_files.fmt.base import FmtBuildFilesRequest
 from pants.backend.python.lint.ruff import subsystem as ruff_subsystem
-from pants.backend.python.lint.ruff.rules import _run_ruff_fmt
+from pants.backend.python.lint.ruff.fmt_rules import _run_ruff_fmt
 from pants.backend.python.lint.ruff.subsystem import Ruff
 from pants.backend.python.subsystems.python_tool_base import get_lockfile_interpreter_constraints
 from pants.core.goals.fmt import FmtResult

src/python/pants/backend/experimental/python/lint/ruff/BUILD

@@ -1,4 +1,4 @@
-# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 python_sources()

src/python/pants/backend/experimental/python/lint/ruff/check/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/experimental/python/lint/ruff/check/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+"""Linter & formatter for Python.
+
+See https://www.pantsbuild.org/docs/python-linters-and-formatters and https://docs.astral.sh/ruff/
+"""
+
+from pants.backend.python.lint.ruff import check_rules as ruff_rules
+from pants.backend.python.lint.ruff import skip_field, subsystem
+
+
+def rules():
+    return (*ruff_rules.rules(), *skip_field.rules(), *subsystem.rules())

src/python/pants/backend/experimental/python/lint/ruff/check/register.py

@@ -0,0 +1,4 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/experimental/python/lint/ruff/format/BUILD

@@ -0,0 +1,14 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+"""Linter & formatter for Python.
+
+See https://www.pantsbuild.org/docs/python-linters-and-formatters and https://docs.astral.sh/ruff/
+"""
+
+from pants.backend.python.lint.ruff import fmt_rules as ruff_fmt_rules
+from pants.backend.python.lint.ruff import skip_field, subsystem
+
+
+def rules():
+    return (*ruff_fmt_rules.rules(), *skip_field.rules(), *subsystem.rules())

src/python/pants/backend/experimental/python/lint/ruff/format/__init__.py

@@ -6,9 +6,15 @@
 See https://www.pantsbuild.org/docs/python-linters-and-formatters and https://docs.astral.sh/ruff/
 """
 
-from pants.backend.python.lint.ruff import rules as ruff_rules
-from pants.backend.python.lint.ruff import skip_field, subsystem
+from pants.backend.experimental.python.lint.ruff.check import register as ruff_check
+from pants.base.deprecated import warn_or_error
 
 
 def rules():
-    return (*ruff_rules.rules(), *skip_field.rules(), *subsystem.rules())
+    warn_or_error(
+        "2.23.0.dev0",
+        "The `pants.backend.experimental.python.lint.ruff` backend",
+        hint="Use `pants.backend.experimental.python.lint.ruff.check` instead.",
+        start_version="2.20.0.dev7",
+    )
+    return ruff_check.rules()

src/python/pants/backend/experimental/python/lint/ruff/format/register.py

@@ -13,7 +13,6 @@
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
 from pants.core.goals.fix import FixResult, FixTargetsRequest
-from pants.core.goals.fmt import AbstractFmtRequest, FmtResult, FmtTargetsRequest
 from pants.core.goals.lint import LintResult, LintTargetsRequest
 from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest
 from pants.core.util_rules.partitions import PartitionerType
@@ -59,43 +58,13 @@ def tool_id(cls) -> str:
         return RuffLintRequest.tool_id
 
 
-class RuffFormatRequest(FmtTargetsRequest):
-    field_set_type = RuffFieldSet
-    tool_subsystem = Ruff
-    partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
-
-    @classproperty
-    def tool_name(cls) -> str:
-        return "ruff format"
-
-    @classproperty
-    def tool_id(self) -> str:
-        return "ruff-format"
-
-
 @dataclass(frozen=True)
 class _RunRuffRequest:
     snapshot: Snapshot
     mode: RuffMode
     interpreter_constraints: Optional[InterpreterConstraints] = None
 
 
-# Note - this function is kept separate because it is invoked from update_build_files.py, but
-# not as a rule.
-async def _run_ruff_fmt(
-    request: AbstractFmtRequest.Batch,
-    ruff: Ruff,
-    interpreter_constraints: Optional[InterpreterConstraints] = None,
-) -> FmtResult:
-    run_ruff_request = _RunRuffRequest(
-        snapshot=request.snapshot,
-        mode=RuffMode.FORMAT,
-        interpreter_constraints=interpreter_constraints,
-    )
-    result = await _run_ruff(run_ruff_request, ruff)
-    return await FmtResult.create(request, result)
-
-
 async def _run_ruff(
     request: _RunRuffRequest,
     ruff: Ruff,
@@ -169,16 +138,10 @@ async def ruff_lint(request: RuffLintRequest.Batch[RuffFieldSet, Any], ruff: Ruf
     return LintResult.create(request, result)
 
 
-@rule(desc="Format with `ruff format`", level=LogLevel.DEBUG)
-async def ruff_fmt(request: RuffFormatRequest.Batch, ruff: Ruff) -> FmtResult:
-    return await _run_ruff_fmt(request, ruff)
-
-
 def rules():
     return [
         *collect_rules(),
         *RuffFixRequest.rules(),
         *RuffLintRequest.rules(),
-        *RuffFormatRequest.rules(),
         *pex.rules(),
     ]

src/python/pants/backend/experimental/python/lint/ruff/register.py

@@ -0,0 +1,57 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from typing import Optional
+
+from pants.backend.python.lint.ruff.check_rules import _run_ruff, _RunRuffRequest
+from pants.backend.python.lint.ruff.subsystem import Ruff, RuffFieldSet, RuffMode
+from pants.backend.python.util_rules import pex
+from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
+from pants.core.goals.fmt import AbstractFmtRequest, FmtResult, FmtTargetsRequest
+from pants.core.util_rules.partitions import PartitionerType
+from pants.engine.rules import collect_rules, rule
+from pants.util.logging import LogLevel
+from pants.util.meta import classproperty
+
+
+class RuffFormatRequest(FmtTargetsRequest):
+    field_set_type = RuffFieldSet
+    tool_subsystem = Ruff
+    partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
+
+    @classproperty
+    def tool_name(cls) -> str:
+        return "ruff format"
+
+    @classproperty
+    def tool_id(self) -> str:
+        return "ruff-format"
+
+
+# Note - this function is kept separate because it is invoked from update_build_files.py, but
+# not as a rule.
+async def _run_ruff_fmt(
+    request: AbstractFmtRequest.Batch,
+    ruff: Ruff,
+    interpreter_constraints: Optional[InterpreterConstraints] = None,
+) -> FmtResult:
+    run_ruff_request = _RunRuffRequest(
+        snapshot=request.snapshot,
+        mode=RuffMode.FORMAT,
+        interpreter_constraints=interpreter_constraints,
+    )
+    result = await _run_ruff(run_ruff_request, ruff)
+    return await FmtResult.create(request, result)
+
+
+@rule(desc="Format with `ruff format`", level=LogLevel.DEBUG)
+async def ruff_fmt(request: RuffFormatRequest.Batch, ruff: Ruff) -> FmtResult:
+    return await _run_ruff_fmt(request, ruff)
+
+
+def rules():
+    return [
+        *collect_rules(),
+        *RuffFormatRequest.rules(),
+        *pex.rules(),
+    ]

src/python/pants/backend/python/lint/ruff/rules.py src/python/pants/backend/python/lint/ruff/check_rules.py

@@ -9,8 +9,10 @@
 
 from pants.backend.python import target_types_rules
 from pants.backend.python.lint.ruff import skip_field
-from pants.backend.python.lint.ruff.rules import RuffFixRequest, RuffFormatRequest, RuffLintRequest
-from pants.backend.python.lint.ruff.rules import rules as ruff_rules
+from pants.backend.python.lint.ruff.check_rules import RuffFixRequest, RuffLintRequest
+from pants.backend.python.lint.ruff.check_rules import rules as ruff_rules
+from pants.backend.python.lint.ruff.fmt_rules import RuffFormatRequest
+from pants.backend.python.lint.ruff.fmt_rules import rules as ruff_fmt_rules
 from pants.backend.python.lint.ruff.subsystem import RuffFieldSet
 from pants.backend.python.lint.ruff.subsystem import rules as ruff_subsystem_rules
 from pants.backend.python.target_types import PythonSourcesGeneratorTarget
@@ -35,6 +37,7 @@ def rule_runner() -> RuleRunner:
     return RuleRunner(
         rules=[
             *ruff_rules(),
+            *ruff_fmt_rules(),
             *skip_field.rules(),
             *ruff_subsystem_rules(),
             *config_files.rules(),

src/python/pants/backend/python/lint/ruff/fmt_rules.py

@@ -23,7 +23,7 @@
 from pants.backend.python.goals import lockfile
 from pants.backend.python.lint.black.rules import _run_black
 from pants.backend.python.lint.black.subsystem import Black
-from pants.backend.python.lint.ruff.rules import _run_ruff_fmt
+from pants.backend.python.lint.ruff.fmt_rules import _run_ruff_fmt
 from pants.backend.python.lint.ruff.subsystem import Ruff
 from pants.backend.python.lint.yapf.rules import _run_yapf
 from pants.backend.python.lint.yapf.subsystem import Yapf