Add sql backend + sqlfluff linter (#20854)

Author: grihabor

build-support/bin/generate_builtin_lockfiles.py

@@ -52,6 +52,7 @@
 from pants.backend.python.typecheck.pytype.subsystem import Pytype
 from pants.backend.scala.lint.scalafmt.subsystem import ScalafmtSubsystem
 from pants.backend.scala.subsystems.scalatest import Scalatest
+from pants.backend.sql.lint.sqlfluff.subsystem import Sqlfluff
 from pants.backend.terraform.dependency_inference import TerraformHcl2Parser
 from pants.backend.tools.semgrep.subsystem import SemgrepSubsystem
 from pants.backend.tools.yamllint.subsystem import Yamllint
@@ -132,6 +133,7 @@ class JvmTool(Tool[JvmToolBase]):
             PythonTool(SemgrepSubsystem, "pants.backend.experimental.tools.semgrep"),
             PythonTool(Setuptools, "pants.backend.python"),
             PythonTool(SetuptoolsSCM, "pants.backend.python"),
+            PythonTool(Sqlfluff, "pants.backend.experimental.sql.lint.sqlfluff"),
             PythonTool(TerraformHcl2Parser, "pants.backend.experimental.terraform"),
             PythonTool(TwineSubsystem, "pants.backend.python"),
             PythonTool(Yamllint, "pants.backend.experimental.tools.yamllint"),

docs/docs/ad-hoc-tools/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Ad-Hoc Tools",
-  "position": 12
+  "position": 13
 }

docs/docs/contributions/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Contributions",
-  "position": 15
+  "position": 16
 }

docs/docs/releases/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Releases",
-  "position": 14
+  "position": 15
 }

docs/docs/sql/_category_.json

@@ -0,0 +1,4 @@
+{
+  "label": "SQL",
+  "position": 12
+}

docs/docs/sql/index.mdx

@@ -0,0 +1,54 @@
+---
+    title: SQL Overview
+    sidebar_position: 999
+---
+
+---
+
+:::caution SQL support is in alpha stage
+Pants is currently building support for SQL. Simple use cases might be
+supported, but many options are missing.
+
+Please share feedback for what you need to use Pants with your SQL queries by
+either [opening a GitHub
+issue](https://github.com/pantsbuild/pants/issues/new/choose) or [joining our
+Slack](/community/getting-help)!
+:::
+
+## Initial setup
+
+First, activate the relevant backend in `pants.toml`:
+
+```toml title="pants.toml"
+[GLOBAL]
+backend_packages = [
+  ...
+  "pants.backend.experimental.sql",
+  ...
+]
+```
+
+The SQL backend adds [`sql_source`](../../reference/targets/sql_source.mdx) and
+[`sql_sources`](../../reference/targets/sql_sources.mdx) target types for SQL
+files. The [`sql_source`](../../reference/targets/sql_source.mdx) behaves like
+[`resource`](../../reference/targets/resource.mdx), so you can use it directly
+without wrappers. The `tailor` goal will automatically generate the targets for
+your .sql files.
+
+## Enable sqlfluff linter
+
+To enable the linter activate the relevant backend in `pants.toml`:
+
+```toml title="pants.toml"
+[GLOBAL]
+backend_packages = [
+  ...
+  "pants.backend.experimental.sql.lint.sqlfluff",
+  ...
+]
+```
+
+You can run the linter via `lint` goal:
+```
+pants lint --only=sqlfluff ::
+```

docs/docs/writing-plugins/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Writing Plugins",
-  "position": 13
+  "position": 14
 }

docs/notes/2.22.x.md

@@ -24,6 +24,12 @@ If you encounter such discrepancies, and you can't resolve them easily, please [
 
 ### Backends
 
+#### NEW: SQL
+
+A new experimental `SQL` backend was added along with the [sqlfluff
+linter](https://www.pantsbuild.org/2.22/reference/subsystems/sqlfluff). See
+docs [here](https://www.pantsbuild.org/2.22/docs/sql).
+
 #### JVM
 
 ##### Scala

src/python/pants/backend/experimental/sql/__init__.py

@@ -0,0 +1,12 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from pants.backend.sql.lint.sqlfluff import rules as sqlfluff_rules
+from pants.backend.sql.lint.sqlfluff import skip_field, subsystem
+
+
+def rules():
+    return [
+        *subsystem.rules(),
+        *sqlfluff_rules.rules(),
+        *skip_field.rules(),
+    ]

src/python/pants/backend/experimental/sql/lint/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from pants.backend.sql import tailor
+from pants.backend.sql.target_types import SqlSourcesGeneratorTarget, SqlSourceTarget
+
+
+def target_types():
+    return [
+        SqlSourceTarget,
+        SqlSourcesGeneratorTarget,
+    ]
+
+
+def rules():
+    return [*tailor.rules()]

src/python/pants/backend/experimental/sql/lint/sqlfluff/__init__.py

@@ -0,0 +1,9 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources(
+    sources=[
+        "tailor.py",
+        "target_types.py",
+    ],
+)

src/python/pants/backend/experimental/sql/lint/sqlfluff/register.py

@@ -0,0 +1,11 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+resource(name="lockfile", source="sqlfluff.lock")
+
+python_sources(
+    overrides={"subsystem.py": {"dependencies": [":lockfile"]}},
+)
+
+python_tests(
+    name="tests",
+)

src/python/pants/backend/experimental/sql/register.py

@@ -0,0 +1,144 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Tuple
+
+from typing_extensions import assert_never
+
+from pants.backend.python.util_rules import pex
+from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess
+from pants.backend.sql.lint.sqlfluff.subsystem import Sqlfluff, SqlfluffFieldSet, SqlfluffMode
+from pants.core.goals.fix import FixResult, FixTargetsRequest
+from pants.core.goals.fmt import FmtResult, FmtTargetsRequest
+from pants.core.goals.lint import LintResult, LintTargetsRequest
+from pants.core.util_rules.config_files import ConfigFiles, ConfigFilesRequest
+from pants.core.util_rules.partitions import PartitionerType
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.fs import Digest, MergeDigests
+from pants.engine.internals.native_engine import Snapshot
+from pants.engine.process import FallibleProcessResult
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.util.logging import LogLevel
+from pants.util.meta import classproperty
+from pants.util.strutil import pluralize
+
+
+class SqlfluffFixRequest(FixTargetsRequest):
+    field_set_type = SqlfluffFieldSet
+    tool_subsystem = Sqlfluff
+    partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
+
+    # We don't need to include automatically added lint rules for this SqlfluffFixRequest,
+    # because these lint rules are already checked by SqlfluffLintRequest.
+    enable_lint_rules = False
+
+
+class SqlfluffLintRequest(LintTargetsRequest):
+    field_set_type = SqlfluffFieldSet
+    tool_subsystem = Sqlfluff
+    partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
+
+
+class SqlfluffFormatRequest(FmtTargetsRequest):
+    field_set_type = SqlfluffFieldSet
+    tool_subsystem = Sqlfluff
+    partitioner_type = PartitionerType.DEFAULT_SINGLE_PARTITION
+
+    @classproperty
+    def tool_name(cls) -> str:
+        return "sqlfluff format"
+
+    @classproperty
+    def tool_id(cls) -> str:
+        return "sqlfluff-format"
+
+
+@dataclass(frozen=True)
+class _RunSqlfluffRequest:
+    snapshot: Snapshot
+    mode: SqlfluffMode
+
+
+@rule(level=LogLevel.DEBUG)
+async def run_sqlfluff(
+    request: _RunSqlfluffRequest,
+    sqlfluff: Sqlfluff,
+) -> FallibleProcessResult:
+    sqlfluff_pex_get = Get(VenvPex, PexRequest, sqlfluff.to_pex_request())
+
+    config_files_get = Get(
+        ConfigFiles, ConfigFilesRequest, sqlfluff.config_request(request.snapshot.dirs)
+    )
+
+    sqlfluff_pex, config_files = await MultiGet(sqlfluff_pex_get, config_files_get)
+
+    input_digest = await Get(
+        Digest,
+        MergeDigests((request.snapshot.digest, config_files.snapshot.digest)),
+    )
+
+    initial_args: Tuple[str, ...] = ()
+    if request.mode is SqlfluffMode.FMT:
+        initial_args = ("format",)
+    elif request.mode is SqlfluffMode.FIX:
+        initial_args = ("fix", *sqlfluff.fix_args)
+    elif request.mode is SqlfluffMode.LINT:
+        initial_args = ("lint",)
+    else:
+        assert_never(request.mode)
+
+    conf_args = ["--config", sqlfluff.config] if sqlfluff.config else []
+
+    result = await Get(
+        FallibleProcessResult,
+        VenvPexProcess(
+            sqlfluff_pex,
+            argv=(*initial_args, *conf_args, *sqlfluff.args, *request.snapshot.files),
+            input_digest=input_digest,
+            output_files=request.snapshot.files,
+            description=f"Run sqlfluff {' '.join(initial_args)} on {pluralize(len(request.snapshot.files), 'file')}.",
+            level=LogLevel.DEBUG,
+        ),
+    )
+    return result
+
+
+@rule(desc="Fix with sqlfluff fix", level=LogLevel.DEBUG)
+async def sqlfluff_fix(request: SqlfluffFixRequest.Batch, sqlfluff: Sqlfluff) -> FixResult:
+    result = await Get(
+        FallibleProcessResult, _RunSqlfluffRequest(snapshot=request.snapshot, mode=SqlfluffMode.FIX)
+    )
+    return await FixResult.create(request, result)
+
+
+@rule(desc="Lint with sqlfluff lint", level=LogLevel.DEBUG)
+async def sqlfluff_lint(request: SqlfluffLintRequest.Batch[SqlfluffFieldSet, Any]) -> LintResult:
+    source_files = await Get(
+        SourceFiles, SourceFilesRequest(field_set.source for field_set in request.elements)
+    )
+    result = await Get(
+        FallibleProcessResult,
+        _RunSqlfluffRequest(snapshot=source_files.snapshot, mode=SqlfluffMode.LINT),
+    )
+    return LintResult.create(request, result)
+
+
+@rule(desc="Format with sqlfluff format", level=LogLevel.DEBUG)
+async def sqlfluff_fmt(request: SqlfluffFormatRequest.Batch, sqlfluff: Sqlfluff) -> FmtResult:
+    result = await Get(
+        FallibleProcessResult,
+        _RunSqlfluffRequest(snapshot=request.snapshot, mode=SqlfluffMode.FMT),
+    )
+    return await FmtResult.create(request, result)
+
+
+def rules():
+    return [
+        *collect_rules(),
+        *SqlfluffLintRequest.rules(),
+        *SqlfluffFixRequest.rules(),
+        *SqlfluffFormatRequest.rules(),
+        *pex.rules(),
+    ]