spark-connect-rust/pyproject.toml at master · apache/spark-connect-rust · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
[build-system]
requires = ["maturin>=1.0,<2.0"]
build-backend = "maturin"

[project]
name = "pyspark-client-rust"
version = "4.3.0"
description = "Rust-backed drop-in for the PySpark Spark Connect client (pyspark-client): the same `pyspark` API and results, powered by a native Rust engine (tonic) instead of grpcio/py4j."
readme = "README.md"
requires-python = ">=3.9"
license = {text = "Apache-2.0"}
authors = [{name = "Apache Spark", email = "dev@spark.apache.org"}]
keywords = ["pyspark", "spark", "spark-connect", "connect", "pyspark-client", "rust", "tonic", "grpc", "arrow", "dataframe"]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: Apache Software License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Rust",
    "Topic :: Software Development :: Libraries",
]

dependencies = [
    "pandas>=1.0.0",
    "pyarrow>=10.0.0",
    "numpy>=1.16.0",
]
# cloudpickle is vendored at pyspark/cloudpickle (matching Apache Spark's own
# bundling), so it is intentionally NOT an external dependency.
#
# No optional-dependencies: unlike upstream PySpark's `connect` extra, this
# package has no pure-Python gRPC client — the Rust extension (pyspark._pyspark)
# speaks gRPC via tonic, so grpcio is never imported at runtime.

[project.urls]
Homepage = "https://spark.apache.org/"
Documentation = "https://apache.github.io/spark-connect-rust/"
Repository = "https://github.com/apache/spark-connect-rust"
"Bug Tracker" = "https://issues.apache.org/jira/browse/SPARK"

[tool.ruff]
# Lint only our authored tooling/scripts. The python/pyspark tree is largely vendored
# from Apache Spark (pandas-on-Spark, the drop-in skin) and must stay
# byte-compatible with upstream, so it is intentionally excluded from our linter.
line-length = 100
target-version = "py39"
extend-exclude = ["python/pyspark", "target", ".claude"]
include = ["scripts/**/*.py"]

[tool.ruff.lint]
select = ["E", "F", "W", "I", "UP", "B"]
ignore = ["E501"]  # line length handled by the formatter, not enforced hard

[tool.maturin]
# Mixed Python/Rust layout: the pure-Python drop-in skin lives under python/pyspark,
# and the PyO3 extension is placed inside it as pyspark._pyspark.
python-source = "python"
module-name = "pyspark._pyspark"
manifest-path = "crates/pyspark-rs/Cargo.toml"
# The wheel auto-includes the license (as dist-info/licenses/LICENSE), but maturin
# does NOT pull the repo-root LICENSE into the sdist -- the crate lives under
# crates/pyspark-rs/, so a root-level file outside it is not packaged. The metadata
# still declares `License-File: LICENSE` in both artifacts, so PyPI (PEP 639) rejects
# the sdist with "License-File LICENSE does not exist in distribution file". Ship the
# LICENSE in the sdist so it satisfies the declared metadata.
include = [{ path = "LICENSE", format = "sdist" }]
# Use abi3 for a stable ABI compatible across Python 3.9+.
features = ["pyo3/abi3-py39", "pyo3/extension-module"]
# Keep tests out of the built wheel/sdist: our own offline suite lives outside the package
# (python/tests), but the vendored pyspark tree carries upstream test modules
# (e.g. pyspark/pipelines/tests, pyspark/testing/tests) that end users don't need.
exclude = [
    "python/pyspark/**/tests/**",
    "python/pyspark/**/test_*.py",
    "pyspark/**/tests/**",
    "pyspark/**/test_*.py",
]