From 60fe1177e4b470393b3ea9be8e4e99fe303af5f5 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 2 Sep 2026 13:41:13 +0200 Subject: [PATCH 1/5] Require Python 3.13 and build the virtualenv from the checked interpreter `bin/pyactivate` validates the interpreter running it and then hands virtualenv creation to `uv venv` without naming an interpreter. `uv` resolves one by its own preference order, which favors uv-managed installs over the system Python, so the virtualenv can end up on a different and older Python than the one the check just accepted. On a machine with system Python 3.14 and a uv-managed 3.10 present, the virtualenv is built on 3.10. Passing `sys.executable` removes that second, independent choice and makes the check authoritative. It also aligns the two creation paths, since the `venv.create` fallback right below already builds from the running interpreter. That fallback passes `clear=True`, and `uv venv` is now given `--clear` to match. Control reaches this branch only when the virtualenv is missing or its Python will not execute, which is exactly when it should be replaced. Without the flag `uv` refuses to touch an existing directory and aborts, so a virtualenv left behind without its `dep_stamp` wedges every later invocation instead of being rebuilt. The floor moves from 3.10 to 3.13. Python 3.10 has been security-only for years and reaches end of life in October 2026, and the tree had already drifted past it: two mzcompose files import `LiteralString` from `typing`, which needs 3.11. That went unnoticed because `ci/test/lint-main/checks/check-python-version.sh` only byte-compiles the tree, and compilation does not resolve imports, so a newer-than-floor import passes the check and fails at run time. 3.13 is what the CI builder already runs, since that is the system Python in Debian 13. Raising `target-version` lets ruff replace `datetime.timezone.utc` with the `datetime.UTC` alias and pull `LiteralString` back from `typing`, which is the whole of the mechanical churn here. Ruff trails at `py312` because the pinned version predates 3.13 and rejects `py313`. Co-Authored-By: Claude Opus 5 (1M context) --- bin/pyactivate | 15 ++++++++++++--- ci/cleanup/aws.py | 10 ++++------ ci/deploy/pypi.py | 2 +- ci/load/periodic.py | 2 +- ci/mkpipeline.py | 8 ++++---- .../lint-main/checks/check-python-version.sh | 17 ++++++++++------- .../materialize/cli/mz_workload_capture.py | 15 +++++++-------- misc/python/materialize/cli/scratch/claude.py | 2 +- misc/python/materialize/cli/scratch/create.py | 2 +- misc/python/materialize/cli/scratch/go.py | 2 +- misc/python/materialize/scratch.py | 4 ++-- .../materialize/workload_replay/replay.py | 4 ++-- pyproject.toml | 8 +++++--- test/canary-load/mzcompose.py | 18 ++++++------------ test/gcp/mzcompose.py | 6 +++--- test/terraform/mzcompose.py | 3 +-- 16 files changed, 61 insertions(+), 57 deletions(-) diff --git a/bin/pyactivate b/bin/pyactivate index a98af7aba76d9..898012fb72645 100755 --- a/bin/pyactivate +++ b/bin/pyactivate @@ -28,8 +28,8 @@ def main(args: list[str]) -> int: logger.debug("args=%s", args) # Validate Python version. - if sys.hexversion < 0x030a0000: - print("fatal: python v3.10.0+ required", file=sys.stderr) + if sys.hexversion < 0x030d0000: + print("fatal: python v3.13.0+ required", file=sys.stderr) print( " hint: you have v{}.{}.{}".format( sys.version_info.major, sys.version_info.minor, sys.version_info.micro @@ -90,7 +90,16 @@ def activate_venv(py_dir: Path) -> Path: logger.info("python exec error: %s", e) print(f"==> Initializing virtualenv in {venv_dir}", file=sys.stderr) try: - subprocess.check_call(["uv", "venv", venv_dir]) + # Build from the interpreter running this script, the one the version + # check above validated. Left to itself, `uv` resolves an interpreter + # by its own preference order, which favors uv-managed installs and so + # can pick a Python older than the system one and older than the check + # accepts. `--clear` matches the `venv.create` call below: reaching + # this point means the virtualenv is absent or unusable, and without + # the flag `uv` refuses to replace one that already exists. + subprocess.check_call( + ["uv", "venv", "--python", sys.executable, "--clear", venv_dir] + ) except FileNotFoundError: # Install Python into the virtualenv via a symlink rather than copying, # except on Windows. This matches the behavior of the `python -m venv` diff --git a/ci/cleanup/aws.py b/ci/cleanup/aws.py index 9665fdfc76825..301efe073eba8 100644 --- a/ci/cleanup/aws.py +++ b/ci/cleanup/aws.py @@ -7,7 +7,7 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import PurePosixPath from typing import Any from urllib.parse import unquote, urlparse @@ -29,7 +29,7 @@ def clean_up_kinesis() -> None: continue desc = client.describe_stream(StreamName=stream) created_at = desc["StreamDescription"]["StreamCreationTimestamp"] - age = datetime.now(timezone.utc) - created_at + age = datetime.now(UTC) - created_at if age <= MAX_AGE: print(f"Skipping stream {stream} whose age is beneath threshold") continue @@ -45,7 +45,7 @@ def clean_up_s3() -> None: if not desc["Name"].startswith("testdrive"): print("Skipping non-testdrive bucket {}".format(desc["Name"])) continue - age = datetime.now(timezone.utc) - desc["CreationDate"] + age = datetime.now(UTC) - desc["CreationDate"] if age <= MAX_AGE: print( "Skipping bucket {} whose age is beneath threshold".format(desc["Name"]) @@ -76,9 +76,7 @@ def clean_up_sqs() -> None: QueueUrl=queue, AttributeNames=["All"] ) created_at = int(attributes["Attributes"]["CreatedTimestamp"]) - age = datetime.now(timezone.utc) - datetime.fromtimestamp( - created_at, timezone.utc - ) + age = datetime.now(UTC) - datetime.fromtimestamp(created_at, UTC) if age <= MAX_AGE: print(f"Skipping queue {name} whose age is beneath threshold") continue diff --git a/ci/deploy/pypi.py b/ci/deploy/pypi.py index fd4ec77207c6d..a6048a0b73bda 100644 --- a/ci/deploy/pypi.py +++ b/ci/deploy/pypi.py @@ -7,7 +7,6 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -import distutils.core # pyright: ignore import os import sys import tarfile @@ -16,6 +15,7 @@ from pathlib import Path from typing import Literal +import distutils.core # pyright: ignore import requests from materialize import spawn diff --git a/ci/load/periodic.py b/ci/load/periodic.py index 4264d863a560f..489a390aa8b90 100644 --- a/ci/load/periodic.py +++ b/ci/load/periodic.py @@ -31,7 +31,7 @@ def main() -> None: now = datetime.datetime.utcnow() scratch.launch_cluster( [desc], - nonce=now.replace(tzinfo=datetime.timezone.utc).isoformat(), + nonce=now.replace(tzinfo=datetime.UTC).isoformat(), # Keep alive for at least a day. delete_after=datetime.datetime.utcnow() + datetime.timedelta(days=1), ) diff --git a/ci/mkpipeline.py b/ci/mkpipeline.py index 909a2d7de7ebd..4a064537d4152 100644 --- a/ci/mkpipeline.py +++ b/ci/mkpipeline.py @@ -31,7 +31,7 @@ from collections import OrderedDict from collections.abc import Iterable, Iterator from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -669,9 +669,9 @@ def switch_jobs_to_aws(pipeline: Any, priority: int) -> None: runnable = job.get("runnable_at") if not runnable or job.get("started_at"): continue - if datetime.now(timezone.utc) - datetime.fromisoformat( - runnable - ) < timedelta(minutes=20): + if datetime.now(UTC) - datetime.fromisoformat(runnable) < timedelta( + minutes=20 + ): continue print( diff --git a/ci/test/lint-main/checks/check-python-version.sh b/ci/test/lint-main/checks/check-python-version.sh index 769008106c93f..fe48de69d2b10 100755 --- a/ci/test/lint-main/checks/check-python-version.sh +++ b/ci/test/lint-main/checks/check-python-version.sh @@ -9,7 +9,7 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. # -# check-python-version.sh — make sure Python 3.10 keeps working +# check-python-version.sh — make sure the minimum supported Python keeps working set -euo pipefail @@ -17,6 +17,9 @@ cd "$(dirname "$0")/../../../.." . misc/shlib/shlib.bash +# Keep in sync with the version check in `bin/pyactivate`. +PYTHON_MIN_VERSION=3.13 + if [[ ! "${MZDEV_NO_PYTHON:-}" ]]; then if ! uv --version >/dev/null 2>/dev/null; then echo "lint: uv is not installed" @@ -24,19 +27,19 @@ if [[ ! "${MZDEV_NO_PYTHON:-}" ]]; then exit 1 fi - py310_venv="$(mktemp -d)/venv-py310" - trap 'rm -rf "$py310_venv"' EXIT + py_min_venv="$(mktemp -d)/venv-py-min" + trap 'rm -rf "$py_min_venv"' EXIT - try uv venv --python 3.10 "$py310_venv" - try uv pip compile --python-version 3.10 ci/builder/requirements.txt - try uv pip install --python "$py310_venv/bin/python" --requirement ci/builder/requirements.txt + try uv venv --python "$PYTHON_MIN_VERSION" "$py_min_venv" + try uv pip compile --python-version "$PYTHON_MIN_VERSION" ci/builder/requirements.txt + try uv pip install --python "$py_min_venv/bin/python" --requirement ci/builder/requirements.txt # Wrap the compileall (the actual work) in `try`, not git_files, and keep it # out of a pipeline: `try` in a pipeline runs in a subshell and loses its # accounting. Guard against empty input too, otherwise compileall with no # file arguments compiles all of sys.path instead of the repo. py_files=$(git_files '*.py') if [[ -n "$py_files" ]]; then - try xargs "$py310_venv/bin/python" -m compileall -q <<< "$py_files" + try xargs "$py_min_venv/bin/python" -m compileall -q <<< "$py_files" fi fi diff --git a/misc/python/materialize/cli/mz_workload_capture.py b/misc/python/materialize/cli/mz_workload_capture.py index 5a6ba3319e1bf..021c31da38edc 100644 --- a/misc/python/materialize/cli/mz_workload_capture.py +++ b/misc/python/materialize/cli/mz_workload_capture.py @@ -12,13 +12,12 @@ import threading import time from contextlib import contextmanager -from datetime import datetime, timezone -from typing import Any +from datetime import UTC, datetime +from typing import Any, LiteralString import psycopg import yaml from psycopg.sql import SQL, Composable, Composed, Identifier, Literal -from typing_extensions import LiteralString @contextmanager @@ -195,7 +194,7 @@ def attach_source_statistics_internal( ) WITH (PROGRESS) AS OF AT LEAST TIMESTAMP {}""").format( Literal(source_id), - Literal(datetime.fromtimestamp(start_time, tz=timezone.utc)), + Literal(datetime.fromtimestamp(start_time, tz=UTC)), ) first_timestamp = None for ( @@ -213,7 +212,7 @@ def attach_source_statistics_internal( continue if "bytes_total" not in source: first_timestamp = datetime.fromtimestamp( - int(mz_timestamp / 1000), tz=timezone.utc + int(mz_timestamp / 1000), tz=UTC ) source["bytes_total"] = int(bytes_received) source["messages_total"] = int(messages_received) @@ -221,7 +220,7 @@ def attach_source_statistics_internal( assert "bytes_total" in source and "messages_total" in source assert first_timestamp duration = ( - datetime.fromtimestamp(int(mz_timestamp / 1000), tz=timezone.utc) + datetime.fromtimestamp(int(mz_timestamp / 1000), tz=UTC) - first_timestamp ).total_seconds() if duration: @@ -287,7 +286,7 @@ def main() -> int: "--output", type=str, help="Path to write the workload.yml, - for stdout", - default=f"workload_{datetime.now(timezone.utc).strftime('%Y-%m-%dT%H-%M-%S')}.yml", + default=f"workload_{datetime.now(UTC).strftime('%Y-%m-%dT%H-%M-%S')}.yml", ) parser.add_argument( "--time", @@ -644,7 +643,7 @@ def main() -> int: conn, SQL( "SELECT sql, cluster_name, database_name, search_path, statement_type, finished_status, params, transaction_isolation, session_id, transaction_id, began_at, finished_at - began_at, result_size FROM mz_internal.mz_recent_activity_log WHERE began_at > {} ORDER BY began_at ASC" - ).format(Literal(datetime.fromtimestamp(start_time, tz=timezone.utc))), + ).format(Literal(datetime.fromtimestamp(start_time, tz=UTC))), ): workload["queries"].append( { diff --git a/misc/python/materialize/cli/scratch/claude.py b/misc/python/materialize/cli/scratch/claude.py index 29ec4e88d3c9f..1e7e078c3c713 100644 --- a/misc/python/materialize/cli/scratch/claude.py +++ b/misc/python/materialize/cli/scratch/claude.py @@ -114,7 +114,7 @@ def run(args: argparse.Namespace) -> None: instances = launch_cluster( descs, extra_tags=extra_tags, - delete_after=datetime.datetime.now(datetime.timezone.utc) + max_age, + delete_after=datetime.datetime.now(datetime.UTC) + max_age, ) print("Launched:") diff --git a/misc/python/materialize/cli/scratch/create.py b/misc/python/materialize/cli/scratch/create.py index c2775261f5180..95db5a858d493 100644 --- a/misc/python/materialize/cli/scratch/create.py +++ b/misc/python/materialize/cli/scratch/create.py @@ -239,7 +239,7 @@ def run(args: argparse.Namespace) -> None: security_group_name=args.security_group_name, instance_profile=args.instance_profile, extra_tags=extra_tags, - delete_after=datetime.datetime.now(datetime.timezone.utc) + max_age, + delete_after=datetime.datetime.now(datetime.UTC) + max_age, git_rev=args.git_rev, extra_env={}, ) diff --git a/misc/python/materialize/cli/scratch/go.py b/misc/python/materialize/cli/scratch/go.py index acaf0fac950b4..625523fed6712 100644 --- a/misc/python/materialize/cli/scratch/go.py +++ b/misc/python/materialize/cli/scratch/go.py @@ -99,7 +99,7 @@ def run(args: argparse.Namespace) -> None: instances = launch_cluster( descs, extra_tags=extra_tags, - delete_after=datetime.datetime.now(datetime.timezone.utc) + max_age, + delete_after=datetime.datetime.now(datetime.UTC) + max_age, ) print("Launched:") diff --git a/misc/python/materialize/scratch.py b/misc/python/materialize/scratch.py index 3e824165e1083..649e5b03e5b9f 100644 --- a/misc/python/materialize/scratch.py +++ b/misc/python/materialize/scratch.py @@ -420,7 +420,7 @@ def launch( tags.setdefault("team", "engineering") tags.setdefault( "deleteAfter", - delete_after.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + delete_after.astimezone(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), ) ec2 = boto3.client("ec2") @@ -708,7 +708,7 @@ def is_old(i: InstanceTypeDef) -> bool: if delete_after is None: return False delete_after = float(delete_after) - return datetime.datetime.now(datetime.timezone.utc).timestamp() > delete_after + return datetime.datetime.now(datetime.UTC).timestamp() > delete_after return [ i diff --git a/misc/python/materialize/workload_replay/replay.py b/misc/python/materialize/workload_replay/replay.py index 8918978b087a4..4dab70e69369d 100644 --- a/misc/python/materialize/workload_replay/replay.py +++ b/misc/python/materialize/workload_replay/replay.py @@ -218,7 +218,7 @@ def submit_query(query: dict[str, Any]) -> None: i = 0 while True: i += 1 - replay_start = datetime.datetime.now(datetime.timezone.utc) + replay_start = datetime.datetime.now(datetime.UTC) for query in workload["queries"]: if stop_event.is_set(): @@ -244,7 +244,7 @@ def submit_query(query: dict[str, Any]) -> None: offset = (query["began_at"] - start) / factor_queries scheduled = replay_start + offset sleep_seconds = ( - scheduled - datetime.datetime.now(datetime.timezone.utc) + scheduled - datetime.datetime.now(datetime.UTC) ).total_seconds() if sleep_seconds > 0: diff --git a/pyproject.toml b/pyproject.toml index 2821f2ca1697f..bd1ed764c7188 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,16 +8,18 @@ # by the Apache License, Version 2.0. [tool.black] -target_version = ["py310"] +target_version = ["py313"] # DEFAULT_EXCLUDES from https://github.com/psf/black/blob/main/src/black/const.py but without "build" directory since we use it in our source code. Instead exclude target and target-xcompile exclude = "(\\.direnv|\\.eggs|\\.git|\\.hg|\\.ipynb_checkpoints|\\.mypy_cache|\\.nox|\\.pytest_cache|\\.ruff_cache|\\.tox|\\.svn|\\.venv|\\.vscode|__pypackages__|_build|buck-out|dist|venv|target|target-xcompile|\\.terraform)" [tool.ruff] -target-version = "py310" +# The pinned ruff predates Python 3.13 and rejects `py313`, so this trails the +# minimum version enforced by `bin/pyactivate`. Raise it once ruff is upgraded. +target-version = "py312" select = [ "F", "I", - "UP", # e.g. PEP585 (Python 3.10+) type annotations + "UP", # e.g. PEP585 built-in generic type annotations "E711", # comparisons to none ] extend-exclude = [ diff --git a/test/canary-load/mzcompose.py b/test/canary-load/mzcompose.py index 18cd5321514aa..d947923d23e4d 100644 --- a/test/canary-load/mzcompose.py +++ b/test/canary-load/mzcompose.py @@ -90,7 +90,7 @@ def take_pending(self) -> list[str]: def _run(self) -> None: while not self._stop.is_set(): try: - now = datetime.datetime.now(tz=datetime.timezone.utc) + now = datetime.datetime.now(tz=datetime.UTC) ts = now.isoformat() payload = { "event_type": "canary_heartbeat", @@ -201,9 +201,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: ) as e: error_msg_str = str(e) if is_connection_error(error_msg_str): - now = datetime.datetime.now( - tz=datetime.timezone.utc - ).isoformat() + now = datetime.datetime.now(tz=datetime.UTC).isoformat() connection_failures.append((now, error_msg_str)) print(f"Connection failure at {now}: {e}; retrying") else: @@ -215,9 +213,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: if "Non-positive multiplicity in DistinctBy" in error.message: continue if is_connection_error(error.message): - now = datetime.datetime.now( - tz=datetime.timezone.utc - ).isoformat() + now = datetime.datetime.now(tz=datetime.UTC).isoformat() connection_failures.append((now, error.message)) print( f"Connection failure at {now}: {error.message}; continuing." @@ -234,9 +230,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: if "Non-positive multiplicity in DistinctBy" in msg: continue if is_connection_error(msg): - now = datetime.datetime.now( - tz=datetime.timezone.utc - ).isoformat() + now = datetime.datetime.now(tz=datetime.UTC).isoformat() connection_failures.append((now, msg)) print(f"Connection failure at {now}: {msg}; continuing.") continue @@ -559,8 +553,8 @@ def validate_webhook( latest_ts = max(received_timestamps) latest_dt = datetime.datetime.fromisoformat(latest_ts) if latest_dt.tzinfo is None: - latest_dt = latest_dt.replace(tzinfo=datetime.timezone.utc) - age = datetime.datetime.now(tz=datetime.timezone.utc) - latest_dt + latest_dt = latest_dt.replace(tzinfo=datetime.UTC) + age = datetime.datetime.now(tz=datetime.UTC) - latest_dt assert ( age.total_seconds() < 120 ), f"Webhook ingestion stall: latest event is {age} old (ts={latest_ts}), expected < 120s" diff --git a/test/gcp/mzcompose.py b/test/gcp/mzcompose.py index b005d6f9b0327..566c8d07a7c01 100644 --- a/test/gcp/mzcompose.py +++ b/test/gcp/mzcompose.py @@ -28,7 +28,7 @@ import urllib.error import urllib.parse import urllib.request -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from typing import Any import fastavro @@ -257,7 +257,7 @@ def _sweep_stale_biglake_namespaces(token: str, project: str, prefix: str) -> No in the namespace name to age out anything older than STALE_NAMESPACE_AGE without an extra metadata round-trip per namespace. """ - today = datetime.now(timezone.utc).date() + today = datetime.now(UTC).date() for ns in _list_biglake_namespaces(token, project, prefix): match = NAMESPACE_RE.match(ns) if not match: @@ -287,7 +287,7 @@ def workflow_default(c: Composition) -> None: # bucket don't collide on table state. The embedded date lets the pre-test # sweep age out namespaces left behind by killed runs. seed = random.getrandbits(32) - today = datetime.now(timezone.utc).strftime(NAMESPACE_DATE_FORMAT) + today = datetime.now(UTC).strftime(NAMESPACE_DATE_FORMAT) namespace = f"{NAMESPACE_PREFIX}_{today}_{seed:08x}" table = "demo_table" # The .td inserts these three rows; verification asserts the table's live diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 311853c4c93c6..62256f03f16f2 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -735,8 +735,7 @@ def setup( delete_after = "2099-12-31T00:00:00Z" else: delete_after = ( - datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(hours=24) + datetime.datetime.now(datetime.UTC) + datetime.timedelta(hours=24) ).strftime("%Y-%m-%dT%H:%M:%SZ") tags = { "Environment": "dev", From 4925340bb5945b9355d43447c397f904798fe9ee Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 2 Sep 2026 14:19:37 +0200 Subject: [PATCH 2/5] Check the minimum Python version against the virtualenv, not just the bootstrap The version gate in `main` inspects the interpreter that runs the script, while the code the repository executes runs inside `misc/python/venv`. Those need not agree, because the virtualenv outlives the interpreter it was built from, and the reuse path only confirmed that a `dep_stamp` existed and that the virtualenv's Python could execute at all. Raising the minimum therefore left every existing virtualenv below it in place, and this branch is the first to depend on that difference, since `datetime.UTC` does not exist before 3.11. The liveness probe now also reports the version, so a virtualenv older than the minimum takes the existing recreation path. `uv venv --clear` refuses a target directory that is not a valid virtualenv and suggests `--force`. That refusal is an error rather than the `FileNotFoundError` the surrounding code catches, so it would abort with a traceback instead of falling back to `venv.create`. The half-finished directory this leaves behind is the case the comment above the check already describes, and `--force` is the flag that actually matches `clear=True`. The developer guide named 3.12 and derived it from the default Python in the most recent Ubuntu LTS. That release is now 26.04 "Resolute Raccoon", which ships 3.14, so the stated rule no longer produced the stated version and taken literally would demand a version newer than the one CI runs. The guide now gives the minimum directly and explains that it tracks the CI builder image, noting that a current LTS satisfies it. The minimum lives in `MIN_HEXVERSION` and `MIN_VERSION` so the gate, the probe, and the operator-facing messages cannot drift apart. Co-Authored-By: Claude Opus 5 (1M context) --- bin/pyactivate | 41 ++++++++++++++++++++++++++++------------- doc/developer/guide.md | 10 ++++++---- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/bin/pyactivate b/bin/pyactivate index 898012fb72645..70a8dbce494a3 100755 --- a/bin/pyactivate +++ b/bin/pyactivate @@ -22,14 +22,20 @@ from typing import Optional logger = logging.getLogger("bootstrap") +# Minimum supported Python. Keep in sync with `PYTHON_MIN_VERSION` in +# `ci/test/lint-main/checks/check-python-version.sh` and with the version named +# in `doc/developer/guide.md`. +MIN_HEXVERSION = 0x030D0000 +MIN_VERSION = "3.13.0" + def main(args: list[str]) -> int: logging.basicConfig(level=os.environ.get("MZ_DEV_LOG", "WARNING").upper()) logger.debug("args=%s", args) # Validate Python version. - if sys.hexversion < 0x030d0000: - print("fatal: python v3.13.0+ required", file=sys.stderr) + if sys.hexversion < MIN_HEXVERSION: + print(f"fatal: python v{MIN_VERSION}+ required", file=sys.stderr) print( " hint: you have v{}.{}.{}".format( sys.version_info.major, sys.version_info.minor, sys.version_info.micro @@ -71,34 +77,43 @@ def activate_venv(py_dir: Path) -> Path: # have a working virtualenv. Instead we use the presence of the # `stamp_path`, as that indicates the virtualenv was once working enough to # have dependencies installed into it. + # The version check in `main` covers the bootstrap interpreter, not this one, + # and the two need not agree: the virtualenv outlives the interpreter it was + # built from. Probe the version here as well, so raising the minimum rebuilds + # existing virtualenvs instead of silently reusing one that is too old and + # failing later on an import the old version does not have. try: os.stat(stamp_path) - subprocess.check_call([python, "-c", ""]) + subprocess.check_call( + [python, "-c", f"import sys; sys.exit(sys.hexversion < {MIN_HEXVERSION})"] + ) except Exception as e: print("==> Checking for existing virtualenv", file=sys.stderr) if isinstance(e, FileNotFoundError): print("no existing virtualenv detected", file=sys.stderr) else: - # Usually just an indication that the user has upgraded the system - # Python that the virtualenv is referring to. Not important to - # bubble up the error here. If it's persistent, it'll occur in the - # new virtual env and bubble up when we exec later. + # Either the interpreter is gone, usually because the system Python + # it referred to was upgraded, or it predates the minimum version. + # Not important to tell the two apart or to bubble the error up: a + # problem that survives the rebuild surfaces when we exec later. print( - "warning: existing virtualenv is unable to execute python; will recreate", + f"warning: existing virtualenv is missing python or older than v{MIN_VERSION}; will recreate", file=sys.stderr ) - logger.info("python exec error: %s", e) + logger.info("python probe error: %s", e) print(f"==> Initializing virtualenv in {venv_dir}", file=sys.stderr) try: # Build from the interpreter running this script, the one the version # check above validated. Left to itself, `uv` resolves an interpreter # by its own preference order, which favors uv-managed installs and so # can pick a Python older than the system one and older than the check - # accepts. `--clear` matches the `venv.create` call below: reaching - # this point means the virtualenv is absent or unusable, and without - # the flag `uv` refuses to replace one that already exists. + # accepts. `--clear --force` matches the `venv.create` call below: + # reaching this point means the virtualenv is absent or unusable, so + # whatever occupies the path should go. `--clear` alone refuses a + # directory that is not a virtualenv, which is exactly the + # half-finished state the comment above describes. subprocess.check_call( - ["uv", "venv", "--python", sys.executable, "--clear", venv_dir] + ["uv", "venv", "--python", sys.executable, "--clear", "--force", venv_dir] ) except FileNotFoundError: # Install Python into the virtualenv via a symlink rather than copying, diff --git a/doc/developer/guide.md b/doc/developer/guide.md index 330361d63aeb2..76e1151c9b571 100644 --- a/doc/developer/guide.md +++ b/doc/developer/guide.md @@ -140,10 +140,12 @@ environment. Most of this should be taken care of by the `bin/pyactivate` script, which constructs a local virtual environment and keeps necessary dependencies up to date. -We support, as a minimum version, the default Python provided in the [most -recent Ubuntu LTS release](https://wiki.ubuntu.com/Releases). As of January 2026 -this is Python 3.12, provided in Ubuntu 24.04 "Noble Numbat". Earlier versions may -work but are not supported. Our recommended installation methods are: +We support Python 3.13 as a minimum version, which is what our CI builder image +provides and therefore the oldest version anything is tested against. It is also +no newer than the default Python in the [most recent Ubuntu LTS +release](https://wiki.ubuntu.com/Releases), which ships 3.14 as of Ubuntu 26.04 +"Resolute Raccoon", so a current LTS satisfies it. Earlier versions may work but +are not supported. Our recommended installation methods are: - macOS: [Homebrew](https://brew.sh) + [uv](https://docs.astral.sh/uv/) 1. `brew install uv` From c2cca652762f3d06bfb4d43b69186f0c5e1cd9c6 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 2 Sep 2026 15:30:36 +0200 Subject: [PATCH 3/5] Clear the virtualenv path directly instead of relying on uv flags `uv venv --force` was added in uv 0.11.17. Earlier versions exit with a usage error, which reaches `subprocess.check_call` as a `CalledProcessError` and not as the `FileNotFoundError` that selects the `venv.create` fallback, so `bin/pyactivate` would abort. Everything in the tree runs through that script, and the CI builder image installs uv 0.9.10, so the flag cannot be used here. The version probe added alongside it makes the timing worse rather than academic. Any developer holding a virtualenv below the new minimum takes the recreation path on their first invocation, which is exactly the path that would have failed, and deleting the virtualenv by hand leads to the same call. Removing the directory before invoking `uv` behaves the same on every version and needs no flag. It also settles what `--clear` and `--force` disagree about, a path that exists but is not a virtualenv, which is the half-finished state the comment above the check already describes. The `venv.create` fallback is unaffected, since it creates a directory that does not exist. Co-Authored-By: Claude Opus 5 (1M context) --- bin/pyactivate | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/bin/pyactivate b/bin/pyactivate index 70a8dbce494a3..104902ddb3d15 100755 --- a/bin/pyactivate +++ b/bin/pyactivate @@ -14,6 +14,7 @@ import logging import os import platform +import shutil import subprocess import sys import venv @@ -103,18 +104,19 @@ def activate_venv(py_dir: Path) -> Path: logger.info("python probe error: %s", e) print(f"==> Initializing virtualenv in {venv_dir}", file=sys.stderr) try: + # Clear the path first. Reaching this point means the virtualenv is + # absent or unusable, so whatever occupies it should go, and removing + # it here rather than asking `uv` to keeps the behavior the same on + # every `uv` version. The flags that would express this, `--clear` and + # `--force`, disagree about a path that is not a virtualenv, and + # `--force` does not exist before uv 0.11.17. + shutil.rmtree(venv_dir, ignore_errors=True) # Build from the interpreter running this script, the one the version # check above validated. Left to itself, `uv` resolves an interpreter # by its own preference order, which favors uv-managed installs and so # can pick a Python older than the system one and older than the check - # accepts. `--clear --force` matches the `venv.create` call below: - # reaching this point means the virtualenv is absent or unusable, so - # whatever occupies the path should go. `--clear` alone refuses a - # directory that is not a virtualenv, which is exactly the - # half-finished state the comment above describes. - subprocess.check_call( - ["uv", "venv", "--python", sys.executable, "--clear", "--force", venv_dir] - ) + # accepts. + subprocess.check_call(["uv", "venv", "--python", sys.executable, venv_dir]) except FileNotFoundError: # Install Python into the virtualenv via a symlink rather than copying, # except on Windows. This matches the behavior of the `python -m venv` From 5a09f2f50af926ae0cb3d1e1d2927e0e024ad5d5 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 2 Sep 2026 14:40:34 +0200 Subject: [PATCH 4/5] Upgrade ruff to 0.16.5 and target Python 3.13 The pinned ruff was 0.0.292, released in October 2023, and it rejects `py313`, so the lint target trailed the minimum version the repository enforces. This upgrade closes that gap and lets the `UP` rules see the language level we actually run on. Two interface changes come with it. Linting now requires the `check` subcommand, since a bare `ruff ` is no longer accepted, which affects `bin/fmt` and the lint check script. Top-level linter settings moved under a `lint` section, so `select` and `isort` become `lint.select` and `lint.isort`. Retargeting to 3.13 surfaces rewrites the old target could not suggest. The mechanical ones are PEP 695 type parameters in place of `TypeVar`, dropping default type arguments, and a few string and annotation modernizations. The `TypeVar` bindings that the PEP 695 conversion left with no remaining references are removed, since ruff rewrites the signature but does not clean up the declaration. The two `mzexplore` enums become `StrEnum`. For `ExplainStage` nothing changes at all, because it already defined `__str__` to return its value, and that override is now redundant. For `ItemType`, `str()` and formatting now produce the value rather than `ItemType.NAME`. No caller observes that: every use goes through `sql()`, which reads the value either way, and formatting the member directly does not appear anywhere. Co-Authored-By: Claude Opus 5 (1M context) --- ci/builder/requirements.txt | 2 +- .../lint-main/checks/check-python-files.sh | 4 ++-- .../mcp_materialize/mz_client.py | 2 +- misc/python/materialize/buildkite.py | 6 ++---- misc/python/materialize/cli/fmt.py | 4 ++-- misc/python/materialize/cli/orchestratord.py | 2 +- misc/python/materialize/cli/scratch/create.py | 3 +-- .../feature_benchmark/benchmark_result.py | 4 ++-- .../benchmark_result_evaluator.py | 6 +----- .../materialize/feature_benchmark/report.py | 4 ++-- misc/python/materialize/git.py | 8 +++++--- misc/python/materialize/mzcompose/__init__.py | 4 +--- misc/python/materialize/mzcompose/service.py | 2 +- misc/python/materialize/mzexplore/common.py | 6 +++--- misc/python/materialize/mzexplore/sql.py | 6 ++---- .../selection/column_selection.py | 4 ++-- misc/python/materialize/spawn.py | 7 ++----- misc/python/materialize/ui.py | 6 ++---- misc/python/materialize/util.py | 12 +++--------- pyproject.toml | 19 ++++++++++--------- test/clusterd-test-driver/run-local.py | 2 +- test/scalability/scalability.ipynb | 2 +- 22 files changed, 48 insertions(+), 67 deletions(-) diff --git a/ci/builder/requirements.txt b/ci/builder/requirements.txt index cfd56bc5696f4..3c12816d86813 100644 --- a/ci/builder/requirements.txt +++ b/ci/builder/requirements.txt @@ -61,7 +61,7 @@ pytest-split==0.11.0 pyyaml==6.0.3 requests==2.34.2 ruamel.yaml==0.18.17 -ruff==0.0.292 +ruff==0.16.5 # NOTE scipy 1.15 is required for Python 3.10 compatibility scipy==1.15.3; python_version < "3.11" scipy==1.17.1; python_version >= "3.11" diff --git a/ci/test/lint-main/checks/check-python-files.sh b/ci/test/lint-main/checks/check-python-files.sh index 4896138710b42..02dbb6a2b8bb4 100755 --- a/ci/test/lint-main/checks/check-python-files.sh +++ b/ci/test/lint-main/checks/check-python-files.sh @@ -28,9 +28,9 @@ if [[ ! "${MZDEV_NO_PYTHON:-}" ]]; then try xargs npx --yes "pyright@$pyright_version" --warnings --threads 4 < "$python_files_list" - try xargs bin/pyactivate -m ruff < "$python_files_list" + try xargs bin/pyactivate -m ruff check < "$python_files_list" # We need to maintain compatibility with older Python versions for this - try xargs bin/pyactivate -m ruff --target-version=py38 < "$dbt_files_list" + try xargs bin/pyactivate -m ruff check --target-version=py38 < "$dbt_files_list" fi try_status_report diff --git a/misc/mcp-materialize/mcp_materialize/mz_client.py b/misc/mcp-materialize/mcp_materialize/mz_client.py index a5d1927df09d3..243c637b82162 100644 --- a/misc/mcp-materialize/mcp_materialize/mz_client.py +++ b/misc/mcp-materialize/mcp_materialize/mz_client.py @@ -95,7 +95,7 @@ async def __aexit__(self, exc_type, exc, tb) -> None: self._bg_task.cancel() try: await asyncio.wait_for(self._bg_task, timeout=5.0) - except (asyncio.CancelledError, asyncio.TimeoutError): + except (TimeoutError, asyncio.CancelledError): pass async def _subscribe(self) -> None: diff --git a/misc/python/materialize/buildkite.py b/misc/python/materialize/buildkite.py index cf29330ced23d..7868d8d8793e0 100644 --- a/misc/python/materialize/buildkite.py +++ b/misc/python/materialize/buildkite.py @@ -13,14 +13,12 @@ from collections.abc import Callable from enum import Enum, auto from pathlib import Path -from typing import Any, TypeVar +from typing import Any import yaml from materialize import git, spawn, ui -T = TypeVar("T") - class BuildkiteEnvVar(Enum): # environment @@ -229,7 +227,7 @@ def notify_qa_team_about_failure(failure: str) -> None: ) -def shard_list(items: list[T], to_identifier: Callable[[T], str]) -> list[T]: +def shard_list[T](items: list[T], to_identifier: Callable[[T], str]) -> list[T]: if len(items) == 0: return [] diff --git a/misc/python/materialize/cli/fmt.py b/misc/python/materialize/cli/fmt.py index 9a12b7f631123..29b891b7abb91 100644 --- a/misc/python/materialize/cli/fmt.py +++ b/misc/python/materialize/cli/fmt.py @@ -137,7 +137,7 @@ def _ruff_cmd(*, check: bool) -> list[str]: return [ "bash", "-c", - f'. misc/shlib/shlib.bash && git_files "*.py" | grep -v "^misc/dbt-materialize/" | xargs bin/pyactivate -m ruff{fix}', + f'. misc/shlib/shlib.bash && git_files "*.py" | grep -v "^misc/dbt-materialize/" | xargs bin/pyactivate -m ruff check{fix}', ] @@ -146,7 +146,7 @@ def _ruff_dbt_cmd(*, check: bool) -> list[str]: return [ "bash", "-c", - f'. misc/shlib/shlib.bash && git_files "misc/dbt-materialize/*.py" | xargs bin/pyactivate -m ruff --target-version=py38{fix}', + f'. misc/shlib/shlib.bash && git_files "misc/dbt-materialize/*.py" | xargs bin/pyactivate -m ruff check --target-version=py38{fix}', ] diff --git a/misc/python/materialize/cli/orchestratord.py b/misc/python/materialize/cli/orchestratord.py index 9747c782345f3..ac39a46066699 100644 --- a/misc/python/materialize/cli/orchestratord.py +++ b/misc/python/materialize/cli/orchestratord.py @@ -477,7 +477,7 @@ def kubectl(*args: str, cluster: str, **subprocess_args): T = TypeVar("T") -def retry( +def retry[T]( f: Callable[[], T], max_attempts: int = 60, sleep_secs: int = 1, diff --git a/misc/python/materialize/cli/scratch/create.py b/misc/python/materialize/cli/scratch/create.py index 95db5a858d493..28b77d88077f3 100644 --- a/misc/python/materialize/cli/scratch/create.py +++ b/misc/python/materialize/cli/scratch/create.py @@ -156,8 +156,7 @@ def configure_parser(parser: argparse.ArgumentParser) -> None: "--instance-profile", type=str, default=DEFAULT_INSTANCE_PROFILE_NAME, - help="EC2 instance profile / IAM role. Defaults to `%s`." - % DEFAULT_INSTANCE_PROFILE_NAME, + help=f"EC2 instance profile / IAM role. Defaults to `{DEFAULT_INSTANCE_PROFILE_NAME}`.", ) parser.add_argument("--output-format", choices=["table", "csv"], default="table") parser.add_argument( diff --git a/misc/python/materialize/feature_benchmark/benchmark_result.py b/misc/python/materialize/feature_benchmark/benchmark_result.py index 3b2bffe545166..cac6ae44eaeca 100644 --- a/misc/python/materialize/feature_benchmark/benchmark_result.py +++ b/misc/python/materialize/feature_benchmark/benchmark_result.py @@ -9,7 +9,7 @@ from __future__ import annotations -from typing import Generic, TypeVar +from typing import TypeVar from materialize.feature_benchmark.measurement import MeasurementType, MeasurementUnit from materialize.feature_benchmark.scenario import Scenario @@ -56,7 +56,7 @@ def get_metric_by_measurement_type( return None -class BenchmarkScenarioMetric(Generic[T]): +class BenchmarkScenarioMetric[T]: def __init__( self, scenario_class: type[Scenario], measurement_type: MeasurementType ) -> None: diff --git a/misc/python/materialize/feature_benchmark/benchmark_result_evaluator.py b/misc/python/materialize/feature_benchmark/benchmark_result_evaluator.py index 87fcd997a07ae..52698da2c72b3 100644 --- a/misc/python/materialize/feature_benchmark/benchmark_result_evaluator.py +++ b/misc/python/materialize/feature_benchmark/benchmark_result_evaluator.py @@ -9,8 +9,6 @@ from __future__ import annotations -from typing import Generic, TypeVar - from materialize.feature_benchmark.benchmark_result import BenchmarkScenarioMetric from materialize.feature_benchmark.measurement import MeasurementType from materialize.feature_benchmark.scenario import Scenario @@ -20,10 +18,8 @@ with_conditional_formatting, ) -T = TypeVar("T") - -class BenchmarkResultEvaluator(Generic[T]): +class BenchmarkResultEvaluator[T]: def ratio(self, metric: BenchmarkScenarioMetric) -> float | None: raise RuntimeError diff --git a/misc/python/materialize/feature_benchmark/report.py b/misc/python/materialize/feature_benchmark/report.py index cb691aa4db9b9..0760d28e5a027 100644 --- a/misc/python/materialize/feature_benchmark/report.py +++ b/misc/python/materialize/feature_benchmark/report.py @@ -8,7 +8,7 @@ # by the Apache License, Version 2.0. from statistics import mean, variance -from typing import Generic, TypeVar +from typing import TypeVar from materialize.feature_benchmark.benchmark_result import BenchmarkScenarioResult from materialize.feature_benchmark.benchmark_result_evaluator import ( @@ -23,7 +23,7 @@ T = TypeVar("T", bound=int | float) -class ReportMeasurement(Generic[T]): +class ReportMeasurement[T: int | float]: result: T | None min: T | None max: T | None diff --git a/misc/python/materialize/git.py b/misc/python/materialize/git.py index 74d1cbb257dec..2d0d3cecd4948 100644 --- a/misc/python/materialize/git.py +++ b/misc/python/materialize/git.py @@ -124,7 +124,7 @@ def expand_globs(root: Path, *specs: Path | str) -> set[str]: return set(f for f in (diff_files + ls_files).split("\0") if f.strip() != "") -def get_version_tags( +def get_version_tags[VERSION_TYPE: TypedVersionBase]( *, version_type: type[VERSION_TYPE], newest_first: bool = True, @@ -157,7 +157,7 @@ def get_version_tags( return sorted(tags, reverse=newest_first) -def get_latest_version( +def get_latest_version[VERSION_TYPE: TypedVersionBase]( version_type: type[VERSION_TYPE], excluded_versions: set[VERSION_TYPE] | None = None, current_version: VERSION_TYPE | None = None, @@ -377,7 +377,9 @@ def contains_commit( return is_ancestor(commit_sha, target) -def get_tagged_release_version(version_type: type[VERSION_TYPE]) -> VERSION_TYPE | None: +def get_tagged_release_version[VERSION_TYPE: TypedVersionBase]( + version_type: type[VERSION_TYPE], +) -> VERSION_TYPE | None: """ This returns the release version if exactly this commit is tagged. If multiple release versions are present, the highest one will be returned. diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 1be7ba3c7ec07..b9b36d9c7216f 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -891,9 +891,7 @@ def _check_tcp( spawn.capture(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: ui.log_in_automation( - "wait-for-tcp ({}{}:{}): error running {}: {}, stdout:\n{}\nstderr:\n{}".format( - kind, host, port, ui.shell_quote(cmd), e, e.stdout, e.stderr - ) + f"wait-for-tcp ({kind}{host}:{port}): error running {ui.shell_quote(cmd)}: {e}, stdout:\n{e.stdout}\nstderr:\n{e.stderr}" ) raise return cmd diff --git a/misc/python/materialize/mzcompose/service.py b/misc/python/materialize/mzcompose/service.py index d6c1b1d3f8ca4..6052181bb1af4 100644 --- a/misc/python/materialize/mzcompose/service.py +++ b/misc/python/materialize/mzcompose/service.py @@ -203,4 +203,4 @@ class Service: def __init__(self, name: str, config: ServiceConfig) -> None: self.name = name self.config = config - self.companions: list["Service"] = [] + self.companions: list[Service] = [] diff --git a/misc/python/materialize/mzexplore/common.py b/misc/python/materialize/mzexplore/common.py index 37ee3fcf06ba5..90a4d164d7f8a 100644 --- a/misc/python/materialize/mzexplore/common.py +++ b/misc/python/materialize/mzexplore/common.py @@ -9,7 +9,7 @@ import re from dataclasses import dataclass, replace -from enum import Enum +from enum import Enum, StrEnum from importlib import resources from pathlib import Path from typing import cast @@ -45,7 +45,7 @@ def ext(self): return "txt" -class ExplainStage(str, Enum): +class ExplainStage(StrEnum): RAW_PLAN = "RAW PLAN" DECORRELATED_PLAN = "DECORRELATED PLAN" LOCAL_PLAN = "LOCALLY OPTIMIZED PLAN" @@ -140,7 +140,7 @@ def convert(self, value, param, ctx): # type: ignore raise ValueError(f"Bad explain option: {value}: {e!r}") from e -class ItemType(str, Enum): +class ItemType(StrEnum): CONNECTION = "connection" FUNCTION = "function" INDEX = "index" diff --git a/misc/python/materialize/mzexplore/sql.py b/misc/python/materialize/mzexplore/sql.py index cdf8be2234aae..99d012da6feb2 100644 --- a/misc/python/materialize/mzexplore/sql.py +++ b/misc/python/materialize/mzexplore/sql.py @@ -23,7 +23,7 @@ from materialize.mzexplore.common import resource_path -DictGenerator = Generator[dict[Any, Any], None, None] +DictGenerator = Generator[dict[Any, Any]] class Database: @@ -125,9 +125,7 @@ def arrangement_sizes(self, id: str) -> DictGenerator: @contextlib.contextmanager -def update_environment( - db: Database, env: dict[str, str] -) -> Generator[Database, None, None]: +def update_environment(db: Database, env: dict[str, str]) -> Generator[Database]: original = dict() for e in db.query_all("SHOW ALL"): key, old_value = e["name"], e["setting"] diff --git a/misc/python/materialize/output_consistency/selection/column_selection.py b/misc/python/materialize/output_consistency/selection/column_selection.py index 90c071e9f6f33..c0ce6871d0c95 100644 --- a/misc/python/materialize/output_consistency/selection/column_selection.py +++ b/misc/python/materialize/output_consistency/selection/column_selection.py @@ -7,7 +7,7 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -from typing import Generic, TypeVar +from typing import TypeVar from materialize.output_consistency.data_value.source_column_identifier import ( SourceColumnIdentifier, @@ -16,7 +16,7 @@ T = TypeVar("T") -class SelectionByKey(Generic[T]): +class SelectionByKey[T]: def __init__(self, keys: set[T] | None = None): self.keys = keys diff --git a/misc/python/materialize/spawn.py b/misc/python/materialize/spawn.py index 7f384f97a1e88..db1cdf17b7fe2 100644 --- a/misc/python/materialize/spawn.py +++ b/misc/python/materialize/spawn.py @@ -21,7 +21,7 @@ import time from collections.abc import Callable, Sequence from pathlib import Path -from typing import IO, TypeVar +from typing import IO from materialize import ui @@ -144,10 +144,7 @@ def run_and_get_return_code( return e.returncode -T = TypeVar("T") # Generic type variable - - -def run_with_retries(fn: Callable[[], T], max_duration: int = 60) -> T: +def run_with_retries[T](fn: Callable[[], T], max_duration: int = 60) -> T: """Retry a function until it doesn't raise a `CalledProcessError`, uses exponential backoff until `max_duration` is reached.""" for retry in range(math.ceil(math.log2(max_duration))): diff --git a/misc/python/materialize/ui.py b/misc/python/materialize/ui.py index 40d68e7f83af9..6e0eb8b3adbc2 100644 --- a/misc/python/materialize/ui.py +++ b/misc/python/materialize/ui.py @@ -82,7 +82,7 @@ def progress(msg: str = "", prefix: str | None = None, *, finish: bool = False) print(msg, file=sys.stderr, flush=True, end=end) -def timeout_loop(timeout: int, tick: float = 1.0) -> Generator[float, None, None]: +def timeout_loop(timeout: int, tick: float = 1.0) -> Generator[float]: """Loop until timeout, optionally sleeping until tick Always iterates at least once @@ -105,9 +105,7 @@ def timeout_loop(timeout: int, tick: float = 1.0) -> Generator[float, None, None time.sleep(tick - (after - before)) -async def async_timeout_loop( - timeout: int, tick: float = 1.0 -) -> AsyncGenerator[float, None]: +async def async_timeout_loop(timeout: int, tick: float = 1.0) -> AsyncGenerator[float]: """Loop until timeout, asynchronously sleeping until tick Always iterates at least once diff --git a/misc/python/materialize/util.py b/misc/python/materialize/util.py index 7af15417ce168..41dd881c4bb1b 100644 --- a/misc/python/materialize/util.py +++ b/misc/python/materialize/util.py @@ -23,7 +23,7 @@ from enum import Enum from pathlib import Path from threading import Thread -from typing import Protocol, TypeVar +from typing import Protocol from urllib.parse import parse_qs, quote, unquote, urlparse import psycopg @@ -37,10 +37,7 @@ def nonce(digits: int) -> str: return "".join(random.choice("0123456789abcdef") for _ in range(digits)) -T = TypeVar("T") - - -def all_subclasses(cls: type[T]) -> set[type[T]]: +def all_subclasses[T](cls: type[T]) -> set[type[T]]: """Returns a recursive set of all subclasses of a class""" sc = cls.__subclasses__() return set(sc).union([subclass for c in sc for subclass in all_subclasses(c)]) @@ -126,10 +123,7 @@ class HasName(Protocol): name: str -U = TypeVar("U", bound=HasName) - - -def selected_by_name(selected: list[str], objs: list[U]) -> Iterator[U]: +def selected_by_name[U: HasName](selected: list[str], objs: list[U]) -> Iterator[U]: for name in selected: for obj in objs: if obj.name == name: diff --git a/pyproject.toml b/pyproject.toml index bd1ed764c7188..8fa2c665b9fa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,22 +13,23 @@ target_version = ["py313"] exclude = "(\\.direnv|\\.eggs|\\.git|\\.hg|\\.ipynb_checkpoints|\\.mypy_cache|\\.nox|\\.pytest_cache|\\.ruff_cache|\\.tox|\\.svn|\\.venv|\\.vscode|__pypackages__|_build|buck-out|dist|venv|target|target-xcompile|\\.terraform)" [tool.ruff] -# The pinned ruff predates Python 3.13 and rejects `py313`, so this trails the -# minimum version enforced by `bin/pyactivate`. Raise it once ruff is upgraded. -target-version = "py312" +# Keep in sync with the minimum version enforced by `bin/pyactivate`. +target-version = "py313" +extend-exclude = [ + "venv", "target", + # This dbt adapter may be published elsewhere, so doesn't follow the same compatibility rules as our internal code. + "misc/dbt-materialize", +] + +[tool.ruff.lint] select = [ "F", "I", "UP", # e.g. PEP585 built-in generic type annotations "E711", # comparisons to none ] -extend-exclude = [ - "venv", "target", - # This dbt adapter may be published elsewhere, so doesn't follow the same compatibility rules as our internal code. - "misc/dbt-materialize", -] -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ["materialize"] [tool.pyright] diff --git a/test/clusterd-test-driver/run-local.py b/test/clusterd-test-driver/run-local.py index 73ddb091d72df..63518c134993d 100644 --- a/test/clusterd-test-driver/run-local.py +++ b/test/clusterd-test-driver/run-local.py @@ -258,7 +258,7 @@ def main() -> int: ensure_cockroach() cargo_build() - launched: "subprocess.Popen[bytes] | None" = None + launched: subprocess.Popen[bytes] | None = None clusterd_pid: int | None = None try: if RUN_CLUSTERD: diff --git a/test/scalability/scalability.ipynb b/test/scalability/scalability.ipynb index 164316ceabc07..aa9f13a5e256f 100644 --- a/test/scalability/scalability.ipynb +++ b/test/scalability/scalability.ipynb @@ -17,7 +17,7 @@ "# by the Apache License, Version 2.0.\n", "\n", "import pandas as pd\n", - "from ipywidgets import widgets, interactive\n", + "from ipywidgets import interactive, widgets\n", "from lib import plotit\n", "\n", "workloads = pd.read_csv(\"results/workloads.csv\")\n", From 383e5772810602134845a0ad685935536e959041 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 2 Sep 2026 14:51:34 +0200 Subject: [PATCH 5/5] Select RUF100 and drop the suppressions that suppress nothing Of 38 `noqa` directives, 29 had no effect. They were invisible because the rule that reports them was not selected, and they predate the ruff upgrade: version 0.0.292 reports the same 29. Most are star imports annotated `F401 F403`. A star import triggers F403 and never F401, so only the F403 half ever did anything, and those directives are narrowed to it. Three name rules from families this repository does not select, `SLF001`, `E731` and `BLE001`, are removed along with one blanket directive on an import that is used. The eight that remain sit on genuine re-exports in `mzexplore/__init__.py`, where F401 does fire. Selecting RUF100 keeps the set honest, since a directive that stops applying now fails the lint rather than accumulating. Co-Authored-By: Claude Opus 5 (1M context) --- .../cli/mz_workload_anonymize_test.py | 2 +- misc/python/materialize/cli/mzcompose.py | 2 +- .../scalability/executor/benchmark_executor.py | 8 ++++---- misc/python/materialize/zippy/scenarios.py | 2 +- pyproject.toml | 1 + test/cloudtest/test_upgrade.py | 2 +- test/data-ingest/mzcompose.py | 2 +- test/feature-benchmark/mzcompose.py | 16 ++++++++-------- test/mcp/mzcompose.py | 2 +- test/parallel-benchmark/mzcompose.py | 2 +- test/platform-checks/mzcompose.py | 10 +++++----- test/scalability/mzcompose.py | 8 ++++---- test/zippy/mzcompose.py | 2 +- 13 files changed, 30 insertions(+), 29 deletions(-) diff --git a/misc/python/materialize/cli/mz_workload_anonymize_test.py b/misc/python/materialize/cli/mz_workload_anonymize_test.py index a5647e23b8c04..d4e0a867a2c81 100644 --- a/misc/python/materialize/cli/mz_workload_anonymize_test.py +++ b/misc/python/materialize/cli/mz_workload_anonymize_test.py @@ -465,7 +465,7 @@ def cross_schema_cdc_workload() -> dict[str, Any]: schema. The child key is built in pass 1 while processing the source's schema (`aaa_src`), before the child's schema (`zzz_upstream`) is mapped, so a stale key leaks the original schema name unless rebuilt in pass 2.""" - empty = lambda: { # noqa: E731 + empty = lambda: { "tables": {}, "views": {}, "materialized_views": {}, diff --git a/misc/python/materialize/cli/mzcompose.py b/misc/python/materialize/cli/mzcompose.py index c438e09ee9e05..2530fad163593 100644 --- a/misc/python/materialize/cli/mzcompose.py +++ b/misc/python/materialize/cli/mzcompose.py @@ -167,7 +167,7 @@ def main(argv: list[str]) -> None: # Convert to a list of the members: shtab renders them via str(), and # argparse still accepts the type-converted member, which converting to # name strings would not (the member never compares equal to its name). - for action in parser._actions: # noqa: SLF001 + for action in parser._actions: if isinstance(action.choices, type) and issubclass(action.choices, enum.Enum): action.choices = list(action.choices) diff --git a/misc/python/materialize/scalability/executor/benchmark_executor.py b/misc/python/materialize/scalability/executor/benchmark_executor.py index 2f6146a747e91..6c0f779e339a0 100644 --- a/misc/python/materialize/scalability/executor/benchmark_executor.py +++ b/misc/python/materialize/scalability/executor/benchmark_executor.py @@ -30,10 +30,10 @@ from materialize.scalability.schema.schema import Schema from materialize.scalability.workload.workload import Workload, WorkloadWithContext from materialize.scalability.workload.workload_markers import ConnectionWorkload -from materialize.scalability.workload.workloads.connection_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.ddl_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.dml_dql_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.self_test_workloads import * # noqa: F401 F403 +from materialize.scalability.workload.workloads.connection_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.ddl_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.dml_dql_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.self_test_workloads import * # noqa: F403 # number of retries in addition to the first run MAX_RETRIES_ON_REGRESSION = 2 diff --git a/misc/python/materialize/zippy/scenarios.py b/misc/python/materialize/zippy/scenarios.py index 2351b43120035..7107fac27c595 100644 --- a/misc/python/materialize/zippy/scenarios.py +++ b/misc/python/materialize/zippy/scenarios.py @@ -18,7 +18,7 @@ from materialize.zippy.blob_store_actions import BlobStoreRestart, BlobStoreStart from materialize.zippy.crdb_actions import CockroachRestart, CockroachStart from materialize.zippy.debezium_actions import CreateDebeziumSource, DebeziumStart -from materialize.zippy.framework import ActionOrFactory # noqa +from materialize.zippy.framework import ActionOrFactory from materialize.zippy.kafka_actions import ( CreateTopicParameterized, Ingest, diff --git a/pyproject.toml b/pyproject.toml index 8fa2c665b9fa5..dbc9432866b7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ select = [ "I", "UP", # e.g. PEP585 built-in generic type annotations "E711", # comparisons to none + "RUF100", # `noqa` that suppresses nothing, including for unselected rules ] [tool.ruff.lint.isort] diff --git a/test/cloudtest/test_upgrade.py b/test/cloudtest/test_upgrade.py index ba3bf79903bd1..0495be85e9ffe 100644 --- a/test/cloudtest/test_upgrade.py +++ b/test/cloudtest/test_upgrade.py @@ -14,7 +14,7 @@ from materialize import buildkite from materialize.checks.actions import Action, Initialize, Manipulate, Validate -from materialize.checks.all_checks import * # noqa: F401 F403 +from materialize.checks.all_checks import * # noqa: F403 from materialize.checks.all_checks.alter_connection import ( AlterConnectionHost, AlterConnectionToNonSsh, diff --git a/test/data-ingest/mzcompose.py b/test/data-ingest/mzcompose.py index a0fc19f6081f5..91371057faf1f 100644 --- a/test/data-ingest/mzcompose.py +++ b/test/data-ingest/mzcompose.py @@ -21,7 +21,7 @@ KafkaExecutor, MySqlExecutor, ) -from materialize.data_ingest.workload import * # noqa: F401 F403 +from materialize.data_ingest.workload import * # noqa: F403 from materialize.data_ingest.workload import WORKLOADS, execute_workload from materialize.mzcompose import get_default_system_parameters from materialize.mzcompose.composition import Composition, WorkflowArgumentParser diff --git a/test/feature-benchmark/mzcompose.py b/test/feature-benchmark/mzcompose.py index 88fd9d8767477..3143ec7322308 100644 --- a/test/feature-benchmark/mzcompose.py +++ b/test/feature-benchmark/mzcompose.py @@ -72,17 +72,17 @@ from materialize.feature_benchmark.executor import Docker from materialize.feature_benchmark.filter import Filter, FilterFirst, NoFilter from materialize.feature_benchmark.measurement import MeasurementType -from materialize.feature_benchmark.scenarios.benchmark_main import * # noqa: F401 F403 +from materialize.feature_benchmark.scenarios.benchmark_main import * # noqa: F403 from materialize.feature_benchmark.scenarios.benchmark_main import ( Scenario, ) -from materialize.feature_benchmark.scenarios.concurrency import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.customer import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.optbench import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.scale import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.skew import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.subscribe import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.temporal import * # noqa: F401 F403 +from materialize.feature_benchmark.scenarios.concurrency import * # noqa: F403 +from materialize.feature_benchmark.scenarios.customer import * # noqa: F403 +from materialize.feature_benchmark.scenarios.optbench import * # noqa: F403 +from materialize.feature_benchmark.scenarios.scale import * # noqa: F403 +from materialize.feature_benchmark.scenarios.skew import * # noqa: F403 +from materialize.feature_benchmark.scenarios.subscribe import * # noqa: F403 +from materialize.feature_benchmark.scenarios.temporal import * # noqa: F403 from materialize.feature_benchmark.termination import ( NormalDistributionOverlap, ProbForMin, diff --git a/test/mcp/mzcompose.py b/test/mcp/mzcompose.py index c4e053dae6a86..b2e5d139f5e25 100644 --- a/test/mcp/mzcompose.py +++ b/test/mcp/mzcompose.py @@ -1023,7 +1023,7 @@ def run(cur: Cursor, sql: str) -> str: """Run `sql`; return ``"ok:"`` or the error message string.""" try: cur.execute(sql.encode()) - except Exception as e: # noqa: BLE001 — the message is the assertion + except Exception as e: return str(e) if cur.description is None: return "ok:0" diff --git a/test/parallel-benchmark/mzcompose.py b/test/parallel-benchmark/mzcompose.py index 1690cb5e53144..1e252c91673ad 100644 --- a/test/parallel-benchmark/mzcompose.py +++ b/test/parallel-benchmark/mzcompose.py @@ -60,7 +60,7 @@ SQLiteStore, State, ) -from materialize.parallel_benchmark.scenarios import * # noqa: F401 F403 +from materialize.parallel_benchmark.scenarios import * # noqa: F403 from materialize.test_analytics.config.test_analytics_db_config import ( create_test_analytics_config, ) diff --git a/test/platform-checks/mzcompose.py b/test/platform-checks/mzcompose.py index 3cc6a788ec91c..b2b9d6419fcbe 100644 --- a/test/platform-checks/mzcompose.py +++ b/test/platform-checks/mzcompose.py @@ -18,15 +18,15 @@ from enum import Enum from materialize import buildkite -from materialize.checks.all_checks import * # noqa: F401 F403 +from materialize.checks.all_checks import * # noqa: F403 from materialize.checks.checks import Check from materialize.checks.executors import MzcomposeExecutor, MzcomposeExecutorParallel from materialize.checks.features import Features -from materialize.checks.scenarios import * # noqa: F401 F403 +from materialize.checks.scenarios import * # noqa: F403 from materialize.checks.scenarios import Scenario, SystemVarChange -from materialize.checks.scenarios_backup_restore import * # noqa: F401 F403 -from materialize.checks.scenarios_upgrade import * # noqa: F401 F403 -from materialize.checks.scenarios_zero_downtime import * # noqa: F401 F403 +from materialize.checks.scenarios_backup_restore import * # noqa: F403 +from materialize.checks.scenarios_upgrade import * # noqa: F403 +from materialize.checks.scenarios_zero_downtime import * # noqa: F403 from materialize.mzcompose.composition import ( Composition, Service, diff --git a/test/scalability/mzcompose.py b/test/scalability/mzcompose.py index 17ecf5589e570..c364a224a3fdb 100644 --- a/test/scalability/mzcompose.py +++ b/test/scalability/mzcompose.py @@ -58,10 +58,10 @@ from materialize.scalability.schema.schema import Schema, TransactionIsolation from materialize.scalability.workload.workload import Workload from materialize.scalability.workload.workload_markers import WorkloadMarker -from materialize.scalability.workload.workloads.connection_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.ddl_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.dml_dql_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.self_test_workloads import * # noqa: F401 F403 +from materialize.scalability.workload.workloads.connection_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.ddl_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.dml_dql_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.self_test_workloads import * # noqa: F403 from materialize.test_analytics.config.test_analytics_db_config import ( create_test_analytics_config, ) diff --git a/test/zippy/mzcompose.py b/test/zippy/mzcompose.py index 7d07106d10350..0078f2f060777 100644 --- a/test/zippy/mzcompose.py +++ b/test/zippy/mzcompose.py @@ -48,7 +48,7 @@ from materialize.mzcompose.services.testdrive import Testdrive from materialize.zippy.framework import Test, ci_additional_system_parameter_defaults from materialize.zippy.mz_actions import Mz0dtDeploy -from materialize.zippy.scenarios import * # noqa: F401 F403 +from materialize.zippy.scenarios import * # noqa: F403 def create_mzs(