diff --git a/bin/pyactivate b/bin/pyactivate index a98af7aba76d9..104902ddb3d15 100755 --- a/bin/pyactivate +++ b/bin/pyactivate @@ -14,6 +14,7 @@ import logging import os import platform +import shutil import subprocess import sys import venv @@ -22,14 +23,20 @@ from typing import Optional logger = logging.getLogger("bootstrap") +# Minimum supported Python. Keep in sync with `PYTHON_MIN_VERSION` in +# `ci/test/lint-main/checks/check-python-version.sh` and with the version named +# in `doc/developer/guide.md`. +MIN_HEXVERSION = 0x030D0000 +MIN_VERSION = "3.13.0" + def main(args: list[str]) -> int: logging.basicConfig(level=os.environ.get("MZ_DEV_LOG", "WARNING").upper()) logger.debug("args=%s", args) # Validate Python version. - if sys.hexversion < 0x030a0000: - print("fatal: python v3.10.0+ required", file=sys.stderr) + if sys.hexversion < MIN_HEXVERSION: + print(f"fatal: python v{MIN_VERSION}+ required", file=sys.stderr) print( " hint: you have v{}.{}.{}".format( sys.version_info.major, sys.version_info.minor, sys.version_info.micro @@ -71,26 +78,45 @@ def activate_venv(py_dir: Path) -> Path: # have a working virtualenv. Instead we use the presence of the # `stamp_path`, as that indicates the virtualenv was once working enough to # have dependencies installed into it. + # The version check in `main` covers the bootstrap interpreter, not this one, + # and the two need not agree: the virtualenv outlives the interpreter it was + # built from. Probe the version here as well, so raising the minimum rebuilds + # existing virtualenvs instead of silently reusing one that is too old and + # failing later on an import the old version does not have. try: os.stat(stamp_path) - subprocess.check_call([python, "-c", ""]) + subprocess.check_call( + [python, "-c", f"import sys; sys.exit(sys.hexversion < {MIN_HEXVERSION})"] + ) except Exception as e: print("==> Checking for existing virtualenv", file=sys.stderr) if isinstance(e, FileNotFoundError): print("no existing virtualenv detected", file=sys.stderr) else: - # Usually just an indication that the user has upgraded the system - # Python that the virtualenv is referring to. Not important to - # bubble up the error here. If it's persistent, it'll occur in the - # new virtual env and bubble up when we exec later. + # Either the interpreter is gone, usually because the system Python + # it referred to was upgraded, or it predates the minimum version. + # Not important to tell the two apart or to bubble the error up: a + # problem that survives the rebuild surfaces when we exec later. print( - "warning: existing virtualenv is unable to execute python; will recreate", + f"warning: existing virtualenv is missing python or older than v{MIN_VERSION}; will recreate", file=sys.stderr ) - logger.info("python exec error: %s", e) + logger.info("python probe error: %s", e) print(f"==> Initializing virtualenv in {venv_dir}", file=sys.stderr) try: - subprocess.check_call(["uv", "venv", venv_dir]) + # Clear the path first. Reaching this point means the virtualenv is + # absent or unusable, so whatever occupies it should go, and removing + # it here rather than asking `uv` to keeps the behavior the same on + # every `uv` version. The flags that would express this, `--clear` and + # `--force`, disagree about a path that is not a virtualenv, and + # `--force` does not exist before uv 0.11.17. + shutil.rmtree(venv_dir, ignore_errors=True) + # Build from the interpreter running this script, the one the version + # check above validated. Left to itself, `uv` resolves an interpreter + # by its own preference order, which favors uv-managed installs and so + # can pick a Python older than the system one and older than the check + # accepts. + subprocess.check_call(["uv", "venv", "--python", sys.executable, venv_dir]) except FileNotFoundError: # Install Python into the virtualenv via a symlink rather than copying, # except on Windows. This matches the behavior of the `python -m venv` diff --git a/ci/builder/requirements.txt b/ci/builder/requirements.txt index cfd56bc5696f4..3c12816d86813 100644 --- a/ci/builder/requirements.txt +++ b/ci/builder/requirements.txt @@ -61,7 +61,7 @@ pytest-split==0.11.0 pyyaml==6.0.3 requests==2.34.2 ruamel.yaml==0.18.17 -ruff==0.0.292 +ruff==0.16.5 # NOTE scipy 1.15 is required for Python 3.10 compatibility scipy==1.15.3; python_version < "3.11" scipy==1.17.1; python_version >= "3.11" diff --git a/ci/cleanup/aws.py b/ci/cleanup/aws.py index 9665fdfc76825..301efe073eba8 100644 --- a/ci/cleanup/aws.py +++ b/ci/cleanup/aws.py @@ -7,7 +7,7 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import PurePosixPath from typing import Any from urllib.parse import unquote, urlparse @@ -29,7 +29,7 @@ def clean_up_kinesis() -> None: continue desc = client.describe_stream(StreamName=stream) created_at = desc["StreamDescription"]["StreamCreationTimestamp"] - age = datetime.now(timezone.utc) - created_at + age = datetime.now(UTC) - created_at if age <= MAX_AGE: print(f"Skipping stream {stream} whose age is beneath threshold") continue @@ -45,7 +45,7 @@ def clean_up_s3() -> None: if not desc["Name"].startswith("testdrive"): print("Skipping non-testdrive bucket {}".format(desc["Name"])) continue - age = datetime.now(timezone.utc) - desc["CreationDate"] + age = datetime.now(UTC) - desc["CreationDate"] if age <= MAX_AGE: print( "Skipping bucket {} whose age is beneath threshold".format(desc["Name"]) @@ -76,9 +76,7 @@ def clean_up_sqs() -> None: QueueUrl=queue, AttributeNames=["All"] ) created_at = int(attributes["Attributes"]["CreatedTimestamp"]) - age = datetime.now(timezone.utc) - datetime.fromtimestamp( - created_at, timezone.utc - ) + age = datetime.now(UTC) - datetime.fromtimestamp(created_at, UTC) if age <= MAX_AGE: print(f"Skipping queue {name} whose age is beneath threshold") continue diff --git a/ci/deploy/pypi.py b/ci/deploy/pypi.py index fd4ec77207c6d..a6048a0b73bda 100644 --- a/ci/deploy/pypi.py +++ b/ci/deploy/pypi.py @@ -7,7 +7,6 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -import distutils.core # pyright: ignore import os import sys import tarfile @@ -16,6 +15,7 @@ from pathlib import Path from typing import Literal +import distutils.core # pyright: ignore import requests from materialize import spawn diff --git a/ci/load/periodic.py b/ci/load/periodic.py index 4264d863a560f..489a390aa8b90 100644 --- a/ci/load/periodic.py +++ b/ci/load/periodic.py @@ -31,7 +31,7 @@ def main() -> None: now = datetime.datetime.utcnow() scratch.launch_cluster( [desc], - nonce=now.replace(tzinfo=datetime.timezone.utc).isoformat(), + nonce=now.replace(tzinfo=datetime.UTC).isoformat(), # Keep alive for at least a day. delete_after=datetime.datetime.utcnow() + datetime.timedelta(days=1), ) diff --git a/ci/mkpipeline.py b/ci/mkpipeline.py index 909a2d7de7ebd..4a064537d4152 100644 --- a/ci/mkpipeline.py +++ b/ci/mkpipeline.py @@ -31,7 +31,7 @@ from collections import OrderedDict from collections.abc import Iterable, Iterator from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -669,9 +669,9 @@ def switch_jobs_to_aws(pipeline: Any, priority: int) -> None: runnable = job.get("runnable_at") if not runnable or job.get("started_at"): continue - if datetime.now(timezone.utc) - datetime.fromisoformat( - runnable - ) < timedelta(minutes=20): + if datetime.now(UTC) - datetime.fromisoformat(runnable) < timedelta( + minutes=20 + ): continue print( diff --git a/ci/test/lint-main/checks/check-python-files.sh b/ci/test/lint-main/checks/check-python-files.sh index 4896138710b42..02dbb6a2b8bb4 100755 --- a/ci/test/lint-main/checks/check-python-files.sh +++ b/ci/test/lint-main/checks/check-python-files.sh @@ -28,9 +28,9 @@ if [[ ! "${MZDEV_NO_PYTHON:-}" ]]; then try xargs npx --yes "pyright@$pyright_version" --warnings --threads 4 < "$python_files_list" - try xargs bin/pyactivate -m ruff < "$python_files_list" + try xargs bin/pyactivate -m ruff check < "$python_files_list" # We need to maintain compatibility with older Python versions for this - try xargs bin/pyactivate -m ruff --target-version=py38 < "$dbt_files_list" + try xargs bin/pyactivate -m ruff check --target-version=py38 < "$dbt_files_list" fi try_status_report diff --git a/ci/test/lint-main/checks/check-python-version.sh b/ci/test/lint-main/checks/check-python-version.sh index 769008106c93f..fe48de69d2b10 100755 --- a/ci/test/lint-main/checks/check-python-version.sh +++ b/ci/test/lint-main/checks/check-python-version.sh @@ -9,7 +9,7 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. # -# check-python-version.sh — make sure Python 3.10 keeps working +# check-python-version.sh — make sure the minimum supported Python keeps working set -euo pipefail @@ -17,6 +17,9 @@ cd "$(dirname "$0")/../../../.." . misc/shlib/shlib.bash +# Keep in sync with the version check in `bin/pyactivate`. +PYTHON_MIN_VERSION=3.13 + if [[ ! "${MZDEV_NO_PYTHON:-}" ]]; then if ! uv --version >/dev/null 2>/dev/null; then echo "lint: uv is not installed" @@ -24,19 +27,19 @@ if [[ ! "${MZDEV_NO_PYTHON:-}" ]]; then exit 1 fi - py310_venv="$(mktemp -d)/venv-py310" - trap 'rm -rf "$py310_venv"' EXIT + py_min_venv="$(mktemp -d)/venv-py-min" + trap 'rm -rf "$py_min_venv"' EXIT - try uv venv --python 3.10 "$py310_venv" - try uv pip compile --python-version 3.10 ci/builder/requirements.txt - try uv pip install --python "$py310_venv/bin/python" --requirement ci/builder/requirements.txt + try uv venv --python "$PYTHON_MIN_VERSION" "$py_min_venv" + try uv pip compile --python-version "$PYTHON_MIN_VERSION" ci/builder/requirements.txt + try uv pip install --python "$py_min_venv/bin/python" --requirement ci/builder/requirements.txt # Wrap the compileall (the actual work) in `try`, not git_files, and keep it # out of a pipeline: `try` in a pipeline runs in a subshell and loses its # accounting. Guard against empty input too, otherwise compileall with no # file arguments compiles all of sys.path instead of the repo. py_files=$(git_files '*.py') if [[ -n "$py_files" ]]; then - try xargs "$py310_venv/bin/python" -m compileall -q <<< "$py_files" + try xargs "$py_min_venv/bin/python" -m compileall -q <<< "$py_files" fi fi diff --git a/doc/developer/guide.md b/doc/developer/guide.md index 330361d63aeb2..76e1151c9b571 100644 --- a/doc/developer/guide.md +++ b/doc/developer/guide.md @@ -140,10 +140,12 @@ environment. Most of this should be taken care of by the `bin/pyactivate` script, which constructs a local virtual environment and keeps necessary dependencies up to date. -We support, as a minimum version, the default Python provided in the [most -recent Ubuntu LTS release](https://wiki.ubuntu.com/Releases). As of January 2026 -this is Python 3.12, provided in Ubuntu 24.04 "Noble Numbat". Earlier versions may -work but are not supported. Our recommended installation methods are: +We support Python 3.13 as a minimum version, which is what our CI builder image +provides and therefore the oldest version anything is tested against. It is also +no newer than the default Python in the [most recent Ubuntu LTS +release](https://wiki.ubuntu.com/Releases), which ships 3.14 as of Ubuntu 26.04 +"Resolute Raccoon", so a current LTS satisfies it. Earlier versions may work but +are not supported. Our recommended installation methods are: - macOS: [Homebrew](https://brew.sh) + [uv](https://docs.astral.sh/uv/) 1. `brew install uv` diff --git a/misc/mcp-materialize/mcp_materialize/mz_client.py b/misc/mcp-materialize/mcp_materialize/mz_client.py index a5d1927df09d3..243c637b82162 100644 --- a/misc/mcp-materialize/mcp_materialize/mz_client.py +++ b/misc/mcp-materialize/mcp_materialize/mz_client.py @@ -95,7 +95,7 @@ async def __aexit__(self, exc_type, exc, tb) -> None: self._bg_task.cancel() try: await asyncio.wait_for(self._bg_task, timeout=5.0) - except (asyncio.CancelledError, asyncio.TimeoutError): + except (TimeoutError, asyncio.CancelledError): pass async def _subscribe(self) -> None: diff --git a/misc/python/materialize/buildkite.py b/misc/python/materialize/buildkite.py index cf29330ced23d..7868d8d8793e0 100644 --- a/misc/python/materialize/buildkite.py +++ b/misc/python/materialize/buildkite.py @@ -13,14 +13,12 @@ from collections.abc import Callable from enum import Enum, auto from pathlib import Path -from typing import Any, TypeVar +from typing import Any import yaml from materialize import git, spawn, ui -T = TypeVar("T") - class BuildkiteEnvVar(Enum): # environment @@ -229,7 +227,7 @@ def notify_qa_team_about_failure(failure: str) -> None: ) -def shard_list(items: list[T], to_identifier: Callable[[T], str]) -> list[T]: +def shard_list[T](items: list[T], to_identifier: Callable[[T], str]) -> list[T]: if len(items) == 0: return [] diff --git a/misc/python/materialize/cli/fmt.py b/misc/python/materialize/cli/fmt.py index 9a12b7f631123..29b891b7abb91 100644 --- a/misc/python/materialize/cli/fmt.py +++ b/misc/python/materialize/cli/fmt.py @@ -137,7 +137,7 @@ def _ruff_cmd(*, check: bool) -> list[str]: return [ "bash", "-c", - f'. misc/shlib/shlib.bash && git_files "*.py" | grep -v "^misc/dbt-materialize/" | xargs bin/pyactivate -m ruff{fix}', + f'. misc/shlib/shlib.bash && git_files "*.py" | grep -v "^misc/dbt-materialize/" | xargs bin/pyactivate -m ruff check{fix}', ] @@ -146,7 +146,7 @@ def _ruff_dbt_cmd(*, check: bool) -> list[str]: return [ "bash", "-c", - f'. misc/shlib/shlib.bash && git_files "misc/dbt-materialize/*.py" | xargs bin/pyactivate -m ruff --target-version=py38{fix}', + f'. misc/shlib/shlib.bash && git_files "misc/dbt-materialize/*.py" | xargs bin/pyactivate -m ruff check --target-version=py38{fix}', ] diff --git a/misc/python/materialize/cli/mz_workload_anonymize_test.py b/misc/python/materialize/cli/mz_workload_anonymize_test.py index a5647e23b8c04..d4e0a867a2c81 100644 --- a/misc/python/materialize/cli/mz_workload_anonymize_test.py +++ b/misc/python/materialize/cli/mz_workload_anonymize_test.py @@ -465,7 +465,7 @@ def cross_schema_cdc_workload() -> dict[str, Any]: schema. The child key is built in pass 1 while processing the source's schema (`aaa_src`), before the child's schema (`zzz_upstream`) is mapped, so a stale key leaks the original schema name unless rebuilt in pass 2.""" - empty = lambda: { # noqa: E731 + empty = lambda: { "tables": {}, "views": {}, "materialized_views": {}, diff --git a/misc/python/materialize/cli/mz_workload_capture.py b/misc/python/materialize/cli/mz_workload_capture.py index 5a6ba3319e1bf..021c31da38edc 100644 --- a/misc/python/materialize/cli/mz_workload_capture.py +++ b/misc/python/materialize/cli/mz_workload_capture.py @@ -12,13 +12,12 @@ import threading import time from contextlib import contextmanager -from datetime import datetime, timezone -from typing import Any +from datetime import UTC, datetime +from typing import Any, LiteralString import psycopg import yaml from psycopg.sql import SQL, Composable, Composed, Identifier, Literal -from typing_extensions import LiteralString @contextmanager @@ -195,7 +194,7 @@ def attach_source_statistics_internal( ) WITH (PROGRESS) AS OF AT LEAST TIMESTAMP {}""").format( Literal(source_id), - Literal(datetime.fromtimestamp(start_time, tz=timezone.utc)), + Literal(datetime.fromtimestamp(start_time, tz=UTC)), ) first_timestamp = None for ( @@ -213,7 +212,7 @@ def attach_source_statistics_internal( continue if "bytes_total" not in source: first_timestamp = datetime.fromtimestamp( - int(mz_timestamp / 1000), tz=timezone.utc + int(mz_timestamp / 1000), tz=UTC ) source["bytes_total"] = int(bytes_received) source["messages_total"] = int(messages_received) @@ -221,7 +220,7 @@ def attach_source_statistics_internal( assert "bytes_total" in source and "messages_total" in source assert first_timestamp duration = ( - datetime.fromtimestamp(int(mz_timestamp / 1000), tz=timezone.utc) + datetime.fromtimestamp(int(mz_timestamp / 1000), tz=UTC) - first_timestamp ).total_seconds() if duration: @@ -287,7 +286,7 @@ def main() -> int: "--output", type=str, help="Path to write the workload.yml, - for stdout", - default=f"workload_{datetime.now(timezone.utc).strftime('%Y-%m-%dT%H-%M-%S')}.yml", + default=f"workload_{datetime.now(UTC).strftime('%Y-%m-%dT%H-%M-%S')}.yml", ) parser.add_argument( "--time", @@ -644,7 +643,7 @@ def main() -> int: conn, SQL( "SELECT sql, cluster_name, database_name, search_path, statement_type, finished_status, params, transaction_isolation, session_id, transaction_id, began_at, finished_at - began_at, result_size FROM mz_internal.mz_recent_activity_log WHERE began_at > {} ORDER BY began_at ASC" - ).format(Literal(datetime.fromtimestamp(start_time, tz=timezone.utc))), + ).format(Literal(datetime.fromtimestamp(start_time, tz=UTC))), ): workload["queries"].append( { diff --git a/misc/python/materialize/cli/mzcompose.py b/misc/python/materialize/cli/mzcompose.py index c438e09ee9e05..2530fad163593 100644 --- a/misc/python/materialize/cli/mzcompose.py +++ b/misc/python/materialize/cli/mzcompose.py @@ -167,7 +167,7 @@ def main(argv: list[str]) -> None: # Convert to a list of the members: shtab renders them via str(), and # argparse still accepts the type-converted member, which converting to # name strings would not (the member never compares equal to its name). - for action in parser._actions: # noqa: SLF001 + for action in parser._actions: if isinstance(action.choices, type) and issubclass(action.choices, enum.Enum): action.choices = list(action.choices) diff --git a/misc/python/materialize/cli/orchestratord.py b/misc/python/materialize/cli/orchestratord.py index 9747c782345f3..ac39a46066699 100644 --- a/misc/python/materialize/cli/orchestratord.py +++ b/misc/python/materialize/cli/orchestratord.py @@ -477,7 +477,7 @@ def kubectl(*args: str, cluster: str, **subprocess_args): T = TypeVar("T") -def retry( +def retry[T]( f: Callable[[], T], max_attempts: int = 60, sleep_secs: int = 1, diff --git a/misc/python/materialize/cli/scratch/claude.py b/misc/python/materialize/cli/scratch/claude.py index 29ec4e88d3c9f..1e7e078c3c713 100644 --- a/misc/python/materialize/cli/scratch/claude.py +++ b/misc/python/materialize/cli/scratch/claude.py @@ -114,7 +114,7 @@ def run(args: argparse.Namespace) -> None: instances = launch_cluster( descs, extra_tags=extra_tags, - delete_after=datetime.datetime.now(datetime.timezone.utc) + max_age, + delete_after=datetime.datetime.now(datetime.UTC) + max_age, ) print("Launched:") diff --git a/misc/python/materialize/cli/scratch/create.py b/misc/python/materialize/cli/scratch/create.py index c2775261f5180..28b77d88077f3 100644 --- a/misc/python/materialize/cli/scratch/create.py +++ b/misc/python/materialize/cli/scratch/create.py @@ -156,8 +156,7 @@ def configure_parser(parser: argparse.ArgumentParser) -> None: "--instance-profile", type=str, default=DEFAULT_INSTANCE_PROFILE_NAME, - help="EC2 instance profile / IAM role. Defaults to `%s`." - % DEFAULT_INSTANCE_PROFILE_NAME, + help=f"EC2 instance profile / IAM role. Defaults to `{DEFAULT_INSTANCE_PROFILE_NAME}`.", ) parser.add_argument("--output-format", choices=["table", "csv"], default="table") parser.add_argument( @@ -239,7 +238,7 @@ def run(args: argparse.Namespace) -> None: security_group_name=args.security_group_name, instance_profile=args.instance_profile, extra_tags=extra_tags, - delete_after=datetime.datetime.now(datetime.timezone.utc) + max_age, + delete_after=datetime.datetime.now(datetime.UTC) + max_age, git_rev=args.git_rev, extra_env={}, ) diff --git a/misc/python/materialize/cli/scratch/go.py b/misc/python/materialize/cli/scratch/go.py index acaf0fac950b4..625523fed6712 100644 --- a/misc/python/materialize/cli/scratch/go.py +++ b/misc/python/materialize/cli/scratch/go.py @@ -99,7 +99,7 @@ def run(args: argparse.Namespace) -> None: instances = launch_cluster( descs, extra_tags=extra_tags, - delete_after=datetime.datetime.now(datetime.timezone.utc) + max_age, + delete_after=datetime.datetime.now(datetime.UTC) + max_age, ) print("Launched:") diff --git a/misc/python/materialize/feature_benchmark/benchmark_result.py b/misc/python/materialize/feature_benchmark/benchmark_result.py index 3b2bffe545166..cac6ae44eaeca 100644 --- a/misc/python/materialize/feature_benchmark/benchmark_result.py +++ b/misc/python/materialize/feature_benchmark/benchmark_result.py @@ -9,7 +9,7 @@ from __future__ import annotations -from typing import Generic, TypeVar +from typing import TypeVar from materialize.feature_benchmark.measurement import MeasurementType, MeasurementUnit from materialize.feature_benchmark.scenario import Scenario @@ -56,7 +56,7 @@ def get_metric_by_measurement_type( return None -class BenchmarkScenarioMetric(Generic[T]): +class BenchmarkScenarioMetric[T]: def __init__( self, scenario_class: type[Scenario], measurement_type: MeasurementType ) -> None: diff --git a/misc/python/materialize/feature_benchmark/benchmark_result_evaluator.py b/misc/python/materialize/feature_benchmark/benchmark_result_evaluator.py index 87fcd997a07ae..52698da2c72b3 100644 --- a/misc/python/materialize/feature_benchmark/benchmark_result_evaluator.py +++ b/misc/python/materialize/feature_benchmark/benchmark_result_evaluator.py @@ -9,8 +9,6 @@ from __future__ import annotations -from typing import Generic, TypeVar - from materialize.feature_benchmark.benchmark_result import BenchmarkScenarioMetric from materialize.feature_benchmark.measurement import MeasurementType from materialize.feature_benchmark.scenario import Scenario @@ -20,10 +18,8 @@ with_conditional_formatting, ) -T = TypeVar("T") - -class BenchmarkResultEvaluator(Generic[T]): +class BenchmarkResultEvaluator[T]: def ratio(self, metric: BenchmarkScenarioMetric) -> float | None: raise RuntimeError diff --git a/misc/python/materialize/feature_benchmark/report.py b/misc/python/materialize/feature_benchmark/report.py index cb691aa4db9b9..0760d28e5a027 100644 --- a/misc/python/materialize/feature_benchmark/report.py +++ b/misc/python/materialize/feature_benchmark/report.py @@ -8,7 +8,7 @@ # by the Apache License, Version 2.0. from statistics import mean, variance -from typing import Generic, TypeVar +from typing import TypeVar from materialize.feature_benchmark.benchmark_result import BenchmarkScenarioResult from materialize.feature_benchmark.benchmark_result_evaluator import ( @@ -23,7 +23,7 @@ T = TypeVar("T", bound=int | float) -class ReportMeasurement(Generic[T]): +class ReportMeasurement[T: int | float]: result: T | None min: T | None max: T | None diff --git a/misc/python/materialize/git.py b/misc/python/materialize/git.py index 74d1cbb257dec..2d0d3cecd4948 100644 --- a/misc/python/materialize/git.py +++ b/misc/python/materialize/git.py @@ -124,7 +124,7 @@ def expand_globs(root: Path, *specs: Path | str) -> set[str]: return set(f for f in (diff_files + ls_files).split("\0") if f.strip() != "") -def get_version_tags( +def get_version_tags[VERSION_TYPE: TypedVersionBase]( *, version_type: type[VERSION_TYPE], newest_first: bool = True, @@ -157,7 +157,7 @@ def get_version_tags( return sorted(tags, reverse=newest_first) -def get_latest_version( +def get_latest_version[VERSION_TYPE: TypedVersionBase]( version_type: type[VERSION_TYPE], excluded_versions: set[VERSION_TYPE] | None = None, current_version: VERSION_TYPE | None = None, @@ -377,7 +377,9 @@ def contains_commit( return is_ancestor(commit_sha, target) -def get_tagged_release_version(version_type: type[VERSION_TYPE]) -> VERSION_TYPE | None: +def get_tagged_release_version[VERSION_TYPE: TypedVersionBase]( + version_type: type[VERSION_TYPE], +) -> VERSION_TYPE | None: """ This returns the release version if exactly this commit is tagged. If multiple release versions are present, the highest one will be returned. diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 1be7ba3c7ec07..b9b36d9c7216f 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -891,9 +891,7 @@ def _check_tcp( spawn.capture(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: ui.log_in_automation( - "wait-for-tcp ({}{}:{}): error running {}: {}, stdout:\n{}\nstderr:\n{}".format( - kind, host, port, ui.shell_quote(cmd), e, e.stdout, e.stderr - ) + f"wait-for-tcp ({kind}{host}:{port}): error running {ui.shell_quote(cmd)}: {e}, stdout:\n{e.stdout}\nstderr:\n{e.stderr}" ) raise return cmd diff --git a/misc/python/materialize/mzcompose/service.py b/misc/python/materialize/mzcompose/service.py index d6c1b1d3f8ca4..6052181bb1af4 100644 --- a/misc/python/materialize/mzcompose/service.py +++ b/misc/python/materialize/mzcompose/service.py @@ -203,4 +203,4 @@ class Service: def __init__(self, name: str, config: ServiceConfig) -> None: self.name = name self.config = config - self.companions: list["Service"] = [] + self.companions: list[Service] = [] diff --git a/misc/python/materialize/mzexplore/common.py b/misc/python/materialize/mzexplore/common.py index 37ee3fcf06ba5..90a4d164d7f8a 100644 --- a/misc/python/materialize/mzexplore/common.py +++ b/misc/python/materialize/mzexplore/common.py @@ -9,7 +9,7 @@ import re from dataclasses import dataclass, replace -from enum import Enum +from enum import Enum, StrEnum from importlib import resources from pathlib import Path from typing import cast @@ -45,7 +45,7 @@ def ext(self): return "txt" -class ExplainStage(str, Enum): +class ExplainStage(StrEnum): RAW_PLAN = "RAW PLAN" DECORRELATED_PLAN = "DECORRELATED PLAN" LOCAL_PLAN = "LOCALLY OPTIMIZED PLAN" @@ -140,7 +140,7 @@ def convert(self, value, param, ctx): # type: ignore raise ValueError(f"Bad explain option: {value}: {e!r}") from e -class ItemType(str, Enum): +class ItemType(StrEnum): CONNECTION = "connection" FUNCTION = "function" INDEX = "index" diff --git a/misc/python/materialize/mzexplore/sql.py b/misc/python/materialize/mzexplore/sql.py index cdf8be2234aae..99d012da6feb2 100644 --- a/misc/python/materialize/mzexplore/sql.py +++ b/misc/python/materialize/mzexplore/sql.py @@ -23,7 +23,7 @@ from materialize.mzexplore.common import resource_path -DictGenerator = Generator[dict[Any, Any], None, None] +DictGenerator = Generator[dict[Any, Any]] class Database: @@ -125,9 +125,7 @@ def arrangement_sizes(self, id: str) -> DictGenerator: @contextlib.contextmanager -def update_environment( - db: Database, env: dict[str, str] -) -> Generator[Database, None, None]: +def update_environment(db: Database, env: dict[str, str]) -> Generator[Database]: original = dict() for e in db.query_all("SHOW ALL"): key, old_value = e["name"], e["setting"] diff --git a/misc/python/materialize/output_consistency/selection/column_selection.py b/misc/python/materialize/output_consistency/selection/column_selection.py index 90c071e9f6f33..c0ce6871d0c95 100644 --- a/misc/python/materialize/output_consistency/selection/column_selection.py +++ b/misc/python/materialize/output_consistency/selection/column_selection.py @@ -7,7 +7,7 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -from typing import Generic, TypeVar +from typing import TypeVar from materialize.output_consistency.data_value.source_column_identifier import ( SourceColumnIdentifier, @@ -16,7 +16,7 @@ T = TypeVar("T") -class SelectionByKey(Generic[T]): +class SelectionByKey[T]: def __init__(self, keys: set[T] | None = None): self.keys = keys diff --git a/misc/python/materialize/scalability/executor/benchmark_executor.py b/misc/python/materialize/scalability/executor/benchmark_executor.py index 2f6146a747e91..6c0f779e339a0 100644 --- a/misc/python/materialize/scalability/executor/benchmark_executor.py +++ b/misc/python/materialize/scalability/executor/benchmark_executor.py @@ -30,10 +30,10 @@ from materialize.scalability.schema.schema import Schema from materialize.scalability.workload.workload import Workload, WorkloadWithContext from materialize.scalability.workload.workload_markers import ConnectionWorkload -from materialize.scalability.workload.workloads.connection_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.ddl_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.dml_dql_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.self_test_workloads import * # noqa: F401 F403 +from materialize.scalability.workload.workloads.connection_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.ddl_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.dml_dql_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.self_test_workloads import * # noqa: F403 # number of retries in addition to the first run MAX_RETRIES_ON_REGRESSION = 2 diff --git a/misc/python/materialize/scratch.py b/misc/python/materialize/scratch.py index 3e824165e1083..649e5b03e5b9f 100644 --- a/misc/python/materialize/scratch.py +++ b/misc/python/materialize/scratch.py @@ -420,7 +420,7 @@ def launch( tags.setdefault("team", "engineering") tags.setdefault( "deleteAfter", - delete_after.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + delete_after.astimezone(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), ) ec2 = boto3.client("ec2") @@ -708,7 +708,7 @@ def is_old(i: InstanceTypeDef) -> bool: if delete_after is None: return False delete_after = float(delete_after) - return datetime.datetime.now(datetime.timezone.utc).timestamp() > delete_after + return datetime.datetime.now(datetime.UTC).timestamp() > delete_after return [ i diff --git a/misc/python/materialize/spawn.py b/misc/python/materialize/spawn.py index 7f384f97a1e88..db1cdf17b7fe2 100644 --- a/misc/python/materialize/spawn.py +++ b/misc/python/materialize/spawn.py @@ -21,7 +21,7 @@ import time from collections.abc import Callable, Sequence from pathlib import Path -from typing import IO, TypeVar +from typing import IO from materialize import ui @@ -144,10 +144,7 @@ def run_and_get_return_code( return e.returncode -T = TypeVar("T") # Generic type variable - - -def run_with_retries(fn: Callable[[], T], max_duration: int = 60) -> T: +def run_with_retries[T](fn: Callable[[], T], max_duration: int = 60) -> T: """Retry a function until it doesn't raise a `CalledProcessError`, uses exponential backoff until `max_duration` is reached.""" for retry in range(math.ceil(math.log2(max_duration))): diff --git a/misc/python/materialize/ui.py b/misc/python/materialize/ui.py index 40d68e7f83af9..6e0eb8b3adbc2 100644 --- a/misc/python/materialize/ui.py +++ b/misc/python/materialize/ui.py @@ -82,7 +82,7 @@ def progress(msg: str = "", prefix: str | None = None, *, finish: bool = False) print(msg, file=sys.stderr, flush=True, end=end) -def timeout_loop(timeout: int, tick: float = 1.0) -> Generator[float, None, None]: +def timeout_loop(timeout: int, tick: float = 1.0) -> Generator[float]: """Loop until timeout, optionally sleeping until tick Always iterates at least once @@ -105,9 +105,7 @@ def timeout_loop(timeout: int, tick: float = 1.0) -> Generator[float, None, None time.sleep(tick - (after - before)) -async def async_timeout_loop( - timeout: int, tick: float = 1.0 -) -> AsyncGenerator[float, None]: +async def async_timeout_loop(timeout: int, tick: float = 1.0) -> AsyncGenerator[float]: """Loop until timeout, asynchronously sleeping until tick Always iterates at least once diff --git a/misc/python/materialize/util.py b/misc/python/materialize/util.py index 7af15417ce168..41dd881c4bb1b 100644 --- a/misc/python/materialize/util.py +++ b/misc/python/materialize/util.py @@ -23,7 +23,7 @@ from enum import Enum from pathlib import Path from threading import Thread -from typing import Protocol, TypeVar +from typing import Protocol from urllib.parse import parse_qs, quote, unquote, urlparse import psycopg @@ -37,10 +37,7 @@ def nonce(digits: int) -> str: return "".join(random.choice("0123456789abcdef") for _ in range(digits)) -T = TypeVar("T") - - -def all_subclasses(cls: type[T]) -> set[type[T]]: +def all_subclasses[T](cls: type[T]) -> set[type[T]]: """Returns a recursive set of all subclasses of a class""" sc = cls.__subclasses__() return set(sc).union([subclass for c in sc for subclass in all_subclasses(c)]) @@ -126,10 +123,7 @@ class HasName(Protocol): name: str -U = TypeVar("U", bound=HasName) - - -def selected_by_name(selected: list[str], objs: list[U]) -> Iterator[U]: +def selected_by_name[U: HasName](selected: list[str], objs: list[U]) -> Iterator[U]: for name in selected: for obj in objs: if obj.name == name: diff --git a/misc/python/materialize/workload_replay/replay.py b/misc/python/materialize/workload_replay/replay.py index 8918978b087a4..4dab70e69369d 100644 --- a/misc/python/materialize/workload_replay/replay.py +++ b/misc/python/materialize/workload_replay/replay.py @@ -218,7 +218,7 @@ def submit_query(query: dict[str, Any]) -> None: i = 0 while True: i += 1 - replay_start = datetime.datetime.now(datetime.timezone.utc) + replay_start = datetime.datetime.now(datetime.UTC) for query in workload["queries"]: if stop_event.is_set(): @@ -244,7 +244,7 @@ def submit_query(query: dict[str, Any]) -> None: offset = (query["began_at"] - start) / factor_queries scheduled = replay_start + offset sleep_seconds = ( - scheduled - datetime.datetime.now(datetime.timezone.utc) + scheduled - datetime.datetime.now(datetime.UTC) ).total_seconds() if sleep_seconds > 0: diff --git a/misc/python/materialize/zippy/scenarios.py b/misc/python/materialize/zippy/scenarios.py index 2351b43120035..7107fac27c595 100644 --- a/misc/python/materialize/zippy/scenarios.py +++ b/misc/python/materialize/zippy/scenarios.py @@ -18,7 +18,7 @@ from materialize.zippy.blob_store_actions import BlobStoreRestart, BlobStoreStart from materialize.zippy.crdb_actions import CockroachRestart, CockroachStart from materialize.zippy.debezium_actions import CreateDebeziumSource, DebeziumStart -from materialize.zippy.framework import ActionOrFactory # noqa +from materialize.zippy.framework import ActionOrFactory from materialize.zippy.kafka_actions import ( CreateTopicParameterized, Ingest, diff --git a/pyproject.toml b/pyproject.toml index 2821f2ca1697f..dbc9432866b7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,25 +8,29 @@ # by the Apache License, Version 2.0. [tool.black] -target_version = ["py310"] +target_version = ["py313"] # DEFAULT_EXCLUDES from https://github.com/psf/black/blob/main/src/black/const.py but without "build" directory since we use it in our source code. Instead exclude target and target-xcompile exclude = "(\\.direnv|\\.eggs|\\.git|\\.hg|\\.ipynb_checkpoints|\\.mypy_cache|\\.nox|\\.pytest_cache|\\.ruff_cache|\\.tox|\\.svn|\\.venv|\\.vscode|__pypackages__|_build|buck-out|dist|venv|target|target-xcompile|\\.terraform)" [tool.ruff] -target-version = "py310" -select = [ - "F", - "I", - "UP", # e.g. PEP585 (Python 3.10+) type annotations - "E711", # comparisons to none -] +# Keep in sync with the minimum version enforced by `bin/pyactivate`. +target-version = "py313" extend-exclude = [ "venv", "target", # This dbt adapter may be published elsewhere, so doesn't follow the same compatibility rules as our internal code. "misc/dbt-materialize", ] -[tool.ruff.isort] +[tool.ruff.lint] +select = [ + "F", + "I", + "UP", # e.g. PEP585 built-in generic type annotations + "E711", # comparisons to none + "RUF100", # `noqa` that suppresses nothing, including for unselected rules +] + +[tool.ruff.lint.isort] known-first-party = ["materialize"] [tool.pyright] diff --git a/test/canary-load/mzcompose.py b/test/canary-load/mzcompose.py index 18cd5321514aa..d947923d23e4d 100644 --- a/test/canary-load/mzcompose.py +++ b/test/canary-load/mzcompose.py @@ -90,7 +90,7 @@ def take_pending(self) -> list[str]: def _run(self) -> None: while not self._stop.is_set(): try: - now = datetime.datetime.now(tz=datetime.timezone.utc) + now = datetime.datetime.now(tz=datetime.UTC) ts = now.isoformat() payload = { "event_type": "canary_heartbeat", @@ -201,9 +201,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: ) as e: error_msg_str = str(e) if is_connection_error(error_msg_str): - now = datetime.datetime.now( - tz=datetime.timezone.utc - ).isoformat() + now = datetime.datetime.now(tz=datetime.UTC).isoformat() connection_failures.append((now, error_msg_str)) print(f"Connection failure at {now}: {e}; retrying") else: @@ -215,9 +213,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: if "Non-positive multiplicity in DistinctBy" in error.message: continue if is_connection_error(error.message): - now = datetime.datetime.now( - tz=datetime.timezone.utc - ).isoformat() + now = datetime.datetime.now(tz=datetime.UTC).isoformat() connection_failures.append((now, error.message)) print( f"Connection failure at {now}: {error.message}; continuing." @@ -234,9 +230,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: if "Non-positive multiplicity in DistinctBy" in msg: continue if is_connection_error(msg): - now = datetime.datetime.now( - tz=datetime.timezone.utc - ).isoformat() + now = datetime.datetime.now(tz=datetime.UTC).isoformat() connection_failures.append((now, msg)) print(f"Connection failure at {now}: {msg}; continuing.") continue @@ -559,8 +553,8 @@ def validate_webhook( latest_ts = max(received_timestamps) latest_dt = datetime.datetime.fromisoformat(latest_ts) if latest_dt.tzinfo is None: - latest_dt = latest_dt.replace(tzinfo=datetime.timezone.utc) - age = datetime.datetime.now(tz=datetime.timezone.utc) - latest_dt + latest_dt = latest_dt.replace(tzinfo=datetime.UTC) + age = datetime.datetime.now(tz=datetime.UTC) - latest_dt assert ( age.total_seconds() < 120 ), f"Webhook ingestion stall: latest event is {age} old (ts={latest_ts}), expected < 120s" diff --git a/test/cloudtest/test_upgrade.py b/test/cloudtest/test_upgrade.py index ba3bf79903bd1..0495be85e9ffe 100644 --- a/test/cloudtest/test_upgrade.py +++ b/test/cloudtest/test_upgrade.py @@ -14,7 +14,7 @@ from materialize import buildkite from materialize.checks.actions import Action, Initialize, Manipulate, Validate -from materialize.checks.all_checks import * # noqa: F401 F403 +from materialize.checks.all_checks import * # noqa: F403 from materialize.checks.all_checks.alter_connection import ( AlterConnectionHost, AlterConnectionToNonSsh, diff --git a/test/clusterd-test-driver/run-local.py b/test/clusterd-test-driver/run-local.py index 73ddb091d72df..63518c134993d 100644 --- a/test/clusterd-test-driver/run-local.py +++ b/test/clusterd-test-driver/run-local.py @@ -258,7 +258,7 @@ def main() -> int: ensure_cockroach() cargo_build() - launched: "subprocess.Popen[bytes] | None" = None + launched: subprocess.Popen[bytes] | None = None clusterd_pid: int | None = None try: if RUN_CLUSTERD: diff --git a/test/data-ingest/mzcompose.py b/test/data-ingest/mzcompose.py index a0fc19f6081f5..91371057faf1f 100644 --- a/test/data-ingest/mzcompose.py +++ b/test/data-ingest/mzcompose.py @@ -21,7 +21,7 @@ KafkaExecutor, MySqlExecutor, ) -from materialize.data_ingest.workload import * # noqa: F401 F403 +from materialize.data_ingest.workload import * # noqa: F403 from materialize.data_ingest.workload import WORKLOADS, execute_workload from materialize.mzcompose import get_default_system_parameters from materialize.mzcompose.composition import Composition, WorkflowArgumentParser diff --git a/test/feature-benchmark/mzcompose.py b/test/feature-benchmark/mzcompose.py index 88fd9d8767477..3143ec7322308 100644 --- a/test/feature-benchmark/mzcompose.py +++ b/test/feature-benchmark/mzcompose.py @@ -72,17 +72,17 @@ from materialize.feature_benchmark.executor import Docker from materialize.feature_benchmark.filter import Filter, FilterFirst, NoFilter from materialize.feature_benchmark.measurement import MeasurementType -from materialize.feature_benchmark.scenarios.benchmark_main import * # noqa: F401 F403 +from materialize.feature_benchmark.scenarios.benchmark_main import * # noqa: F403 from materialize.feature_benchmark.scenarios.benchmark_main import ( Scenario, ) -from materialize.feature_benchmark.scenarios.concurrency import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.customer import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.optbench import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.scale import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.skew import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.subscribe import * # noqa: F401 F403 -from materialize.feature_benchmark.scenarios.temporal import * # noqa: F401 F403 +from materialize.feature_benchmark.scenarios.concurrency import * # noqa: F403 +from materialize.feature_benchmark.scenarios.customer import * # noqa: F403 +from materialize.feature_benchmark.scenarios.optbench import * # noqa: F403 +from materialize.feature_benchmark.scenarios.scale import * # noqa: F403 +from materialize.feature_benchmark.scenarios.skew import * # noqa: F403 +from materialize.feature_benchmark.scenarios.subscribe import * # noqa: F403 +from materialize.feature_benchmark.scenarios.temporal import * # noqa: F403 from materialize.feature_benchmark.termination import ( NormalDistributionOverlap, ProbForMin, diff --git a/test/gcp/mzcompose.py b/test/gcp/mzcompose.py index b005d6f9b0327..566c8d07a7c01 100644 --- a/test/gcp/mzcompose.py +++ b/test/gcp/mzcompose.py @@ -28,7 +28,7 @@ import urllib.error import urllib.parse import urllib.request -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from typing import Any import fastavro @@ -257,7 +257,7 @@ def _sweep_stale_biglake_namespaces(token: str, project: str, prefix: str) -> No in the namespace name to age out anything older than STALE_NAMESPACE_AGE without an extra metadata round-trip per namespace. """ - today = datetime.now(timezone.utc).date() + today = datetime.now(UTC).date() for ns in _list_biglake_namespaces(token, project, prefix): match = NAMESPACE_RE.match(ns) if not match: @@ -287,7 +287,7 @@ def workflow_default(c: Composition) -> None: # bucket don't collide on table state. The embedded date lets the pre-test # sweep age out namespaces left behind by killed runs. seed = random.getrandbits(32) - today = datetime.now(timezone.utc).strftime(NAMESPACE_DATE_FORMAT) + today = datetime.now(UTC).strftime(NAMESPACE_DATE_FORMAT) namespace = f"{NAMESPACE_PREFIX}_{today}_{seed:08x}" table = "demo_table" # The .td inserts these three rows; verification asserts the table's live diff --git a/test/mcp/mzcompose.py b/test/mcp/mzcompose.py index c4e053dae6a86..b2e5d139f5e25 100644 --- a/test/mcp/mzcompose.py +++ b/test/mcp/mzcompose.py @@ -1023,7 +1023,7 @@ def run(cur: Cursor, sql: str) -> str: """Run `sql`; return ``"ok:"`` or the error message string.""" try: cur.execute(sql.encode()) - except Exception as e: # noqa: BLE001 — the message is the assertion + except Exception as e: return str(e) if cur.description is None: return "ok:0" diff --git a/test/parallel-benchmark/mzcompose.py b/test/parallel-benchmark/mzcompose.py index 1690cb5e53144..1e252c91673ad 100644 --- a/test/parallel-benchmark/mzcompose.py +++ b/test/parallel-benchmark/mzcompose.py @@ -60,7 +60,7 @@ SQLiteStore, State, ) -from materialize.parallel_benchmark.scenarios import * # noqa: F401 F403 +from materialize.parallel_benchmark.scenarios import * # noqa: F403 from materialize.test_analytics.config.test_analytics_db_config import ( create_test_analytics_config, ) diff --git a/test/platform-checks/mzcompose.py b/test/platform-checks/mzcompose.py index 3cc6a788ec91c..b2b9d6419fcbe 100644 --- a/test/platform-checks/mzcompose.py +++ b/test/platform-checks/mzcompose.py @@ -18,15 +18,15 @@ from enum import Enum from materialize import buildkite -from materialize.checks.all_checks import * # noqa: F401 F403 +from materialize.checks.all_checks import * # noqa: F403 from materialize.checks.checks import Check from materialize.checks.executors import MzcomposeExecutor, MzcomposeExecutorParallel from materialize.checks.features import Features -from materialize.checks.scenarios import * # noqa: F401 F403 +from materialize.checks.scenarios import * # noqa: F403 from materialize.checks.scenarios import Scenario, SystemVarChange -from materialize.checks.scenarios_backup_restore import * # noqa: F401 F403 -from materialize.checks.scenarios_upgrade import * # noqa: F401 F403 -from materialize.checks.scenarios_zero_downtime import * # noqa: F401 F403 +from materialize.checks.scenarios_backup_restore import * # noqa: F403 +from materialize.checks.scenarios_upgrade import * # noqa: F403 +from materialize.checks.scenarios_zero_downtime import * # noqa: F403 from materialize.mzcompose.composition import ( Composition, Service, diff --git a/test/scalability/mzcompose.py b/test/scalability/mzcompose.py index 17ecf5589e570..c364a224a3fdb 100644 --- a/test/scalability/mzcompose.py +++ b/test/scalability/mzcompose.py @@ -58,10 +58,10 @@ from materialize.scalability.schema.schema import Schema, TransactionIsolation from materialize.scalability.workload.workload import Workload from materialize.scalability.workload.workload_markers import WorkloadMarker -from materialize.scalability.workload.workloads.connection_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.ddl_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.dml_dql_workloads import * # noqa: F401 F403 -from materialize.scalability.workload.workloads.self_test_workloads import * # noqa: F401 F403 +from materialize.scalability.workload.workloads.connection_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.ddl_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.dml_dql_workloads import * # noqa: F403 +from materialize.scalability.workload.workloads.self_test_workloads import * # noqa: F403 from materialize.test_analytics.config.test_analytics_db_config import ( create_test_analytics_config, ) diff --git a/test/scalability/scalability.ipynb b/test/scalability/scalability.ipynb index 164316ceabc07..aa9f13a5e256f 100644 --- a/test/scalability/scalability.ipynb +++ b/test/scalability/scalability.ipynb @@ -17,7 +17,7 @@ "# by the Apache License, Version 2.0.\n", "\n", "import pandas as pd\n", - "from ipywidgets import widgets, interactive\n", + "from ipywidgets import interactive, widgets\n", "from lib import plotit\n", "\n", "workloads = pd.read_csv(\"results/workloads.csv\")\n", diff --git a/test/terraform/mzcompose.py b/test/terraform/mzcompose.py index 311853c4c93c6..62256f03f16f2 100644 --- a/test/terraform/mzcompose.py +++ b/test/terraform/mzcompose.py @@ -735,8 +735,7 @@ def setup( delete_after = "2099-12-31T00:00:00Z" else: delete_after = ( - datetime.datetime.now(datetime.timezone.utc) - + datetime.timedelta(hours=24) + datetime.datetime.now(datetime.UTC) + datetime.timedelta(hours=24) ).strftime("%Y-%m-%dT%H:%M:%SZ") tags = { "Environment": "dev", diff --git a/test/zippy/mzcompose.py b/test/zippy/mzcompose.py index 7d07106d10350..0078f2f060777 100644 --- a/test/zippy/mzcompose.py +++ b/test/zippy/mzcompose.py @@ -48,7 +48,7 @@ from materialize.mzcompose.services.testdrive import Testdrive from materialize.zippy.framework import Test, ci_additional_system_parameter_defaults from materialize.zippy.mz_actions import Mz0dtDeploy -from materialize.zippy.scenarios import * # noqa: F401 F403 +from materialize.zippy.scenarios import * # noqa: F403 def create_mzs(