diff --git a/assets/scripts/build_gallery.py b/assets/scripts/build_gallery.py index dbea246dcd98..be94f2f97de0 100644 --- a/assets/scripts/build_gallery.py +++ b/assets/scripts/build_gallery.py @@ -12,7 +12,7 @@ from glob import glob from io import StringIO from subprocess import CalledProcessError, CompletedProcess, check_output, run -from typing import TYPE_CHECKING, Dict, Final, List, Optional, Tuple +from typing import TYPE_CHECKING, Final, Optional import click import pkg_resources # noqa: TID251 # TODO: switch to importlib.metadata or importlib.resources @@ -441,7 +441,7 @@ def combine_backend_results( # noqa: C901 - too complex json.dump(expectations_info, outfile, indent=4) -def get_contrib_requirements(filepath: str) -> Dict: +def get_contrib_requirements(filepath: str) -> dict: """ Parse the python file from filepath to identify a "library_metadata" dictionary in any defined classes, and return a requirements_info object that includes a list of pip-installable requirements for each class that defines them. @@ -491,8 +491,8 @@ def build_gallery( # noqa: C901 - 17 ignore_suppress: bool = False, ignore_only_for: bool = False, outfile_name: str = "", - only_these_expectations: List[str] | None = None, - only_consider_these_backends: List[str] | None = None, + only_these_expectations: list[str] | None = None, + only_consider_these_backends: list[str] | None = None, context: Optional[FileDataContext] = None, ) -> None: """ @@ -704,7 +704,7 @@ def format_docstring_to_markdown(docstr: str) -> str: # noqa: C901 - too comple return clean_docstr -def _disable_progress_bars() -> Tuple[str, FileDataContext]: +def _disable_progress_bars() -> tuple[str, FileDataContext]: """Return context_dir and context that was created""" context_dir = os.path.join( # noqa: PTH118 os.path.sep, "tmp", f"gx-context-{os.getpid()}" diff --git a/assets/scripts/build_package_gallery.py b/assets/scripts/build_package_gallery.py index ecc175786614..22b82b13fce7 100644 --- a/assets/scripts/build_package_gallery.py +++ b/assets/scripts/build_package_gallery.py @@ -6,7 +6,7 @@ import json import logging import os -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING import pip from great_expectations_contrib.commands import read_package_from_file, sync_package @@ -22,7 +22,7 @@ logger.setLevel(logging.INFO) -def gather_all_contrib_package_paths() -> List[str]: +def gather_all_contrib_package_paths() -> list[str]: """Iterate through contrib/ and identify the relative paths to all contrib packages. A contrib package is defined by the existence of a .great_expectations_package.json file. @@ -30,7 +30,7 @@ def gather_all_contrib_package_paths() -> List[str]: Returns: List of relative paths pointing to contrib packages """ - package_paths: List[str] = [] + package_paths: list[str] = [] for root, _, files in os.walk("contrib/"): for file in files: if file == "package_info.yml": @@ -40,7 +40,7 @@ def gather_all_contrib_package_paths() -> List[str]: return package_paths -def gather_all_package_manifests(package_paths: List[str]) -> List[dict]: +def gather_all_package_manifests(package_paths: list[str]) -> list[dict]: """Takes a list of relative paths to contrib packages and collects dictionaries to represent package state. Args: @@ -49,7 +49,7 @@ def gather_all_package_manifests(package_paths: List[str]) -> List[dict]: Returns: A list of dictionaries that represents contributor package manifests """ # noqa: E501 - payload: List[dict] = [] + payload: list[dict] = [] root = os.getcwd() # noqa: PTH109 for path in package_paths: try: @@ -77,14 +77,14 @@ def gather_all_package_manifests(package_paths: List[str]) -> List[dict]: def _run_pip(stmt: str) -> None: - args: List[str] = stmt.split(" ") + args: list[str] = stmt.split(" ") if hasattr(pip, "main"): pip.main(args) else: pip._internal.main(args) -def write_results_to_disk(path: str, package_manifests: List[dict]) -> None: +def write_results_to_disk(path: str, package_manifests: list[dict]) -> None: """Take the list of package manifests and write to JSON file. Args: diff --git a/ci/checks/check_integration_test_gets_run.py b/ci/checks/check_integration_test_gets_run.py index 35dc3a617fd3..a6a42d314e2e 100644 --- a/ci/checks/check_integration_test_gets_run.py +++ b/ci/checks/check_integration_test_gets_run.py @@ -22,7 +22,6 @@ import shutil import subprocess import sys -from typing import Set IGNORED_VIOLATIONS = [ # TODO: Add IntegrationTestFixture for these tests or remove them if no longer needed @@ -73,7 +72,7 @@ def check_dependencies(*deps: str) -> None: raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003 -def get_test_files(target_dir: pathlib.Path) -> Set[str]: +def get_test_files(target_dir: pathlib.Path) -> set[str]: try: res_snippets = subprocess.run( # noqa: PLW1510 [ @@ -103,7 +102,7 @@ def get_test_files(target_dir: pathlib.Path) -> Set[str]: ) from e -def get_test_files_in_test_suite(target_dir: pathlib.Path) -> Set[str]: +def get_test_files_in_test_suite(target_dir: pathlib.Path) -> set[str]: try: res_test_fixtures = subprocess.run( # noqa: PLW1510 [ diff --git a/ci/checks/check_name_tag_snippets_referenced.py b/ci/checks/check_name_tag_snippets_referenced.py index 375ccf48bf23..b919865e8f4c 100644 --- a/ci/checks/check_name_tag_snippets_referenced.py +++ b/ci/checks/check_name_tag_snippets_referenced.py @@ -23,7 +23,6 @@ import shutil import subprocess import sys -from typing import List # TODO: address ignored snippets by deleting snippet or test file, or adding documentation that references them # noqa: E501 IGNORED_VIOLATIONS = [ @@ -164,7 +163,7 @@ def check_dependencies(*deps: str) -> None: raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003 -def get_snippet_definitions(target_dir: pathlib.Path) -> List[str]: +def get_snippet_definitions(target_dir: pathlib.Path) -> list[str]: try: res_snippets = subprocess.run( # noqa: PLW1510 [ @@ -194,7 +193,7 @@ def get_snippet_definitions(target_dir: pathlib.Path) -> List[str]: ) from e -def get_snippets_used(target_dir: pathlib.Path) -> List[str]: +def get_snippets_used(target_dir: pathlib.Path) -> list[str]: try: res_snippet_usages = subprocess.run( # noqa: PLW1510 [ diff --git a/ci/checks/check_no_line_number_snippets.py b/ci/checks/check_no_line_number_snippets.py index fff69901d637..e15d78e2cb54 100644 --- a/ci/checks/check_no_line_number_snippets.py +++ b/ci/checks/check_no_line_number_snippets.py @@ -8,7 +8,6 @@ import shutil import subprocess import sys -from typing import List ITEMS_IGNORED_FROM_LINE_NUMBER_SNIPPET_CHECKER = { "docs/prepare_to_build_docs.sh", @@ -22,7 +21,7 @@ def check_dependencies(*deps: str) -> None: raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003 -def run_grep(target_dir: pathlib.Path) -> List[str]: +def run_grep(target_dir: pathlib.Path) -> list[str]: try: res = subprocess.run( # noqa: PLW1510 [ diff --git a/ci/checks/check_only_name_tag_snippets.py b/ci/checks/check_only_name_tag_snippets.py index bd0c22860509..85da4346f8d7 100644 --- a/ci/checks/check_only_name_tag_snippets.py +++ b/ci/checks/check_only_name_tag_snippets.py @@ -40,7 +40,6 @@ import shutil import subprocess import sys -from typing import List ITEMS_IGNORED_FROM_NAME_TAG_SNIPPET_CHECKER = { "docs/docusaurus/docs/components/connect_to_data/cloud/_abs_fluent_data_asset_config_keys.mdx", @@ -54,7 +53,7 @@ def check_dependencies(*deps: str) -> None: raise Exception(f"Must have `{dep}` installed in PATH to run {__file__}") # noqa: TRY002, TRY003 -def run_grep(target_dir: pathlib.Path) -> List[str]: +def run_grep(target_dir: pathlib.Path) -> list[str]: try: res_positive = subprocess.run( # noqa: PLW1510 [ diff --git a/ci/checks/validate_docs_snippets.py b/ci/checks/validate_docs_snippets.py index 9247285bef04..02ff6601606a 100644 --- a/ci/checks/validate_docs_snippets.py +++ b/ci/checks/validate_docs_snippets.py @@ -10,7 +10,6 @@ import subprocess import sys import tempfile -from typing import List def check_dependencies(*deps: str) -> None: @@ -32,7 +31,7 @@ def run_docusaurus_build(target_dir: str) -> None: ) -def run_grep(target_dir: str) -> List[str]: +def run_grep(target_dir: str) -> list[str]: try: res = subprocess.run( # noqa: PLW1510 [ diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_between_exclusive_threshold_range.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_between_exclusive_threshold_range.py index 5343a8b05c6a..4f2635aab8b6 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_between_exclusive_threshold_range.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_between_exclusive_threshold_range.py @@ -1,6 +1,6 @@ import copy import os -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -36,10 +36,10 @@ class DataProfilerProfileNumericColumnsDiffBetweenThresholdRange(DataProfilerPro def _pandas( # noqa: C901 - too complex cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_diff = metrics.get("data_profiler.profile_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_between_inclusive_threshold_range.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_between_inclusive_threshold_range.py index 92ae18d32cb1..58da54593f7d 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_between_inclusive_threshold_range.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_between_inclusive_threshold_range.py @@ -1,6 +1,6 @@ import copy import os -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -38,10 +38,10 @@ class DataProfilerProfileNumericColumnsDiffBetweenInclusiveThresholdRange( def _pandas( # noqa: C901 - too complex cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_diff = metrics.get("data_profiler.profile_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_greater_than_or_equal_to_threshold.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_greater_than_or_equal_to_threshold.py index 0767aa0c9711..f40cc6dbf65f 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_greater_than_or_equal_to_threshold.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_greater_than_or_equal_to_threshold.py @@ -1,6 +1,6 @@ import copy import os -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -38,10 +38,10 @@ class DataProfilerProfileNumericColumnsDiffGreaterThanOrEqualToThreshold( def _pandas( # noqa: C901 - too complex cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_diff = metrics.get("data_profiler.profile_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_greater_than_threshold.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_greater_than_threshold.py index 1aba78b5d2c2..7e1f0a8d52a1 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_greater_than_threshold.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_greater_than_threshold.py @@ -1,6 +1,6 @@ import copy import os -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -36,10 +36,10 @@ class DataProfilerProfileNumericColumnsDiffGreaterThanThreshold(DataProfilerProf def _pandas( # noqa: C901 - too complex cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_diff = metrics.get("data_profiler.profile_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_less_than_or_equal_to_threshold.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_less_than_or_equal_to_threshold.py index bd2261c70977..b43612a6f81b 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_less_than_or_equal_to_threshold.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_less_than_or_equal_to_threshold.py @@ -1,6 +1,6 @@ import copy import os -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -38,10 +38,10 @@ class DataProfilerProfileNumericColumnsDiffLessThanOrEqualToThreshold( def _pandas( # noqa: C901 - too complex cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_diff = metrics.get("data_profiler.profile_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_less_than_threshold.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_less_than_threshold.py index f7ab3b01fbb0..f75c2b24596e 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_less_than_threshold.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_diff_less_than_threshold.py @@ -1,6 +1,6 @@ import copy import os -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -36,10 +36,10 @@ class DataProfilerProfileNumericColumnsDiffLessThanThreshold(DataProfilerProfile def _pandas( # noqa: C901 - too complex cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_diff = metrics.get("data_profiler.profile_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_between_exclusive_threshold_range.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_between_exclusive_threshold_range.py index abea48ff6e8e..6b18ba373ef8 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_between_exclusive_threshold_range.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_between_exclusive_threshold_range.py @@ -2,7 +2,7 @@ import json import os import warnings -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -40,10 +40,10 @@ class DataProfilerProfileNumericColumnsPercentDiffBetweenThresholdRange( def _pandas( # noqa: C901 - 22 cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_percent_diff = metrics.get("data_profiler.profile_percent_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_between_inclusive_threshold_range.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_between_inclusive_threshold_range.py index ba67702725d5..26154bb787a5 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_between_inclusive_threshold_range.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_between_inclusive_threshold_range.py @@ -2,7 +2,7 @@ import json import os import warnings -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -42,10 +42,10 @@ class DataProfilerProfileNumericColumnsPercentDiffBetweenInclusiveThresholdRange def _pandas( # noqa: C901 - 22 cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_percent_diff = metrics.get("data_profiler.profile_percent_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_greater_than_or_equal_to_threshold.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_greater_than_or_equal_to_threshold.py index 523ab3f81c3f..45e87903daf3 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_greater_than_or_equal_to_threshold.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_greater_than_or_equal_to_threshold.py @@ -2,7 +2,7 @@ import json import os import warnings -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -42,10 +42,10 @@ class DataProfilerProfileNumericColumnsPercentDiffGreaterThanOrEqualToThreshold( def _pandas( # noqa: C901 - 22 cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_percent_diff = metrics.get("data_profiler.profile_percent_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_greater_than_threshold.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_greater_than_threshold.py index e554fd066be5..07a2ea138047 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_greater_than_threshold.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_greater_than_threshold.py @@ -2,7 +2,7 @@ import json import os import warnings -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -40,10 +40,10 @@ class DataProfilerProfileNumericColumnsPercentDiffGreaterThanThreshold( def _pandas( # noqa: C901 - 22 cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_percent_diff = metrics.get("data_profiler.profile_percent_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_less_than_or_equal_to_threshold.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_less_than_or_equal_to_threshold.py index 3a41f4366bbf..4e1ba5fc4f19 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_less_than_or_equal_to_threshold.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_less_than_or_equal_to_threshold.py @@ -2,7 +2,7 @@ import json import os import warnings -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -42,10 +42,10 @@ class DataProfilerProfileNumericColumnsPercentDiffLessThanOrEqualToThreshold( def _pandas( # noqa: C901 - 22 cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_percent_diff = metrics.get("data_profiler.profile_percent_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_less_than_threshold.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_less_than_threshold.py index 374d64ea1024..64944d54402c 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_less_than_threshold.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/expect_profile_numeric_columns_percent_diff_less_than_threshold.py @@ -2,7 +2,7 @@ import json import os import warnings -from typing import Any, Dict, Optional +from typing import Any, Optional import dataprofiler as dp import pandas as pd @@ -40,10 +40,10 @@ class DataProfilerProfileNumericColumnsPercentDiffLessThanThreshold( def _pandas( # noqa: C901 - 22 cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, ): profile_percent_diff = metrics.get("data_profiler.profile_percent_diff") numeric_columns = metrics.get("data_profiler.profile_numeric_columns") diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/profile_numeric_columns_diff_expectation.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/profile_numeric_columns_diff_expectation.py index 2c5a82abf7a8..fd97e6a6725d 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/profile_numeric_columns_diff_expectation.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/expectations/profile_numeric_columns_diff_expectation.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from great_expectations.execution_engine.execution_engine import ExecutionEngine from great_expectations.expectations.expectation import BatchExpectation @@ -49,7 +49,7 @@ def get_validation_dependencies( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/metrics/data_profiler_metrics/data_profiler_table_column_list.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/metrics/data_profiler_metrics/data_profiler_table_column_list.py index 18894e1479d8..c6931d19f4ac 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/metrics/data_profiler_metrics/data_profiler_table_column_list.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/metrics/data_profiler_metrics/data_profiler_table_column_list.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Optional from capitalone_dataprofiler_expectations.metrics.data_profiler_metrics.data_profiler_profile_metric_provider import ( DataProfilerProfileMetricProvider, @@ -38,7 +38,7 @@ def _pandas( "profile_report_accepted_filtering_values" ] profile_report_column_data_stats: dict = metrics["data_profiler.table_column_infos"] - profile_report_column_names: List[str] = list(profile_report_column_data_stats.keys()) + profile_report_column_names: list[str] = list(profile_report_column_data_stats.keys()) profile_report_column_names = get_dbms_compatible_column_names( column_names=profile_report_column_names, batch_columns_list=metrics["table.columns"], diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/data_assistant/data_profiler_structured_data_assistant.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/data_assistant/data_profiler_structured_data_assistant.py index f80f712d638e..756fd6502a0a 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/data_assistant/data_profiler_structured_data_assistant.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/data_assistant/data_profiler_structured_data_assistant.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Optional from capitalone_dataprofiler_expectations.rule_based_profiler.data_assistant_result import ( DataProfilerStructuredDataAssistantResult, @@ -48,14 +48,14 @@ def __init__( validator=validator, ) - def get_variables(self) -> Optional[Dict[str, Any]]: + def get_variables(self) -> Optional[dict[str, Any]]: """ Returns: Optional "variables" configuration attribute name/value pairs (overrides), commonly-used in Builder objects. """ return None - def get_rules(self) -> Optional[List[Rule]]: + def get_rules(self) -> Optional[list[Rule]]: """ Returns: Optional custom list of "Rule" objects implementing particular "DataAssistant" functionality. @@ -112,7 +112,7 @@ def _build_numeric_rule() -> Rule: data_profiler_profile_report_metric_single_batch_parameter_builder_for_validations: ParameterBuilder = data_profiler_profile_report_metric_single_batch_parameter_builder_for_metrics - validation_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + validation_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] validation_parameter_builder_configs = [ ParameterBuilderConfig( @@ -179,11 +179,11 @@ def _build_numeric_rule() -> Rule: "profile_report_accepted_filtering_values": ["int", "float", "string"], } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ data_profiler_profile_report_metric_single_batch_parameter_builder_for_metrics, ] - expectation_configuration_builders: List[ExpectationConfigurationBuilder] = [ + expectation_configuration_builders: list[ExpectationConfigurationBuilder] = [ expect_column_min_to_be_between_expectation_configuration_builder, expect_column_max_to_be_between_expectation_configuration_builder, expect_column_mean_to_be_between_expectation_configuration_builder, @@ -223,7 +223,7 @@ def _build_float_rule() -> Rule: data_profiler_profile_report_metric_single_batch_parameter_builder_for_validations: ParameterBuilder = data_profiler_profile_report_metric_single_batch_parameter_builder_for_metrics - validation_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + validation_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] validation_parameter_builder_configs = [ ParameterBuilderConfig( @@ -290,11 +290,11 @@ def _build_float_rule() -> Rule: "profile_report_accepted_filtering_values": ["float"], } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ data_profiler_profile_report_metric_single_batch_parameter_builder_for_metrics, ] - expectation_configuration_builders: List[ExpectationConfigurationBuilder] = [ + expectation_configuration_builders: list[ExpectationConfigurationBuilder] = [ expect_column_min_to_be_between_expectation_configuration_builder, expect_column_max_to_be_between_expectation_configuration_builder, expect_column_mean_to_be_between_expectation_configuration_builder, diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/data_assistant_result/data_profiler_structured_data_assistant_result.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/data_assistant_result/data_profiler_structured_data_assistant_result.py index adbc63d852c5..45ef225a703a 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/data_assistant_result/data_profiler_structured_data_assistant_result.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/data_assistant_result/data_profiler_structured_data_assistant_result.py @@ -1,4 +1,4 @@ -from typing import Dict, Tuple, Union +from typing import Union from great_expectations.experimental.rule_based_profiler.altair import AltairDataTypes from great_expectations.experimental.rule_based_profiler.data_assistant_result import ( @@ -12,14 +12,14 @@ class DataProfilerStructuredDataAssistantResult(DataAssistantResult): """ @property - def metric_expectation_map(self) -> Dict[Union[str, Tuple[str]], str]: + def metric_expectation_map(self) -> dict[Union[str, tuple[str]], str]: """ A mapping is defined for which metrics to plot and their associated expectations. """ return {} @property - def metric_types(self) -> Dict[str, AltairDataTypes]: + def metric_types(self) -> dict[str, AltairDataTypes]: """ A mapping is defined for the Altair data type associated with each metric. """ diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/domain_builder/data_profiler_column_domain_builder.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/domain_builder/data_profiler_column_domain_builder.py index 7023a3d15765..fb15f3bd17d5 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/domain_builder/data_profiler_column_domain_builder.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/rule_based_profiler/domain_builder/data_profiler_column_domain_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Iterable, List, Optional, Union +from typing import TYPE_CHECKING, Iterable, Optional, Union from great_expectations.core.domain import ( Domain, @@ -33,17 +33,17 @@ class DataProfilerColumnDomainBuilder(ColumnDomainBuilder): def __init__( self, - include_column_names: Optional[Union[str, Optional[List[str]]]] = None, - exclude_column_names: Optional[Union[str, Optional[List[str]]]] = None, - include_column_name_suffixes: Optional[Union[str, Iterable, List[str]]] = None, - exclude_column_name_suffixes: Optional[Union[str, Iterable, List[str]]] = None, + include_column_names: Optional[Union[str, Optional[list[str]]]] = None, + exclude_column_names: Optional[Union[str, Optional[list[str]]]] = None, + include_column_name_suffixes: Optional[Union[str, Iterable, list[str]]] = None, + exclude_column_name_suffixes: Optional[Union[str, Iterable, list[str]]] = None, semantic_type_filter_module_name: Optional[str] = None, semantic_type_filter_class_name: Optional[str] = None, include_semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, exclude_semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: @@ -79,7 +79,7 @@ def _get_domains( rule_name: str, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: """Return domains matching the specified tolerance limits. Args: @@ -90,11 +90,11 @@ def _get_domains( Returns: List of domains that match the desired tolerance limits. """ - batch_ids: List[str] = self.get_batch_ids(variables=variables) # type: ignore[assignment] # could be None + batch_ids: list[str] = self.get_batch_ids(variables=variables) # type: ignore[assignment] # could be None validator: Validator = self.get_validator(variables=variables) # type: ignore[assignment] # could be None - table_column_names: List[str] = self.get_table_column_names( + table_column_names: list[str] = self.get_table_column_names( batch_ids=batch_ids, validator=validator, variables=variables, @@ -128,7 +128,7 @@ def _get_domains( ) # get metrics and profile path from variables and then pass them into here - profile_report_column_names: List[str] = validator.get_metric( # type: ignore[assignment] # could be None + profile_report_column_names: list[str] = validator.get_metric( # type: ignore[assignment] # could be None metric=MetricConfiguration( metric_name="data_profiler.table_column_list", metric_domain_kwargs={}, @@ -152,14 +152,14 @@ def _get_domains( message=f"Error: Some of profiled columns in {self.__class__.__name__} are not found in Batch table." ) - effective_column_names: List[str] = self.get_filtered_column_names( + effective_column_names: list[str] = self.get_filtered_column_names( column_names=profile_report_column_names, batch_ids=batch_ids, validator=validator, variables=variables, ) - domains: List[Domain] = build_domains_from_column_names( + domains: list[Domain] = build_domains_from_column_names( rule_name=rule_name, column_names=effective_column_names, domain_type=self.domain_type, diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/tests/rule_based_profiler/data_assistant/test_data_profiler_structured_data_assistant.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/tests/rule_based_profiler/data_assistant/test_data_profiler_structured_data_assistant.py index eb87b7a53fe0..45e1fa6d346b 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/tests/rule_based_profiler/data_assistant/test_data_profiler_structured_data_assistant.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/tests/rule_based_profiler/data_assistant/test_data_profiler_structured_data_assistant.py @@ -1,7 +1,7 @@ from __future__ import annotations import os -from typing import TYPE_CHECKING, Dict, Optional, cast +from typing import TYPE_CHECKING, Optional, cast import pytest from capitalone_dataprofiler_expectations.metrics import * # noqa: F403 @@ -150,7 +150,7 @@ def test_profile_data_profiler_structured_data_assistant_metrics_count( bobby_profile_data_profiler_structured_data_assistant_result: DataProfilerStructuredDataAssistantResult, ) -> None: domain: Domain - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] num_metrics: int domain_key = Domain( @@ -182,11 +182,11 @@ def test_profile_data_profiler_structured_data_assistant_metrics_count( def test_profile_data_profiler_structured_data_assistant_result_batch_id_to_batch_identifier_display_name_map_coverage( bobby_profile_data_profiler_structured_data_assistant_result: DataProfilerStructuredDataAssistantResult, ): - metrics_by_domain: Optional[Dict[Domain, Dict[str, ParameterNode]]] = ( + metrics_by_domain: Optional[dict[Domain, dict[str, ParameterNode]]] = ( bobby_profile_data_profiler_structured_data_assistant_result.metrics_by_domain ) - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] parameter_node: ParameterNode batch_id: str assert all( diff --git a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/tests/rule_based_profiler/domain_builder/test_data_profiler_column_domain_builder.py b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/tests/rule_based_profiler/domain_builder/test_data_profiler_column_domain_builder.py index fb6852115dc1..78cefd2bbb19 100644 --- a/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/tests/rule_based_profiler/domain_builder/test_data_profiler_column_domain_builder.py +++ b/contrib/capitalone_dataprofiler_expectations/capitalone_dataprofiler_expectations/tests/rule_based_profiler/domain_builder/test_data_profiler_column_domain_builder.py @@ -1,7 +1,7 @@ from __future__ import annotations import os -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from unittest import mock import pytest @@ -77,7 +77,7 @@ def test_data_profiler_column_domain_builder_with_profile_path_as_value( domain_builder: DomainBuilder = DataProfilerColumnDomainBuilder( data_context=data_context, ) - domains: List[Domain] = domain_builder.get_domains( + domains: list[Domain] = domain_builder.get_domains( rule_name="my_rule", variables=variables, batch_request=batch_request, @@ -348,7 +348,7 @@ def test_data_profiler_column_domain_builder_with_profile_path_as_default_refere domain_builder: DomainBuilder = DataProfilerColumnDomainBuilder( data_context=data_context, ) - domains: List[Domain] = domain_builder.get_domains( + domains: list[Domain] = domain_builder.get_domains( rule_name="my_rule", variables=variables, batch_request=batch_request, @@ -619,7 +619,7 @@ def test_data_profiler_column_domain_builder_with_profile_path_as_reference( domain_builder: DomainBuilder = DataProfilerColumnDomainBuilder( data_context=data_context, ) - domains: List[Domain] = domain_builder.get_domains( + domains: list[Domain] = domain_builder.get_domains( rule_name="my_rule", variables=variables, batch_request=batch_request, @@ -897,7 +897,7 @@ def test_data_profiler_column_domain_builder_with_profile_path_as_reference_with ], data_context=data_context, ) - domains: List[Domain] = domain_builder.get_domains( + domains: list[Domain] = domain_builder.get_domains( rule_name="my_rule", variables=variables, batch_request=batch_request, @@ -1101,7 +1101,7 @@ def test_data_profiler_column_domain_builder_with_profile_path_as_reference_with "dataprofiler.profilers.profile_builder.BaseProfiler.load", return_value=mock_base_data_profiler, ): - domains: List[Domain] = domain_builder.get_domains( + domains: list[Domain] = domain_builder.get_domains( rule_name="my_rule", variables=variables, batch_request=batch_request, diff --git a/contrib/capitalone_dataprofiler_expectations/setup.py b/contrib/capitalone_dataprofiler_expectations/setup.py index 3fe83cd9d308..bee9806aea7c 100644 --- a/contrib/capitalone_dataprofiler_expectations/setup.py +++ b/contrib/capitalone_dataprofiler_expectations/setup.py @@ -1,10 +1,8 @@ -from typing import List - import setuptools from setuptools import find_packages -def get_requirements() -> List[str]: +def get_requirements() -> list[str]: with open("requirements.txt") as f: requirements = f.read().splitlines() return requirements diff --git a/contrib/cli/great_expectations_contrib/package.py b/contrib/cli/great_expectations_contrib/package.py index 03b5162dcd20..145bbd71919a 100644 --- a/contrib/cli/great_expectations_contrib/package.py +++ b/contrib/cli/great_expectations_contrib/package.py @@ -5,7 +5,7 @@ import sys from dataclasses import asdict, dataclass from enum import Enum -from typing import Any, Dict, List, Optional, Type +from typing import Any, Optional import pkg_resources from ruamel.yaml import YAML @@ -61,7 +61,7 @@ class SocialLink(SerializableDictDot): @dataclass class DomainExpert(SerializableDictDot): full_name: str - social_links: Optional[List[SocialLink]] = None + social_links: Optional[list[SocialLink]] = None picture: Optional[str] = None title: Optional[str] = None bio: Optional[str] = None @@ -80,16 +80,16 @@ class GreatExpectationsContribPackageManifest(SerializableDictDot): package_name: Optional[str] = None icon: Optional[str] = None description: Optional[str] = None - expectations: Optional[Dict[str, ExpectationDiagnostics]] = None + expectations: Optional[dict[str, ExpectationDiagnostics]] = None expectation_count: Optional[int] = None - dependencies: Optional[List[Dependency]] = None + dependencies: Optional[list[Dependency]] = None maturity: Optional[Maturity] = None status: Optional[PackageCompletenessStatus] = None # Users - owners: Optional[List[GitHubUser]] = None - contributors: Optional[List[GitHubUser]] = None - domain_experts: Optional[List[DomainExpert]] = None + owners: Optional[list[GitHubUser]] = None + contributors: Optional[list[GitHubUser]] = None + domain_experts: Optional[list[DomainExpert]] = None # Metadata version: Optional[str] = None @@ -111,7 +111,7 @@ def update_package_state(self) -> None: ) self._update_attrs_with_diagnostics(diagnostics) - def _update_attrs_with_diagnostics(self, diagnostics: List[ExpectationDiagnostics]) -> None: + def _update_attrs_with_diagnostics(self, diagnostics: list[ExpectationDiagnostics]) -> None: self._update_from_package_info("package_info.yml") self._update_expectations(diagnostics) self._update_dependencies("requirements.txt") @@ -182,7 +182,7 @@ def _update_from_package_info(self, path: str) -> None: # noqa: C901 - too comp domain_expert = DomainExpert(**expert) self.domain_experts.append(domain_expert) - def _update_expectations(self, diagnostics: List[ExpectationDiagnostics]) -> None: + def _update_expectations(self, diagnostics: list[ExpectationDiagnostics]) -> None: expectations = {} status = {maturity.name: 0 for maturity in Maturity} @@ -228,7 +228,7 @@ def _convert_to_dependency( dependencies = list(map(_convert_to_dependency, requirements)) self.dependencies = dependencies - def _update_contributors(self, diagnostics: List[ExpectationDiagnostics]) -> None: + def _update_contributors(self, diagnostics: list[ExpectationDiagnostics]) -> None: contributors = [] for diagnostic in diagnostics: for contributor in diagnostic.library_metadata.contributors: @@ -239,7 +239,7 @@ def _update_contributors(self, diagnostics: List[ExpectationDiagnostics]) -> Non self.contributors = contributors @staticmethod - def retrieve_package_expectations_diagnostics() -> List[ExpectationDiagnostics]: + def retrieve_package_expectations_diagnostics() -> list[ExpectationDiagnostics]: try: package = GreatExpectationsContribPackageManifest._identify_user_package() expectations_module = ( @@ -282,18 +282,15 @@ def _import_expectations_module(package: str) -> Any: # Need to add user's project to the PYTHONPATH cwd = os.getcwd() # noqa: PTH109 sys.path.append(cwd) - try: - expectations_module = importlib.import_module(f"{package}.expectations") - return expectations_module - except ModuleNotFoundError: # noqa: TRY203 - raise + expectations_module = importlib.import_module(f"{package}.expectations") + return expectations_module @staticmethod def _retrieve_expectations_from_module( expectations_module: Any, - ) -> List[Type[Expectation]]: - expectations: List[Type[Expectation]] = [] - names: List[str] = [] + ) -> list[type[Expectation]]: + expectations: list[type[Expectation]] = [] + names: list[str] = [] for name, obj in inspect.getmembers(expectations_module): # ProfileNumericColumnsDiffExpectation from capitalone_dataprofiler_expectations # is a base class that the contrib Expectations in that package all inherit from @@ -306,8 +303,8 @@ def _retrieve_expectations_from_module( @staticmethod def _gather_diagnostics( - expectations: List[Type[Expectation]], - ) -> List[ExpectationDiagnostics]: + expectations: list[type[Expectation]], + ) -> list[ExpectationDiagnostics]: diagnostics_list = [] for expectation in expectations: instance = expectation() diff --git a/contrib/cli/setup.py b/contrib/cli/setup.py index cf180c5d1d9a..8040a72b1ddf 100644 --- a/contrib/cli/setup.py +++ b/contrib/cli/setup.py @@ -1,9 +1,7 @@ -from typing import List - import setuptools -def get_requirements() -> List[str]: +def get_requirements() -> list[str]: with open("requirements.txt") as f: requirements = f.read().splitlines() return requirements diff --git a/contrib/cli/tests/test_package.py b/contrib/cli/tests/test_package.py index 1b28434da1ce..f0c65b1147d1 100644 --- a/contrib/cli/tests/test_package.py +++ b/contrib/cli/tests/test_package.py @@ -1,5 +1,3 @@ -from typing import List - import py import pytest from great_expectations_contrib.package import ( @@ -23,7 +21,7 @@ def package() -> GreatExpectationsContribPackageManifest: @pytest.fixture -def diagnostics() -> List[ExpectationDiagnostics]: +def diagnostics() -> list[ExpectationDiagnostics]: expectations = [ gxe.ExpectColumnMinToBeBetween, gxe.ExpectColumnMostCommonValueToBeInSet, @@ -40,7 +38,7 @@ def diagnostics() -> List[ExpectationDiagnostics]: def test_update_expectations( package: GreatExpectationsContribPackageManifest, - diagnostics: List[ExpectationDiagnostics], + diagnostics: list[ExpectationDiagnostics], ): package._update_expectations(diagnostics) diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_chisquare_simple_test_p_value_to_be_greater_than.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_chisquare_simple_test_p_value_to_be_greater_than.py index 75123e93953f..42df5eebf6a1 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_chisquare_simple_test_p_value_to_be_greater_than.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_chisquare_simple_test_p_value_to_be_greater_than.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional import scipy.stats as stats @@ -113,7 +113,7 @@ class ExpectColumnChisquareSimpleTestPValueToBeGreaterThan(BatchExpectation): # This method performs a validation of your metrics against your success keys, returning a dict indicating the success or failure of the Expectation. def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_discrete_entropy_to_be_between.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_discrete_entropy_to_be_between.py index be5132bc5aa9..3109e2937b5a 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_discrete_entropy_to_be_between.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_discrete_entropy_to_be_between.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Optional import scipy.stats @@ -72,10 +72,10 @@ def _pandas(cls, column, base, **kwargs): def _spark( cls, execution_engine: "SparkDFExecutionEngine", - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[Tuple, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[tuple, Any], + runtime_configuration: dict, ): ( _df, @@ -337,7 +337,7 @@ def _prescriptive_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_distinct_values_to_be_continuous.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_distinct_values_to_be_continuous.py index 5990831660c0..2fdeac800f15 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_distinct_values_to_be_continuous.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_distinct_values_to_be_continuous.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any from great_expectations.execution_engine import ExecutionEngine from great_expectations.expectations.expectation import ( @@ -254,7 +254,7 @@ def _expected_list( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_distribution_to_match_benfords_law.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_distribution_to_match_benfords_law.py index 04763bcf122f..755ad28d6a91 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_distribution_to_match_benfords_law.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_distribution_to_match_benfords_law.py @@ -1,5 +1,5 @@ import math -from typing import Dict, Optional +from typing import Optional from great_expectations.execution_engine import ExecutionEngine, PandasExecutionEngine from great_expectations.execution_engine.sqlalchemy_execution_engine import ( @@ -300,7 +300,7 @@ class ExpectColumnDistributionToMatchBenfordsLaw(ColumnAggregateExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_kolmogoro_smirnov_test_p_value_to_be_greater_than.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_kolmogoro_smirnov_test_p_value_to_be_greater_than.py index efe69e944dd6..6fc2b7dc23c0 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_kolmogoro_smirnov_test_p_value_to_be_greater_than.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_kolmogoro_smirnov_test_p_value_to_be_greater_than.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional import scipy.stats as stats @@ -113,7 +113,7 @@ class ExpectColumnKolmogorovSmirnovTestPValueToBeGreaterThan(BatchExpectation): # This method performs a validation of your metrics against your success keys, returning a dict indicating the success or failure of the Expectation. def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_kurtosis_to_be_between.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_kurtosis_to_be_between.py index 867e20f1e5e2..9be2d771635f 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_kurtosis_to_be_between.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_kurtosis_to_be_between.py @@ -1,5 +1,3 @@ -from typing import Dict - from scipy import stats from great_expectations.compatibility.pyspark import functions as F @@ -319,7 +317,7 @@ class ExpectColumnKurtosisToBeBetween(ColumnAggregateExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_percentile_value_to_be_above.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_percentile_value_to_be_above.py index 2bb397b7d46e..57484dc5803d 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_percentile_value_to_be_above.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_percentile_value_to_be_above.py @@ -1,5 +1,3 @@ -from typing import Dict - from great_expectations.execution_engine import ( ExecutionEngine, PandasExecutionEngine, @@ -89,7 +87,7 @@ class ExpectColumnPercentileToBeAbove(ColumnAggregateExpectation): # This method performs a validation of your metrics against your success keys, returning a dict indicating the success or failure of the Expectation. def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_skew_to_be_between.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_skew_to_be_between.py index a66800f24812..31178a8c994a 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_skew_to_be_between.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_skew_to_be_between.py @@ -1,6 +1,6 @@ import logging import traceback -from typing import Any, Dict, Tuple +from typing import Any import numpy as np from scipy import stats @@ -50,10 +50,10 @@ def _spark(cls, column, abs=False, **kwargs): def _sqlalchemy( cls, execution_engine: "SqlAlchemyExecutionEngine", - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[Tuple, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[tuple, Any], + runtime_configuration: dict, ): ( selectable, @@ -379,7 +379,7 @@ class ExpectColumnSkewToBeBetween(ColumnAggregateExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_sum_to_be.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_sum_to_be.py index a0a8c81b0be2..59ec5d723dc2 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_sum_to_be.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_sum_to_be.py @@ -4,8 +4,6 @@ https://docs.greatexpectations.io/docs/guides/expectations/creating_custom_expectations/how_to_create_custom_column_aggregate_expectations """ -from typing import Dict - from great_expectations.execution_engine import ExecutionEngine from great_expectations.expectations.expectation import ColumnAggregateExpectation @@ -50,7 +48,7 @@ class ExpectColumnSumToBe(ColumnAggregateExpectation): # This method performs a validation of your metrics against your success keys, returning a dict indicating the success or failure of the Expectation. def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_to_have_no_days_missing.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_to_have_no_days_missing.py index 0080160072c1..a0d91143e8a5 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_to_have_no_days_missing.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_to_have_no_days_missing.py @@ -1,5 +1,3 @@ -from typing import Dict - from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa from great_expectations.core.metric_domain_types import MetricDomainTypes from great_expectations.execution_engine import ( @@ -110,7 +108,7 @@ class ExpectColumnToHaveNoDaysMissing(ColumnAggregateExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_to_have_no_months_missing.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_to_have_no_months_missing.py index ace9ad34a0ea..09d02c859602 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_to_have_no_months_missing.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_to_have_no_months_missing.py @@ -1,5 +1,3 @@ -from typing import Dict - from dateutil.relativedelta import relativedelta from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa @@ -141,7 +139,7 @@ class ExpectColumnToHaveNoMonthsMissing(ColumnAggregateExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_after_split_to_be_in_set.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_after_split_to_be_in_set.py index 2d1cf34b496a..f506b7fcb52b 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_after_split_to_be_in_set.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_after_split_to_be_in_set.py @@ -4,8 +4,6 @@ https://docs.greatexpectations.io/docs/guides/expectations/creating_custom_expectations/how_to_create_custom_column_map_expectations """ -from typing import List - from great_expectations.execution_engine import PandasExecutionEngine from great_expectations.expectations.expectation import ColumnMapExpectation from great_expectations.expectations.metrics import ( @@ -14,7 +12,7 @@ ) -def are_values_after_split_in_value_set(val: str, delimiter: str, value_set: List[str]) -> bool: +def are_values_after_split_in_value_set(val: str, delimiter: str, value_set: list[str]) -> bool: all_split_values = [v.strip() for v in val.split(delimiter)] return all(val in value_set for val in all_split_values) diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_in_set_spark_optimized.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_in_set_spark_optimized.py index 8df2b37d719c..a5e66b07bbac 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_in_set_spark_optimized.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_in_set_spark_optimized.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from great_expectations.compatibility.pyspark import functions as F from great_expectations.core.metric_domain_types import MetricDomainTypes @@ -119,7 +119,7 @@ def validate_configuration( # This method performs a validation of your metrics against your success keys, returning a dict indicating the success or failure of the Expectation. def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_normally_distributed.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_normally_distributed.py index 9b6ae52ba8ee..8542bff0d4d9 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_normally_distributed.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_normally_distributed.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime -from typing import Dict, Union +from typing import Union from scipy import stats @@ -268,7 +268,7 @@ class ExpectColumnValuesToBeNormallyDistributed(ColumnAggregateExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_present_in_other_table.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_present_in_other_table.py index 87a954739295..75ce27f0253b 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_present_in_other_table.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_present_in_other_table.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, ClassVar, Optional, Union import pandas as pd @@ -92,18 +92,18 @@ class ExpectColumnValuesToBePresentInOtherTable(QueryExpectation): "manually_reviewed_code": True, } - success_keys: ClassVar[Tuple[str, ...]] = ( + success_keys: ClassVar[tuple[str, ...]] = ( "template_dict", "query", ) - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "query", "batch_id", "row_condition", "condition_parser", ) - examples: ClassVar[List[dict]] = [ + examples: ClassVar[list[dict]] = [ { "data": [ { @@ -202,7 +202,7 @@ def _prescriptive_renderer( configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: runtime_configuration = runtime_configuration or {} styling = runtime_configuration.get("styling") @@ -247,7 +247,7 @@ def _diagnostic_unexpected_table_renderer( if result_dict is None: return None - unexpected_index_list: Optional[List[dict]] = result_dict.get("unexpected_index_list") + unexpected_index_list: Optional[list[dict]] = result_dict.get("unexpected_index_list") # Don't render table if we don't have unexpected_values if not unexpected_index_list: return None diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_string_integers_increasing.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_string_integers_increasing.py index c3fcd4709be0..a708a0fc0933 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_string_integers_increasing.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_be_string_integers_increasing.py @@ -1,5 +1,5 @@ import logging -from typing import Callable, Dict, Optional +from typing import Callable, Optional import numpy as np @@ -191,7 +191,7 @@ def _validate_success_key( param: str, required: bool, configuration: Optional[ExpectationConfiguration], - validation_rules: Dict[Callable, str], + validation_rules: dict[Callable, str], ) -> None: """""" if param not in configuration.kwargs: @@ -238,7 +238,7 @@ def get_validation_dependencies( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ) -> ExpectationValidationResult: diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_match_xml_schema.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_match_xml_schema.py index d99dd7399778..9aa0afeed552 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_match_xml_schema.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_match_xml_schema.py @@ -37,41 +37,25 @@ class ColumnValuesMatchXmlSchema(ColumnMapMetricProvider): @column_condition_partial(engine=PandasExecutionEngine) def _pandas(cls, column, xml_schema, format, **kwargs): - try: - xmlschema_doc = etree.fromstring(xml_schema) - xmlschema = etree.XMLSchema(xmlschema_doc) - except etree.ParseError: # noqa: TRY203 - raise - except: # noqa: TRY203 - raise + xmlschema_doc = etree.fromstring(xml_schema) + xmlschema = etree.XMLSchema(xmlschema_doc) def matches_xml_schema(val): - try: - xml_doc = etree.fromstring(val) - return xmlschema(xml_doc) - except: # noqa: TRY203 - raise + xml_doc = etree.fromstring(val) + return xmlschema(xml_doc) return column.map(matches_xml_schema) @column_condition_partial(engine=SparkDFExecutionEngine) def _spark(cls, column, xml_schema, **kwargs): - try: - xmlschema_doc = etree.fromstring(xml_schema) - xmlschema = etree.XMLSchema(xmlschema_doc) - except etree.ParseError: # noqa: TRY203 - raise - except: # noqa: TRY203 - raise + xmlschema_doc = etree.fromstring(xml_schema) + xmlschema = etree.XMLSchema(xmlschema_doc) def matches_xml_schema(val): if val is None: return False - try: - xml_doc = etree.fromstring(val) - return xmlschema(xml_doc) - except: # noqa: TRY203 - raise + xml_doc = etree.fromstring(val) + return xmlschema(xml_doc) matches_xml_schema_udf = F.udf(matches_xml_schema, pyspark.types.BooleanType()) diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_not_be_null_and_column_to_not_be_empty.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_not_be_null_and_column_to_not_be_empty.py index dd8d228342c7..9ee1a95bfcf3 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_not_be_null_and_column_to_not_be_empty.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_values_to_not_be_null_and_column_to_not_be_empty.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Optional import numpy as np @@ -306,7 +306,7 @@ def _descriptive_column_properties_table_missing_percent_row_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_column_wasserstein_distance_to_be_less_than.py b/contrib/experimental/great_expectations_experimental/expectations/expect_column_wasserstein_distance_to_be_less_than.py index 9ef7239b9ce7..2a7360c15ebd 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_column_wasserstein_distance_to_be_less_than.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_column_wasserstein_distance_to_be_less_than.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Dict, Optional +from typing import Optional from scipy import stats @@ -280,7 +280,7 @@ class ExpectColumnWassersteinDistanceToBeLessThan(ColumnAggregateExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_day_count_to_be_close_to_equivalent_week_day_mean.py b/contrib/experimental/great_expectations_experimental/expectations/expect_day_count_to_be_close_to_equivalent_week_day_mean.py index 6c8c45611c2b..2c527eae42b3 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_day_count_to_be_close_to_equivalent_week_day_mean.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_day_count_to_be_close_to_equivalent_week_day_mean.py @@ -1,5 +1,4 @@ from datetime import datetime, timedelta -from typing import Dict, List from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa from great_expectations.core.metric_domain_types import MetricDomainTypes @@ -217,7 +216,7 @@ class ExpectDayCountToBeCloseToEquivalentWeekDayMean(ColumnAggregateExpectation) def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): @@ -229,7 +228,7 @@ def _validate( days_ago_dict = get_days_ago_dict(run_date) - equivalent_previous_days: List[datetime] = [days_ago_dict[i] for i in FOUR_PREVIOUS_WEEKS] + equivalent_previous_days: list[datetime] = [days_ago_dict[i] for i in FOUR_PREVIOUS_WEEKS] assert min(equivalent_previous_days) > ( datetime.today() - timedelta(METRIC_SAMPLE_LIMIT) @@ -263,7 +262,7 @@ def _validate( def get_counts_per_day_as_dict( metrics: dict, run_date: str, equivalent_previous_days: list ) -> dict: - equivalent_previous_days_str: List[str] = [ + equivalent_previous_days_str: list[str] = [ datetime.strftime(i, date_format) for i in equivalent_previous_days ] all_days_list = equivalent_previous_days_str + [run_date] @@ -287,11 +286,11 @@ def get_diff_fraction( Added +1 to both nuemrator and denominator, to account for cases when previous average is 0. """ - equivalent_previous_days_str: List[str] = [ + equivalent_previous_days_str: list[str] = [ datetime.strftime(i, date_format) for i in equivalent_previous_days ] - previous_days_counts: List[int] = [ + previous_days_counts: list[int] = [ day_counts_dict[i] for i in day_counts_dict if i in equivalent_previous_days_str ] diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_day_sum_to_be_close_to_equivalent_week_day_mean.py b/contrib/experimental/great_expectations_experimental/expectations/expect_day_sum_to_be_close_to_equivalent_week_day_mean.py index 9cdc4643869f..3b0715a8aa82 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_day_sum_to_be_close_to_equivalent_week_day_mean.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_day_sum_to_be_close_to_equivalent_week_day_mean.py @@ -1,5 +1,4 @@ from datetime import date, datetime, timedelta -from typing import Dict, List from great_expectations.execution_engine import ExecutionEngine from great_expectations.expectations.expectation import QueryExpectation @@ -182,7 +181,7 @@ class ExpectDaySumToBeCloseToEquivalentWeekDayMean(QueryExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): @@ -226,14 +225,14 @@ def average_if_nonempty(list_: list): return sum(list_) / len(list_) if len(list_) > 0 else 0 -def get_diff_fraction(yesterday_sum: int, result_dict: dict, days_back_list: List[int]): +def get_diff_fraction(yesterday_sum: int, result_dict: dict, days_back_list: list[int]): days_ago_dict = {days_ago: TODAY - timedelta(days=days_ago) for days_ago in days_back_list} - equivalent_previous_days: List[date] = list(days_ago_dict.values()) - equivalent_previous_days_str: List[str] = [ + equivalent_previous_days: list[date] = list(days_ago_dict.values()) + equivalent_previous_days_str: list[str] = [ datetime.strftime(i, date_format) for i in equivalent_previous_days ] - previous_days_sums: List[int] = [ + previous_days_sums: list[int] = [ result_dict[equiv_day] for equiv_day in equivalent_previous_days_str ] diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_multicolumn_values_to_be_equal.py b/contrib/experimental/great_expectations_experimental/expectations/expect_multicolumn_values_to_be_equal.py index d99d2d284d80..858747b7f3d5 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_multicolumn_values_to_be_equal.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_multicolumn_values_to_be_equal.py @@ -1,7 +1,7 @@ from __future__ import annotations from functools import reduce -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Optional import sqlalchemy as sa @@ -388,7 +388,7 @@ def _descriptive_column_properties_table_missing_percent_row_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_queried_column_value_frequency_to_meet_threshold.py b/contrib/experimental/great_expectations_experimental/expectations/expect_queried_column_value_frequency_to_meet_threshold.py index 36effe65e99e..81139296e7cf 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_queried_column_value_frequency_to_meet_threshold.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_queried_column_value_frequency_to_meet_threshold.py @@ -4,7 +4,7 @@ https://docs.greatexpectations.io/docs/guides/expectations/creating_custom_expectations/how_to_create_custom_query_expectations """ -from typing import List, Optional, Union +from typing import Optional, Union from great_expectations.exceptions.exceptions import ( InvalidExpectationConfigurationError, @@ -23,8 +23,8 @@ class ExpectQueriedColumnValueFrequencyToMeetThreshold(QueryExpectation): """Expect the frequency of occurrences of a specified value in a queried column to be at least percent of values in that column.""" column: str - threshold: Union[float, List[float]] - value: Union[str, List[str]] + threshold: Union[float, list[float]] + value: Union[str, list[str]] metric_dependencies = ("query.column",) diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_queried_table_row_count_to_be.py b/contrib/experimental/great_expectations_experimental/expectations/expect_queried_table_row_count_to_be.py index 2e773a7874d6..2a41d6a67173 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_queried_table_row_count_to_be.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_queried_table_row_count_to_be.py @@ -4,7 +4,7 @@ https://docs.greatexpectations.io/docs/guides/expectations/creating_custom_expectations/how_to_create_custom_query_expectations """ -from typing import ClassVar, List, Tuple, Union +from typing import ClassVar, Union from great_expectations.execution_engine import ExecutionEngine from great_expectations.expectations.expectation import ( @@ -46,20 +46,20 @@ class ExpectQueriedTableRowCountToBe(QueryExpectation): FROM {batch} """ - metric_dependencies: ClassVar[Tuple[str, ...]] = ("query.table",) + metric_dependencies: ClassVar[tuple[str, ...]] = ("query.table",) - success_keys: ClassVar[Tuple[str, ...]] = ( + success_keys: ClassVar[tuple[str, ...]] = ( "value", "query", ) - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "batch_id", "row_condition", "condition_parser", ) - examples: ClassVar[List[dict]] = [ + examples: ClassVar[list[dict]] = [ { "data": [ { diff --git a/contrib/experimental/great_expectations_experimental/expectations/expect_table_checksum_to_equal_other_table.py b/contrib/experimental/great_expectations_experimental/expectations/expect_table_checksum_to_equal_other_table.py index ad33ecec7b50..9f086598af61 100644 --- a/contrib/experimental/great_expectations_experimental/expectations/expect_table_checksum_to_equal_other_table.py +++ b/contrib/experimental/great_expectations_experimental/expectations/expect_table_checksum_to_equal_other_table.py @@ -10,7 +10,7 @@ # For most Expectations, the main business logic for calculation will live here. # To learn about the relationship between Metrics and Expectations, please visit # https://docs.greatexpectations.io/en/latest/reference/core_concepts.html#expectations-and-metrics. -from typing import Any, Dict, Optional, Tuple +from typing import Any, Optional from great_expectations.core import ( ExpectationValidationResult, @@ -74,10 +74,10 @@ class TableChecksum(TableMetricProvider): def _sqlalchemy( cls, execution_engine: SqlAlchemyExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[Tuple, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[tuple, Any], + runtime_configuration: dict, ): ( selectable, @@ -186,10 +186,10 @@ class TableChecksumValues(TableMetricProvider): def _sqlalchemy( cls, execution_engine: SqlAlchemyExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[Tuple, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[tuple, Any], + runtime_configuration: dict, ): cksum_value_self = metrics.get("table.checksum.self") cksum_value_other = metrics.get("table.checksum.other") @@ -478,7 +478,7 @@ def _prescriptive_renderer( # This method will utilize the computed metric to validate that your Expectation about the Table is true def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant/growth_numeric_data_assistant.py b/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant/growth_numeric_data_assistant.py index acbc687c8f3a..71b01a549851 100644 --- a/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant/growth_numeric_data_assistant.py +++ b/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant/growth_numeric_data_assistant.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Optional from contrib.experimental.great_expectations_experimental.rule_based_profiler.data_assistant_result import ( GrowthNumericDataAssistantResult, @@ -61,14 +61,14 @@ def __init__( validator=validator, ) - def get_variables(self) -> Optional[Dict[str, Any]]: + def get_variables(self) -> Optional[dict[str, Any]]: """ Returns: Optional "variables" configuration attribute name/value pairs (overrides), commonly-used in Builder objects. """ return None - def get_rules(self) -> Optional[List[Rule]]: + def get_rules(self) -> Optional[list[Rule]]: """ Returns: Optional custom list of "Rule" objects implementing particular "DataAssistant" functionality. @@ -137,7 +137,7 @@ def _build_table_rule() -> Rule: # Step-3: Declare "ParameterBuilder" for every "validation" need in "ExpectationConfigurationBuilder" objects. - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = [ + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = [ ParameterBuilderConfig( **table_row_count_metric_multi_batch_parameter_builder_for_metrics.to_json_dict() ), @@ -159,7 +159,7 @@ def _build_table_rule() -> Rule: # Step-4: Pass "validation" "ParameterBuilderConfig" objects to every "DefaultExpectationConfigurationBuilder", responsible for emitting "ExpectationConfiguration" (with specified "expectation_type"). - validation_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + validation_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] validation_parameter_builder_configs = [ ParameterBuilderConfig( @@ -211,11 +211,11 @@ def _build_table_rule() -> Rule: "exact_match": None, "success_ratio": 1.0, } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ table_row_count_metric_multi_batch_parameter_builder_for_metrics, table_columns_metric_multi_batch_parameter_builder_for_metrics, ] - expectation_configuration_builders: List[ExpectationConfigurationBuilder] = [ + expectation_configuration_builders: list[ExpectationConfigurationBuilder] = [ expect_table_row_count_to_be_between_expectation_configuration_builder, expect_table_columns_to_match_set_expectation_configuration_builder, ] @@ -270,7 +270,7 @@ def _build_numeric_columns_rule() -> Rule: # Step-3: Declare "ParameterBuilder" for every "validation" need in "ExpectationConfigurationBuilder" objects. - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] suite_parameter_builder_configs = [ ParameterBuilderConfig( @@ -343,7 +343,7 @@ def _build_numeric_columns_rule() -> Rule: # Step-4: Pass "validation" "ParameterBuilderConfig" objects to every "DefaultExpectationConfigurationBuilder", responsible for emitting "ExpectationConfiguration" (with specified "expectation_type"). - validation_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + validation_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] validation_parameter_builder_configs = [ ParameterBuilderConfig( @@ -505,7 +505,7 @@ def _build_numeric_columns_rule() -> Rule: }, "round_decimals": None, } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ column_histogram_single_batch_parameter_builder_for_metrics, column_min_metric_multi_batch_parameter_builder_for_metrics, column_max_metric_multi_batch_parameter_builder_for_metrics, @@ -514,7 +514,7 @@ def _build_numeric_columns_rule() -> Rule: column_mean_metric_multi_batch_parameter_builder_for_metrics, column_standard_deviation_metric_multi_batch_parameter_builder_for_metrics, ] - expectation_configuration_builders: List[ExpectationConfigurationBuilder] = [ + expectation_configuration_builders: list[ExpectationConfigurationBuilder] = [ expect_column_min_to_be_between_expectation_configuration_builder, expect_column_max_to_be_between_expectation_configuration_builder, expect_column_values_to_be_between_expectation_configuration_builder, @@ -565,7 +565,7 @@ def _build_categorical_columns_rule() -> Rule: # Step-3: Declare "ParameterBuilder" for every "validation" need in "ExpectationConfigurationBuilder" objects. - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] suite_parameter_builder_configs = [ ParameterBuilderConfig( @@ -585,9 +585,9 @@ def _build_categorical_columns_rule() -> Rule: # Step-4: Pass "validation" "ParameterBuilderConfig" objects to every "DefaultExpectationConfigurationBuilder", responsible for emitting "ExpectationConfiguration" (with specified "expectation_type"). - validation_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + validation_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] - validation_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = [ + validation_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = [ ParameterBuilderConfig( **column_distinct_values_count_range_parameter_builder_for_validations.to_json_dict(), ), @@ -644,7 +644,7 @@ def _build_categorical_columns_rule() -> Rule: }, "round_decimals": None, } - expectation_configuration_builders: List[ExpectationConfigurationBuilder] = [ + expectation_configuration_builders: list[ExpectationConfigurationBuilder] = [ expect_column_unique_value_count_to_be_between_expectation_configuration_builder, expect_column_proportion_of_unique_values_to_be_between_expectation_configuration_builder, ] diff --git a/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant/statistics_data_assistant.py b/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant/statistics_data_assistant.py index 781ccb52b752..a7d18318cb1a 100644 --- a/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant/statistics_data_assistant.py +++ b/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant/statistics_data_assistant.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import Any, Optional from contrib.experimental.great_expectations_experimental.rule_based_profiler.data_assistant_result import ( StatisticsDataAssistantResult, @@ -51,14 +51,14 @@ def __init__( validator=validator, ) - def get_variables(self) -> Optional[Dict[str, Any]]: + def get_variables(self) -> Optional[dict[str, Any]]: """ Returns: Optional "variables" configuration attribute name/value pairs (overrides), commonly-used in Builder objects. """ return None - def get_rules(self) -> Optional[List[Rule]]: + def get_rules(self) -> Optional[list[Rule]]: """ Returns: Optional custom list of "Rule" objects implementing particular "DataAssistant" functionality. @@ -114,7 +114,7 @@ def _build_table_rule() -> Rule: # Step-3: Declare "ParameterBuilder" configurations for all additional statistics needed. - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = [ + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = [ ParameterBuilderConfig( **table_row_count_metric_multi_batch_parameter_builder_for_metrics.to_json_dict() ), @@ -153,7 +153,7 @@ def _build_table_rule() -> Rule: "exact_match": None, "success_ratio": 1.0, } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ table_row_count_metric_multi_batch_parameter_builder_for_metrics, table_columns_metric_multi_batch_parameter_builder_for_metrics, table_row_count_range_parameter_builder_for_validations, @@ -199,14 +199,14 @@ def _build_column_integrity_rule( # Step-3: Declare "ParameterBuilder" configurations for all additional statistics needed. - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] if total_count_metric_multi_batch_parameter_builder_for_evaluations is None: total_count_metric_multi_batch_parameter_builder_for_evaluations = DataAssistant.commonly_used_parameter_builders.get_table_row_count_metric_multi_batch_parameter_builder() column_values_nonnull_unexpected_count_metric_multi_batch_parameter_builder_for_evaluations = column_values_nonnull_unexpected_count_metric_multi_batch_parameter_builder_for_metrics - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = [ + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = [ ParameterBuilderConfig( **total_count_metric_multi_batch_parameter_builder_for_evaluations.to_json_dict() ), @@ -256,7 +256,7 @@ def _build_column_integrity_rule( variables: dict = { "success_ratio": 7.5e-1, } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ column_values_unique_mean_unexpected_value_multi_batch_parameter_builder_for_validations, column_values_null_mean_unexpected_value_multi_batch_parameter_builder_for_validations, column_values_nonnull_mean_unexpected_value_multi_batch_parameter_builder_for_validations, @@ -309,7 +309,7 @@ def _build_numeric_columns_rule() -> Rule: # Step-3: Declare "ParameterBuilder" configurations for all additional statistics needed. - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] suite_parameter_builder_configs = [ ParameterBuilderConfig( @@ -412,7 +412,7 @@ def _build_numeric_columns_rule() -> Rule: }, "round_decimals": 15, } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ column_min_values_range_parameter_builder_for_validations, column_max_values_range_parameter_builder_for_validations, column_quantile_values_range_parameter_builder_for_validations, @@ -461,7 +461,7 @@ def _build_datetime_columns_rule() -> Rule: # Step-3: Declare "ParameterBuilder" configurations for all additional statistics needed. - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] suite_parameter_builder_configs = [ ParameterBuilderConfig( @@ -507,7 +507,7 @@ def _build_datetime_columns_rule() -> Rule: }, "round_decimals": 1, } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ column_min_values_range_parameter_builder_for_validations, column_max_values_range_parameter_builder_for_validations, ] @@ -549,7 +549,7 @@ def _build_text_columns_rule() -> Rule: # Step-3: Declare "ParameterBuilder" configurations for all additional statistics needed. - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] suite_parameter_builder_configs = [ ParameterBuilderConfig( @@ -596,7 +596,7 @@ def _build_text_columns_rule() -> Rule: "round_decimals": 0, "success_ratio": 7.5e-1, } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ column_min_length_range_parameter_builder_for_validations, column_max_length_range_parameter_builder_for_validations, ] @@ -670,7 +670,7 @@ def _build_categorical_columns_rule() -> Rule: }, "round_decimals": 15, } - parameter_builders: List[ParameterBuilder] = [ + parameter_builders: list[ParameterBuilder] = [ column_unique_proportion_range_parameter_builder_for_validations, ] rule = Rule( diff --git a/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant_result/growth_numeric_data_assistant_result.py b/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant_result/growth_numeric_data_assistant_result.py index eaf01da91dd0..471761211d8d 100644 --- a/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant_result/growth_numeric_data_assistant_result.py +++ b/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant_result/growth_numeric_data_assistant_result.py @@ -1,4 +1,4 @@ -from typing import Dict, Tuple, Union +from typing import Union from great_expectations.experimental.rule_based_profiler.altair import AltairDataTypes from great_expectations.experimental.rule_based_profiler.data_assistant_result import ( @@ -12,7 +12,7 @@ class GrowthNumericDataAssistantResult(DataAssistantResult): """ @property - def metric_expectation_map(self) -> Dict[Union[str, Tuple[str]], str]: + def metric_expectation_map(self) -> dict[Union[str, tuple[str]], str]: """ A mapping is defined for which metrics to plot and their associated expectations. """ @@ -30,7 +30,7 @@ def metric_expectation_map(self) -> Dict[Union[str, Tuple[str]], str]: } @property - def metric_types(self) -> Dict[str, AltairDataTypes]: + def metric_types(self) -> dict[str, AltairDataTypes]: """ A mapping is defined for the Altair data type associated with each metric. """ diff --git a/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant_result/statistics_data_assistant_result.py b/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant_result/statistics_data_assistant_result.py index 932665415a7a..b7253584c780 100644 --- a/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant_result/statistics_data_assistant_result.py +++ b/contrib/experimental/great_expectations_experimental/rule_based_profiler/data_assistant_result/statistics_data_assistant_result.py @@ -1,4 +1,4 @@ -from typing import Dict, Tuple, Union +from typing import Union from great_expectations.experimental.rule_based_profiler.altair import AltairDataTypes from great_expectations.experimental.rule_based_profiler.data_assistant_result import ( @@ -12,14 +12,14 @@ class StatisticsDataAssistantResult(DataAssistantResult): """ @property - def metric_expectation_map(self) -> Dict[Union[str, Tuple[str]], str]: + def metric_expectation_map(self) -> dict[Union[str, tuple[str]], str]: """ A mapping is defined for which metrics to plot and their associated expectations. """ return {} @property - def metric_types(self) -> Dict[str, AltairDataTypes]: + def metric_types(self) -> dict[str, AltairDataTypes]: """ A mapping is defined for the Altair data type associated with each metric. """ diff --git a/contrib/experimental/great_expectations_experimental/tests/rule_based_profiler/data_assistant/test_growth_numeric_data_assistant.py b/contrib/experimental/great_expectations_experimental/tests/rule_based_profiler/data_assistant/test_growth_numeric_data_assistant.py index b1ecd14616a7..1f87d7529426 100644 --- a/contrib/experimental/great_expectations_experimental/tests/rule_based_profiler/data_assistant/test_growth_numeric_data_assistant.py +++ b/contrib/experimental/great_expectations_experimental/tests/rule_based_profiler/data_assistant/test_growth_numeric_data_assistant.py @@ -1,5 +1,5 @@ import os -from typing import Dict, Optional, cast +from typing import Optional, cast import pytest from freezegun import freeze_time @@ -146,7 +146,7 @@ def test_growth_numeric_data_assistant_metrics_count( bobby_growth_numeric_data_assistant_result: GrowthNumericDataAssistantResult, ) -> None: domain: Domain - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] num_metrics: int domain_key = Domain( @@ -177,11 +177,11 @@ def test_growth_numeric_data_assistant_metrics_count( def test_growth_numeric_data_assistant_result_batch_id_to_batch_identifier_display_name_map_coverage( bobby_growth_numeric_data_assistant_result: GrowthNumericDataAssistantResult, ): - metrics_by_domain: Optional[Dict[Domain, Dict[str, ParameterNode]]] = ( + metrics_by_domain: Optional[dict[Domain, dict[str, ParameterNode]]] = ( bobby_growth_numeric_data_assistant_result.metrics_by_domain ) - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] parameter_node: ParameterNode batch_id: str assert all( diff --git a/contrib/experimental/great_expectations_experimental/tests/rule_based_profiler/data_assistant/test_statistics_data_assistant.py b/contrib/experimental/great_expectations_experimental/tests/rule_based_profiler/data_assistant/test_statistics_data_assistant.py index 609fb1146c24..48d0cc4cc101 100644 --- a/contrib/experimental/great_expectations_experimental/tests/rule_based_profiler/data_assistant/test_statistics_data_assistant.py +++ b/contrib/experimental/great_expectations_experimental/tests/rule_based_profiler/data_assistant/test_statistics_data_assistant.py @@ -1,6 +1,6 @@ import math import os -from typing import Any, Dict, Optional, cast +from typing import Any, Optional, cast import numpy as np import pytest @@ -79,7 +79,7 @@ def test_statistics_data_assistant_metrics_count( bobby_statistics_data_assistant_result: StatisticsDataAssistantResult, ) -> None: domain: Domain - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] num_metrics: int domain_key = Domain( @@ -110,11 +110,11 @@ def test_statistics_data_assistant_metrics_count( def test_statistics_data_assistant_result_batch_id_to_batch_identifier_display_name_map_coverage( bobby_statistics_data_assistant_result: StatisticsDataAssistantResult, ): - metrics_by_domain: Optional[Dict[Domain, Dict[str, ParameterNode]]] = ( + metrics_by_domain: Optional[dict[Domain, dict[str, ParameterNode]]] = ( bobby_statistics_data_assistant_result.metrics_by_domain ) - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] parameter_node: ParameterNode batch_id: str assert all( @@ -145,7 +145,7 @@ def test_statistics_data_assistant_result_normalized_metrics_vector_output( larger dataset, but working with smaller datasets is more efficient (data exploration, Expectation authoring, etc.). """ domain: Domain - metrics: Dict[str, ParameterNode] + metrics: dict[str, ParameterNode] parameter_name: str parameter_node: ParameterNode parameter_value: Any diff --git a/contrib/experimental/great_expectations_experimental/tests/test_utils.py b/contrib/experimental/great_expectations_experimental/tests/test_utils.py index cec49ada0c8a..61aa3a440d66 100644 --- a/contrib/experimental/great_expectations_experimental/tests/test_utils.py +++ b/contrib/experimental/great_expectations_experimental/tests/test_utils.py @@ -1,5 +1,4 @@ import os -from typing import List # noinspection PyUnresolvedReferences from great_expectations.data_context.util import file_relative_path @@ -20,7 +19,7 @@ def load_data_into_postgres_database(sa): from tests.test_utils import load_data_into_test_database file_name: str - data_paths: List[str] = [ + data_paths: list[str] = [ file_relative_path( __file__, os.path.join( # noqa: PTH118 @@ -71,7 +70,7 @@ def load_data_into_postgres_database(sa): # 2020 data file_name = "yellow_tripdata_sample_2020-01.csv" - data_paths: List[str] = [ + data_paths: list[str] = [ file_relative_path( __file__, os.path.join( # noqa: PTH118 diff --git a/contrib/great_expectations_ethical_ai_expectations/great_expectations_ethical_ai_expectations/expectations/expect_table_binary_label_model_bias.py b/contrib/great_expectations_ethical_ai_expectations/great_expectations_ethical_ai_expectations/expectations/expect_table_binary_label_model_bias.py index 0fdd70fc881c..e6c90889e42b 100644 --- a/contrib/great_expectations_ethical_ai_expectations/great_expectations_ethical_ai_expectations/expectations/expect_table_binary_label_model_bias.py +++ b/contrib/great_expectations_ethical_ai_expectations/great_expectations_ethical_ai_expectations/expectations/expect_table_binary_label_model_bias.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple +from typing import Any, Optional from aequitas.bias import Bias from aequitas.fairness import Fairness @@ -30,10 +30,10 @@ class TableEvaluateBinaryLabelModelBias(TableMetricProvider): def _pandas( cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[Tuple, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[tuple, Any], + runtime_configuration: dict, ): y_true = metric_value_kwargs.get("y_true") y_pred = metric_value_kwargs.get("y_pred") diff --git a/contrib/great_expectations_ethical_ai_expectations/great_expectations_ethical_ai_expectations/expectations/expect_table_linear_feature_importances_to_be.py b/contrib/great_expectations_ethical_ai_expectations/great_expectations_ethical_ai_expectations/expectations/expect_table_linear_feature_importances_to_be.py index 4d29bc8c1dd7..e2ae419e91a1 100644 --- a/contrib/great_expectations_ethical_ai_expectations/great_expectations_ethical_ai_expectations/expectations/expect_table_linear_feature_importances_to_be.py +++ b/contrib/great_expectations_ethical_ai_expectations/great_expectations_ethical_ai_expectations/expectations/expect_table_linear_feature_importances_to_be.py @@ -2,7 +2,7 @@ # For most Expectations, the main business logic for calculation will live here. # To learn about the relationship between Metrics and Expectations, please visit # https://docs.greatexpectations.io/en/latest/reference/core_concepts.html#expectations-and-metrics. -from typing import Any, Dict, Optional, Tuple +from typing import Any, Optional from sklearn.inspection import permutation_importance from sklearn.linear_model import LinearRegression @@ -32,10 +32,10 @@ class TableModelingRidgeFeatureImportances(TableMetricProvider): def _pandas( cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[Tuple, Any], - runtime_configuration: Dict, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[tuple, Any], + runtime_configuration: dict, ): df, _, _ = execution_engine.get_compute_domain( metric_domain_kwargs, domain_type=MetricDomainTypes.TABLE diff --git a/contrib/great_expectations_ethical_ai_expectations/setup.py b/contrib/great_expectations_ethical_ai_expectations/setup.py index 5827d5150826..f512282d4e01 100644 --- a/contrib/great_expectations_ethical_ai_expectations/setup.py +++ b/contrib/great_expectations_ethical_ai_expectations/setup.py @@ -1,9 +1,7 @@ -from typing import List - import setuptools -def get_requirements() -> List[str]: +def get_requirements() -> list[str]: with open("requirements.txt") as f: requirements = f.read().splitlines() return requirements diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_average_lat_lon_pairwise_distance_to_be_less_than.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_average_lat_lon_pairwise_distance_to_be_less_than.py index c11848c26d3a..89473871e59a 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_average_lat_lon_pairwise_distance_to_be_less_than.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_average_lat_lon_pairwise_distance_to_be_less_than.py @@ -1,5 +1,4 @@ from math import radians -from typing import Dict import numpy as np from scipy.spatial.distance import pdist @@ -139,7 +138,7 @@ class ExpectColumnAverageLatLonPairwiseDistanceToBeLessThan(ColumnAggregateExpec # This method performs a validation of your metrics against your success keys, returning a dict indicating the success or failure of the Expectation. def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_average_to_be_within_range_of_given_point.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_average_to_be_within_range_of_given_point.py index d0bf277cf3d6..75ad115613a4 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_average_to_be_within_range_of_given_point.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_average_to_be_within_range_of_given_point.py @@ -6,7 +6,7 @@ from math import cos, sqrt from statistics import mean -from typing import Any, Dict, List, Union +from typing import Any, Union from great_expectations.core import ExpectationValidationResult from great_expectations.execution_engine import ExecutionEngine, PandasExecutionEngine @@ -127,7 +127,7 @@ class ExpectColumnAverageToBeWithinRangeOfGivenPoint(ColumnAggregateExpectation) # This method performs a validation of your metrics against your success keys, returning a dict indicating the success or failure of the Expectation. def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): @@ -159,7 +159,7 @@ def _prescriptive_renderer( result: ExpectationValidationResult = None, runtime_configuration: dict = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_minimum_bounding_radius_to_be_between.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_minimum_bounding_radius_to_be_between.py index 8718ac92c321..e3f1cde687b7 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_minimum_bounding_radius_to_be_between.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_minimum_bounding_radius_to_be_between.py @@ -1,5 +1,3 @@ -from typing import Dict - import pandas as pd import pygeos as geos @@ -206,7 +204,7 @@ class ExpectColumnMinimumBoundingRadiusToBeBetween(ColumnAggregateExpectation): # This method performs a validation of your metrics against your success keys, returning a dict indicating the success or failure of the Expectation. def _validate( # noqa: C901 - too complex self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_pair_values_lat_lng_matches_geohash.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_pair_values_lat_lng_matches_geohash.py index ba724cd944f2..8960d105aa47 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_pair_values_lat_lng_matches_geohash.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_pair_values_lat_lng_matches_geohash.py @@ -1,4 +1,4 @@ -from typing import ClassVar, List, Literal, Tuple +from typing import ClassVar, Literal import geohash as gh import pandas as pd @@ -67,7 +67,7 @@ class ExpectColumnPairValuesLatLngMatchesGeohash(ColumnPairMapExpectation): "both_values_are_missing" ) - examples: ClassVar[List[dict]] = [ + examples: ClassVar[list[dict]] = [ { "data": { "latlngs": [ @@ -112,7 +112,7 @@ class ExpectColumnPairValuesLatLngMatchesGeohash(ColumnPairMapExpectation): ] map_metric: ClassVar[str] = "column_pair_values.lat_lng_matches_geohash" - success_keys: ClassVar[Tuple[str, ...]] = ( + success_keys: ClassVar[tuple[str, ...]] = ( "column_A", "column_B", "ignore_row_if", diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_geometry_to_be_of_type.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_geometry_to_be_of_type.py index 958b74e84d5a..e346414df98f 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_geometry_to_be_of_type.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_geometry_to_be_of_type.py @@ -1,4 +1,4 @@ -from typing import Any, List, Union +from typing import Any, Union from shapely.geometry import LineString, MultiPolygon, Point, Polygon, mapping, shape @@ -149,7 +149,7 @@ def _prescriptive_renderer( result: ExpectationValidationResult = None, runtime_configuration: dict = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_lat_lon_to_be_land_or_ocean.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_lat_lon_to_be_land_or_ocean.py index 21c562ed226c..9249a605b703 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_lat_lon_to_be_land_or_ocean.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_lat_lon_to_be_land_or_ocean.py @@ -1,4 +1,4 @@ -from typing import Any, List, Union +from typing import Any, Union from global_land_mask import globe @@ -155,7 +155,7 @@ def _prescriptive_renderer( result: ExpectationValidationResult = None, runtime_configuration: dict = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_reverse_geocoded_lat_lon_to_contain.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_reverse_geocoded_lat_lon_to_contain.py index a8bbcfe549dc..802400abe597 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_reverse_geocoded_lat_lon_to_contain.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_reverse_geocoded_lat_lon_to_contain.py @@ -1,4 +1,4 @@ -from typing import Any, List, Optional, Union +from typing import Any, Optional, Union import geopandas from shapely.geometry import Point @@ -172,7 +172,7 @@ def _prescriptive_renderer( result: ExpectationValidationResult = None, runtime_configuration: dict = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_lat_lon_coordinates_in_range_of_given_point.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_lat_lon_coordinates_in_range_of_given_point.py index 156e2fdfc037..403c701f19b4 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_lat_lon_coordinates_in_range_of_given_point.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_lat_lon_coordinates_in_range_of_given_point.py @@ -5,7 +5,7 @@ """ from math import cos, pi, sqrt -from typing import Any, List, Optional, Union +from typing import Any, Optional, Union from great_expectations.compatibility import pyspark from great_expectations.compatibility.pyspark import functions as F @@ -428,7 +428,7 @@ def _prescriptive_renderer( result: ExpectationValidationResult = None, runtime_configuration: dict = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_nonempty_geometries.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_nonempty_geometries.py index b63842cc17a0..bca12054d986 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_nonempty_geometries.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_nonempty_geometries.py @@ -1,4 +1,4 @@ -from typing import Any, List, Union +from typing import Any, Union from shapely.geometry import LineString, MultiPolygon, Point, Polygon, mapping, shape @@ -123,7 +123,7 @@ def _prescriptive_renderer( result: ExpectationValidationResult = None, runtime_configuration: dict = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_polygon_area_between.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_polygon_area_between.py index 99613c40ea16..4ad695508ad2 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_polygon_area_between.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_polygon_area_between.py @@ -1,4 +1,4 @@ -from typing import Any, List, Union +from typing import Any, Union import geopandas from shapely.geometry import mapping, shape @@ -200,7 +200,7 @@ def _prescriptive_renderer( result: ExpectationValidationResult = None, runtime_configuration: dict = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_valid_geojson.py b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_valid_geojson.py index e56636426f0c..c17af799d55f 100644 --- a/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_valid_geojson.py +++ b/contrib/great_expectations_geospatial_expectations/great_expectations_geospatial_expectations/expectations/expect_column_values_to_be_valid_geojson.py @@ -1,4 +1,4 @@ -from typing import Any, List, Union +from typing import Any, Union import pygeos @@ -139,7 +139,7 @@ def _prescriptive_renderer( result: ExpectationValidationResult = None, runtime_configuration: dict = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/contrib/great_expectations_geospatial_expectations/setup.py b/contrib/great_expectations_geospatial_expectations/setup.py index b1eeeca95cfa..2041d1b267bb 100644 --- a/contrib/great_expectations_geospatial_expectations/setup.py +++ b/contrib/great_expectations_geospatial_expectations/setup.py @@ -1,9 +1,7 @@ -from typing import List - import setuptools -def get_requirements() -> List[str]: +def get_requirements() -> list[str]: with open("requirements.txt") as f: requirements = f.read().splitlines() return requirements diff --git a/contrib/great_expectations_semantic_types_expectations/great_expectations_semantic_types_expectations/expectations/expect_column_values_to_be_valid_currency_code.py b/contrib/great_expectations_semantic_types_expectations/great_expectations_semantic_types_expectations/expectations/expect_column_values_to_be_valid_currency_code.py index 95acc33b7977..7e76e7e6782d 100644 --- a/contrib/great_expectations_semantic_types_expectations/great_expectations_semantic_types_expectations/expectations/expect_column_values_to_be_valid_currency_code.py +++ b/contrib/great_expectations_semantic_types_expectations/great_expectations_semantic_types_expectations/expectations/expect_column_values_to_be_valid_currency_code.py @@ -1,5 +1,3 @@ -from typing import Tuple - from moneyed import list_all_currencies from great_expectations.compatibility.pyspark import functions as F @@ -15,11 +13,11 @@ ) -def generate_all_currency_codes() -> Tuple[str]: +def generate_all_currency_codes() -> tuple[str]: return [str(currency_code) for currency_code in list_all_currencies()] -def is_valid_currency_code(code: str, currency_codes: Tuple[str]) -> bool: +def is_valid_currency_code(code: str, currency_codes: tuple[str]) -> bool: return code in currency_codes @@ -32,7 +30,7 @@ class ColumnValuesCurrencyCode(ColumnMapMetricProvider): # This method implements the core logic for the PandasExecutionEngine @column_condition_partial(engine=PandasExecutionEngine) def _pandas(cls, column, **kwargs): - currency_codes: Tuple[str] = generate_all_currency_codes() + currency_codes: tuple[str] = generate_all_currency_codes() return column.apply(lambda x: is_valid_currency_code(x, currency_codes)) @@ -44,7 +42,7 @@ def _pandas(cls, column, **kwargs): # This method defines the business logic for evaluating your metric when using a SparkDFExecutionEngine @column_condition_partial(engine=SparkDFExecutionEngine) def _spark(cls, column, **kwargs): - currency_codes: Tuple[str] = generate_all_currency_codes() + currency_codes: tuple[str] = generate_all_currency_codes() # Register the UDF is_valid_currency_code_udf = F.udf(is_valid_currency_code, types.BooleanType()) diff --git a/contrib/great_expectations_semantic_types_expectations/setup.py b/contrib/great_expectations_semantic_types_expectations/setup.py index 9015369ecb14..70b639d5eec9 100644 --- a/contrib/great_expectations_semantic_types_expectations/setup.py +++ b/contrib/great_expectations_semantic_types_expectations/setup.py @@ -1,9 +1,7 @@ -from typing import List - import setuptools -def get_requirements() -> List[str]: +def get_requirements() -> list[str]: with open("requirements.txt") as f: requirements = f.read().splitlines() return requirements diff --git a/contrib/great_expectations_zipcode_expectations/setup.py b/contrib/great_expectations_zipcode_expectations/setup.py index 534c439d0c23..9c104b92fae0 100644 --- a/contrib/great_expectations_zipcode_expectations/setup.py +++ b/contrib/great_expectations_zipcode_expectations/setup.py @@ -1,9 +1,7 @@ -from typing import List - import setuptools -def get_requirements() -> List[str]: +def get_requirements() -> list[str]: with open("requirements.txt") as f: requirements = f.read().splitlines() return requirements diff --git a/contrib/time_series_expectations/setup.py b/contrib/time_series_expectations/setup.py index 541f467498fb..3ef57277b1b9 100644 --- a/contrib/time_series_expectations/setup.py +++ b/contrib/time_series_expectations/setup.py @@ -1,9 +1,7 @@ -from typing import List - import setuptools -def get_requirements() -> List[str]: +def get_requirements() -> list[str]: with open("requirements.txt") as f: requirements = f.read().splitlines() return requirements diff --git a/contrib/time_series_expectations/time_series_expectations/expectations/column_aggregate_time_series_expectation.py b/contrib/time_series_expectations/time_series_expectations/expectations/column_aggregate_time_series_expectation.py index f1296dd403ed..acc1a491f10e 100644 --- a/contrib/time_series_expectations/time_series_expectations/expectations/column_aggregate_time_series_expectation.py +++ b/contrib/time_series_expectations/time_series_expectations/expectations/column_aggregate_time_series_expectation.py @@ -1,5 +1,4 @@ from abc import ABC -from typing import Dict import pandas as pd @@ -39,7 +38,7 @@ def metric_dependencies(self): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/time_series_expectations/time_series_expectations/expectations/expect_batch_row_count_to_match_prophet_date_model.py b/contrib/time_series_expectations/time_series_expectations/expectations/expect_batch_row_count_to_match_prophet_date_model.py index 26da29b81ccf..30d1b2973cda 100644 --- a/contrib/time_series_expectations/time_series_expectations/expectations/expect_batch_row_count_to_match_prophet_date_model.py +++ b/contrib/time_series_expectations/time_series_expectations/expectations/expect_batch_row_count_to_match_prophet_date_model.py @@ -1,5 +1,3 @@ -from typing import Dict - import pandas as pd from great_expectations.data_context.util import file_relative_path @@ -98,7 +96,7 @@ class ExpectBatchRowCountToMatchProphetDateModel(BatchExpectation): def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: dict = None, execution_engine: ExecutionEngine = None, ): diff --git a/contrib/time_series_expectations/time_series_expectations/generator/daily_time_series_generator.py b/contrib/time_series_expectations/time_series_expectations/generator/daily_time_series_generator.py index 2490ba5ac2ce..6daf1abfbc4e 100644 --- a/contrib/time_series_expectations/time_series_expectations/generator/daily_time_series_generator.py +++ b/contrib/time_series_expectations/time_series_expectations/generator/daily_time_series_generator.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple +from typing import Optional import numpy as np import pandas as pd @@ -15,7 +15,7 @@ class DailyTimeSeriesGenerator(TimeSeriesGenerator): def _generate_trend( self, date_range: np.ndarray, - trend_params: List[TrendParams], + trend_params: list[TrendParams], ) -> np.ndarray: """Generate a trend component for a time series.""" @@ -31,7 +31,7 @@ def _generate_trend( def _generate_weekday_seasonality( self, date_range: np.ndarray, - weekday_dummy_params: List[float], + weekday_dummy_params: list[float], ) -> np.ndarray: """Generate a weekday seasonality component for a time series.""" @@ -40,7 +40,7 @@ def _generate_weekday_seasonality( def _generate_annual_seasonality( self, date_range: np.ndarray, - annual_seasonality_params: List[Tuple[float, float]], + annual_seasonality_params: list[tuple[float, float]], ) -> np.ndarray: """Generate an annual seasonality component for a time series.""" @@ -65,9 +65,9 @@ def _generate_posneg_pareto( def _generate_component_time_series( self, size: int, - trend_params: List[TrendParams], - weekday_dummy_params: List[float], - annual_seasonality_params: List[Tuple[float, float]], + trend_params: list[TrendParams], + weekday_dummy_params: list[float], + annual_seasonality_params: list[tuple[float, float]], holiday_alpha: float, outlier_alpha: float, noise_scale: float, @@ -97,9 +97,9 @@ def _generate_component_time_series( def _generate_daily_time_series( self, size: int = 365 * 3, - trend_params: Optional[List[TrendParams]] = None, - weekday_dummy_params: Optional[List[float]] = None, - annual_seasonality_params: Optional[List[Tuple[float, float]]] = None, + trend_params: Optional[list[TrendParams]] = None, + weekday_dummy_params: Optional[list[float]] = None, + annual_seasonality_params: Optional[list[tuple[float, float]]] = None, holiday_alpha: float = 3.5, outlier_alpha: float = 2.5, noise_scale: float = 1.0, @@ -170,9 +170,9 @@ def generate_df( self, size: Optional[int] = 365 * 3, start_date: Optional[str] = "2018-01-01", - trend_params: Optional[List[TrendParams]] = None, - weekday_dummy_params: Optional[List[float]] = None, - annual_seasonality_params: Optional[List[Tuple[float, float]]] = None, + trend_params: Optional[list[TrendParams]] = None, + weekday_dummy_params: Optional[list[float]] = None, + annual_seasonality_params: Optional[list[tuple[float, float]]] = None, holiday_alpha: float = 3.5, outlier_alpha: float = 2.5, noise_scale: float = 1.0, diff --git a/contrib/time_series_expectations/time_series_expectations/generator/hourly_time_series_generator.py b/contrib/time_series_expectations/time_series_expectations/generator/hourly_time_series_generator.py index 1cac0529f7f8..32343d0b4163 100644 --- a/contrib/time_series_expectations/time_series_expectations/generator/hourly_time_series_generator.py +++ b/contrib/time_series_expectations/time_series_expectations/generator/hourly_time_series_generator.py @@ -1,5 +1,5 @@ from math import ceil -from typing import List, Optional, Tuple +from typing import Optional import numpy as np import pandas as pd @@ -16,7 +16,7 @@ class HourlyTimeSeriesGenerator(DailyTimeSeriesGenerator): def _generate_hourly_seasonality( self, time_range: np.ndarray, - hourly_seasonality_params: List[Tuple[float, float]], + hourly_seasonality_params: list[tuple[float, float]], ) -> np.ndarray: """Generate an annual seasonality component for a time series.""" @@ -30,10 +30,10 @@ def _generate_hourly_time_series( self, size: int, hourly_seasonality: float, - hourly_seasonality_params: Optional[List[Tuple[float, float]]] = None, - trend_params: Optional[List[TrendParams]] = None, - weekday_dummy_params: Optional[List[float]] = None, - annual_seasonality_params: Optional[List[Tuple[float, float]]] = None, + hourly_seasonality_params: Optional[list[tuple[float, float]]] = None, + trend_params: Optional[list[TrendParams]] = None, + weekday_dummy_params: Optional[list[float]] = None, + annual_seasonality_params: Optional[list[tuple[float, float]]] = None, holiday_alpha: float = 3.5, outlier_alpha: float = 2.5, noise_scale: float = 1.0, @@ -80,10 +80,10 @@ def generate_df( size: Optional[int] = 90 * 24, # 90 days worth of data start_date: Optional[str] = "2018-01-01", hourly_seasonality: float = 1.0, - hourly_seasonality_params: Optional[List[Tuple[float, float]]] = None, - trend_params: Optional[List[TrendParams]] = None, - weekday_dummy_params: Optional[List[float]] = None, - annual_seasonality_params: Optional[List[Tuple[float, float]]] = None, + hourly_seasonality_params: Optional[list[tuple[float, float]]] = None, + trend_params: Optional[list[TrendParams]] = None, + weekday_dummy_params: Optional[list[float]] = None, + annual_seasonality_params: Optional[list[tuple[float, float]]] = None, holiday_alpha: float = 3.5, outlier_alpha: float = 2.5, noise_scale: float = 1.0, diff --git a/contrib/time_series_expectations/time_series_expectations/generator/monthly_time_series_generator.py b/contrib/time_series_expectations/time_series_expectations/generator/monthly_time_series_generator.py index 79416efe6bd4..2827f7e15fd2 100644 --- a/contrib/time_series_expectations/time_series_expectations/generator/monthly_time_series_generator.py +++ b/contrib/time_series_expectations/time_series_expectations/generator/monthly_time_series_generator.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple +from typing import Optional import pandas as pd @@ -15,9 +15,9 @@ def generate_df( self, size: Optional[int] = 365 * 3, start_date: Optional[str] = "2018-01-01", - trend_params: Optional[List[TrendParams]] = None, - weekday_dummy_params: Optional[List[float]] = None, - annual_seasonality_params: Optional[List[Tuple[float, float]]] = None, + trend_params: Optional[list[TrendParams]] = None, + weekday_dummy_params: Optional[list[float]] = None, + annual_seasonality_params: Optional[list[tuple[float, float]]] = None, holiday_alpha: float = 3.5, outlier_alpha: float = 2.5, noise_scale: float = 1.0, diff --git a/contrib/time_series_expectations/time_series_expectations/generator/weekly_time_series_generator.py b/contrib/time_series_expectations/time_series_expectations/generator/weekly_time_series_generator.py index 2eb0418ae062..95bcdbca8a42 100644 --- a/contrib/time_series_expectations/time_series_expectations/generator/weekly_time_series_generator.py +++ b/contrib/time_series_expectations/time_series_expectations/generator/weekly_time_series_generator.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Tuple +from typing import Optional import pandas as pd @@ -16,9 +16,9 @@ def generate_df( size: Optional[int] = 52 * 3, day_of_week: Optional[int] = 0, start_date: Optional[str] = "2018-01-01", - trend_params: Optional[List[TrendParams]] = None, - weekday_dummy_params: Optional[List[float]] = None, - annual_seasonality_params: Optional[List[Tuple[float, float]]] = None, + trend_params: Optional[list[TrendParams]] = None, + weekday_dummy_params: Optional[list[float]] = None, + annual_seasonality_params: Optional[list[tuple[float, float]]] = None, holiday_alpha: float = 3.5, outlier_alpha: float = 2.5, noise_scale: float = 1.0, diff --git a/docs/checks/docs_link_checker.py b/docs/checks/docs_link_checker.py index 4ec36cbbfc86..dca22d0499dc 100644 --- a/docs/checks/docs_link_checker.py +++ b/docs/checks/docs_link_checker.py @@ -19,7 +19,7 @@ import pathlib import re import sys -from typing import List, Optional +from typing import Optional import click import requests @@ -309,7 +309,7 @@ def _check_link( # noqa: PLR0912, C901 # too complex return result - def check_file(self, file: pathlib.Path) -> List[LinkReport]: + def check_file(self, file: pathlib.Path) -> list[LinkReport]: """Looks for all the links in a file and checks them. Returns: @@ -320,7 +320,7 @@ def check_file(self, file: pathlib.Path) -> List[LinkReport]: matches = self._markdown_link_pattern.finditer(contents) - result: List[LinkReport] = [] + result: list[LinkReport] = [] for match in matches: report = self._check_link(match, file) @@ -403,7 +403,7 @@ def scan_docs( # noqa: C901, PLR0913 return 1, f"Docs root path: {docs_root} is not a directory" # prepare our return value - result: List[LinkReport] = list() + result: list[LinkReport] = list() checker = LinkChecker( path, docs_root, static_root, site_prefix, static_prefix, skip_external ) diff --git a/docs/docusaurus/docs/snippets/actions.py b/docs/docusaurus/docs/snippets/actions.py index 759b7dd8e588..261fd2c8a701 100644 --- a/docs/docusaurus/docs/snippets/actions.py +++ b/docs/docusaurus/docs/snippets/actions.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, Union from great_expectations.checkpoint.actions import ValidationAction from great_expectations.compatibility.typing_extensions import override @@ -21,7 +21,7 @@ class DocsAction(ValidationAction): def __init__( self, data_context: AbstractDataContext, - site_names: Union[List[str], str, None] = None, + site_names: Union[list[str], str, None] = None, ) -> None: """ :param data_context: Data Context diff --git a/docs/docusaurus/docs/snippets/expect_column_max_to_be_between_custom.py b/docs/docusaurus/docs/snippets/expect_column_max_to_be_between_custom.py index 0e43f1b850ba..ede1a41aeced 100644 --- a/docs/docusaurus/docs/snippets/expect_column_max_to_be_between_custom.py +++ b/docs/docusaurus/docs/snippets/expect_column_max_to_be_between_custom.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from great_expectations.compatibility.pyspark import functions as F from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa @@ -214,7 +214,7 @@ def validate_configuration( # def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: ExecutionEngine = None, ): diff --git a/docs/docusaurus/docs/snippets/expect_column_values_to_equal_three.py b/docs/docusaurus/docs/snippets/expect_column_values_to_equal_three.py index 76ce3f94d506..8230f4633e99 100644 --- a/docs/docusaurus/docs/snippets/expect_column_values_to_equal_three.py +++ b/docs/docusaurus/docs/snippets/expect_column_values_to_equal_three.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from great_expectations.compatibility.pyspark import functions as F from great_expectations.core.metric_domain_types import MetricDomainTypes @@ -98,18 +98,18 @@ def _get_evaluation_dependencies( metric: MetricConfiguration, configuration: Optional[ExpectationConfiguration] = None, execution_engine: Optional[ExecutionEngine] = None, - runtime_configuration: Optional[Dict] = None, + runtime_configuration: Optional[dict] = None, ): """Returns a dictionary of given metric names and their corresponding configuration, specifying the metric types and their respective domains""" - dependencies: Dict = super()._get_evaluation_dependencies( + dependencies: dict = super()._get_evaluation_dependencies( metric=metric, configuration=configuration, execution_engine=execution_engine, runtime_configuration=runtime_configuration, ) - table_domain_kwargs: Dict = { + table_domain_kwargs: dict = { k: v for k, v in metric.metric_domain_kwargs.items() if k != "column" } dependencies["table.column_types"] = MetricConfiguration( diff --git a/docs/sphinx_api_docs_source/build_sphinx_api_docs.py b/docs/sphinx_api_docs_source/build_sphinx_api_docs.py index 2e4074e66f95..74a2b5c812c8 100644 --- a/docs/sphinx_api_docs_source/build_sphinx_api_docs.py +++ b/docs/sphinx_api_docs_source/build_sphinx_api_docs.py @@ -26,7 +26,6 @@ def my_task( import shutil import sys from dataclasses import dataclass -from typing import Dict from urllib.parse import urlparse import invoke @@ -91,8 +90,8 @@ def __init__( self.docusaurus_api_docs_path = self.docs_path / pathlib.Path( "docusaurus/docs/reference/api" ) - self.definitions: Dict[str, Definition] = {} - self.sidebar_entries: Dict[str, SidebarEntry] = {} + self.definitions: dict[str, Definition] = {} + self.sidebar_entries: dict[str, SidebarEntry] = {} self.written_class_md_stubs: dict[ pathlib.Path, list[str] ] = {} # Dict of {path_to_class_def: ["ClassName1", "ClassName2", ...]} diff --git a/docs/sphinx_api_docs_source/check_public_api_docstrings.py b/docs/sphinx_api_docs_source/check_public_api_docstrings.py index 1f00f2b0ff81..c25657877d7b 100644 --- a/docs/sphinx_api_docs_source/check_public_api_docstrings.py +++ b/docs/sphinx_api_docs_source/check_public_api_docstrings.py @@ -15,7 +15,6 @@ import re import subprocess from dataclasses import dataclass -from typing import List, Set, Tuple from .public_api_report import ( CodeParser, @@ -52,10 +51,10 @@ def __str__(self): # type: ignore[explicit-override] # FIXME return self.raw_error -def parse_ruff_errors(raw_errors: List[str]) -> List[DocstringError]: +def parse_ruff_errors(raw_errors: list[str]) -> list[DocstringError]: """Parse raw string output of ruff to DocstringError.""" - docstring_errors: List[DocstringError] = [] + docstring_errors: list[DocstringError] = [] pattern = re.compile(r"^D\d{3}") for raw_error in raw_errors: if not raw_error: @@ -101,7 +100,7 @@ def _repo_relative_filepath(filepath: pathlib.Path) -> pathlib.Path: return filepath -def run_ruff(paths: List[pathlib.Path]) -> List[str]: +def run_ruff(paths: list[pathlib.Path]) -> list[str]: """Run ruff to identify issues with docstrings.""" _log_with_timestamp("Running ruff") @@ -135,7 +134,7 @@ def _log_with_timestamp(content: str) -> None: def _get_docstring_errors( select_paths: list[pathlib.Path] | None = None, -) -> List[DocstringError]: +) -> list[DocstringError]: """Get all docstring errors.""" if select_paths: filepaths_containing_public_api_entities = [p.resolve() for p in select_paths] @@ -149,7 +148,7 @@ def _get_docstring_errors( return parsed_ruff_errors -def get_public_api_definitions() -> Set[Definition]: +def get_public_api_definitions() -> set[Definition]: """Get entities marked with the @public_api decorator.""" code_file_contents = FileContents.create_from_local_files( _default_code_absolute_paths() @@ -162,7 +161,7 @@ def get_public_api_definitions() -> Set[Definition]: return public_api_checker.get_all_public_api_definitions() -def get_public_api_module_level_function_definitions() -> Set[Definition]: +def get_public_api_module_level_function_definitions() -> set[Definition]: """Get module level functions marked with the @public_api decorator.""" code_file_contents = FileContents.create_from_local_files( _default_code_absolute_paths() @@ -177,23 +176,23 @@ def get_public_api_module_level_function_definitions() -> Set[Definition]: def _public_api_docstring_errors( select_paths: list[pathlib.Path] | None = None, -) -> Set[DocstringError]: +) -> set[DocstringError]: """Get all docstring errors for entities marked with the @public_api decorator.""" _log_with_timestamp("Getting public api definitions.") public_api_definitions = get_public_api_definitions() - public_api_definition_tuples: Set[Tuple[str, str]] = { + public_api_definition_tuples: set[tuple[str, str]] = { (str(_repo_relative_filepath(d.filepath)), d.name) for d in public_api_definitions } _log_with_timestamp("Getting docstring errors.") - public_api_docstring_errors: List[DocstringError] = [] + public_api_docstring_errors: list[DocstringError] = [] docstring_errors = _get_docstring_errors(select_paths=select_paths) _log_with_timestamp("Getting docstring errors applicable to public api.") for docstring_error in docstring_errors: - docstring_error_tuple: Tuple[str, str] = ( + docstring_error_tuple: tuple[str, str] = ( str(docstring_error.filepath_relative_to_repo_root), docstring_error.name, ) diff --git a/docs/sphinx_api_docs_source/public_api_report.py b/docs/sphinx_api_docs_source/public_api_report.py index 9e7c040f29f5..d34858f50003 100755 --- a/docs/sphinx_api_docs_source/public_api_report.py +++ b/docs/sphinx_api_docs_source/public_api_report.py @@ -59,7 +59,7 @@ import re import sys from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Set, Union, cast +from typing import TYPE_CHECKING, Union, cast from docs.sphinx_api_docs_source import ( public_api_excludes, @@ -114,17 +114,17 @@ def create_from_local_file(cls, filepath: pathlib.Path) -> FileContents: return cls(filepath=filepath, contents=file_contents) @classmethod - def create_from_local_files(cls, filepaths: Set[pathlib.Path]) -> Set[FileContents]: + def create_from_local_files(cls, filepaths: set[pathlib.Path]) -> set[FileContents]: return {cls.create_from_local_file(filepath) for filepath in filepaths} class DocsExampleParser: """Parse examples from docs to find classes, methods and functions used.""" - def __init__(self, file_contents: Set[FileContents]) -> None: + def __init__(self, file_contents: set[FileContents]) -> None: self.file_contents = file_contents - def get_names_from_usage_in_docs_examples(self) -> Set[str]: + def get_names_from_usage_in_docs_examples(self) -> set[str]: """Get names in docs examples of classes, methods and functions used. Usages are retrieved from imports and function / method calls. @@ -140,7 +140,7 @@ def get_names_from_usage_in_docs_examples(self) -> Set[str]: all_usages |= file_usages return all_usages - def _get_names_of_all_usages_in_file(self, file_contents: FileContents) -> Set[str]: + def _get_names_of_all_usages_in_file(self, file_contents: FileContents) -> set[str]: """Retrieve the names of all class, method + functions used in file_contents.""" tree = ast.parse(file_contents.contents) @@ -161,10 +161,10 @@ def _get_names_of_all_usages_in_file(self, file_contents: FileContents) -> Set[s def _list_all_gx_imports( self, tree: ast.AST - ) -> List[Union[ast.Import, ast.ImportFrom]]: + ) -> list[Union[ast.Import, ast.ImportFrom]]: """Get all the GX related imports in an ast tree.""" - imports: List[Union[ast.Import, ast.ImportFrom]] = [] + imports: list[Union[ast.Import, ast.ImportFrom]] = [] for node in ast.walk(tree): node_is_imported_from_gx = isinstance( @@ -185,8 +185,8 @@ def _list_all_gx_imports( return imports def _get_non_private_gx_import_names( - self, imports: List[Union[ast.Import, ast.ImportFrom]] - ) -> Set[str]: + self, imports: list[Union[ast.Import, ast.ImportFrom]] + ) -> set[str]: """From ast trees, get names of all non private GX related imports.""" names = [] @@ -202,7 +202,7 @@ def _get_non_private_gx_import_names( return set(names) - def _get_all_function_calls(self, tree: ast.AST) -> List[ast.Call]: + def _get_all_function_calls(self, tree: ast.AST) -> list[ast.Call]: """Get all the function calls from an ast tree.""" calls = [] for node in ast.walk(tree): @@ -211,7 +211,7 @@ def _get_all_function_calls(self, tree: ast.AST) -> List[ast.Call]: return calls - def _get_non_private_function_names(self, calls: List[ast.Call]) -> Set[str]: + def _get_non_private_function_names(self, calls: list[ast.Call]) -> set[str]: """Get function names that are not private from ast.Call objects.""" names = [] for call in calls: @@ -232,7 +232,7 @@ class CodeParser: def __init__( self, - file_contents: Set[FileContents], + file_contents: set[FileContents], ) -> None: """Create a CodeParser. @@ -243,7 +243,7 @@ def __init__( def get_all_class_method_and_function_names( self, - ) -> Set[str]: + ) -> set[str]: """Get string names of all classes, methods and functions in all FileContents.""" all_usages = set() for file_contents in self.file_contents: @@ -255,9 +255,9 @@ def get_all_class_method_and_function_names( def _get_all_class_method_and_function_names_from_file_contents( self, file_contents: FileContents - ) -> Set[str]: + ) -> set[str]: """Get string names of all classes, methods and functions in a single FileContents.""" - definitions: Set[Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef]] = ( + definitions: set[Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef]] = ( self._get_all_entity_definitions_from_file_contents( file_contents=file_contents ) @@ -267,9 +267,9 @@ def _get_all_class_method_and_function_names_from_file_contents( def get_all_class_method_and_function_definitions( self, - ) -> Set[Definition]: + ) -> set[Definition]: """Get Definition objects for all class, method and function definitions.""" - all_usages: Set[Definition] = set() + all_usages: set[Definition] = set() for file_contents in self.file_contents: entity_definitions = self._get_all_entity_definitions_from_file_contents( file_contents=file_contents @@ -279,9 +279,9 @@ def get_all_class_method_and_function_definitions( ) return all_usages - def get_module_level_function_definitions(self) -> Set[Definition]: + def get_module_level_function_definitions(self) -> set[Definition]: """Get Definition objects only for functions defined at the module level.""" - all_usages: Set[Definition] = set() + all_usages: set[Definition] = set() for file_contents in self.file_contents: module_level_function_definitions = ( self._get_module_level_function_definitions_from_file_contents( @@ -297,12 +297,12 @@ def get_module_level_function_definitions(self) -> Set[Definition]: def _build_file_usage_definitions( self, file_contents: FileContents, - entity_definitions: Set[ + entity_definitions: set[ Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef] ], - ) -> Set[Definition]: + ) -> set[Definition]: """Build Definitions from FileContents.""" - file_usages_definitions: List[Definition] = [] + file_usages_definitions: list[Definition] = [] for usage in entity_definitions: candidate_definition = Definition( name=usage.name, @@ -315,10 +315,10 @@ def _build_file_usage_definitions( def _get_all_entity_definitions_from_file_contents( self, file_contents: FileContents - ) -> Set[Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef]]: + ) -> set[Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef]]: """Parse FileContents to retrieve entity definitions as ast trees.""" tree = ast.parse(file_contents.contents) - all_defs: List[Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef]] = [] + all_defs: list[Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef]] = [] all_defs.extend(self._list_class_definitions(tree=tree)) all_defs.extend(self._list_function_definitions(tree=tree)) @@ -326,13 +326,13 @@ def _get_all_entity_definitions_from_file_contents( def _get_module_level_function_definitions_from_file_contents( self, file_contents: FileContents - ) -> Set[Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef]]: + ) -> set[Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef]]: """Parse FileContents to retrieve module level function definitions as ast trees.""" tree = ast.parse(file_contents.contents) defs = self._list_module_level_function_definitions(tree=tree) return set(defs) - def _list_class_definitions(self, tree: ast.AST) -> List[ast.ClassDef]: + def _list_class_definitions(self, tree: ast.AST) -> list[ast.ClassDef]: """List class definitions from an ast tree.""" class_defs = [] @@ -345,7 +345,7 @@ def _list_class_definitions(self, tree: ast.AST) -> List[ast.ClassDef]: def _list_function_definitions( self, tree: ast.AST - ) -> List[Union[ast.FunctionDef, ast.AsyncFunctionDef]]: + ) -> list[Union[ast.FunctionDef, ast.AsyncFunctionDef]]: """List function definitions from an ast tree.""" function_definitions = [] for node in ast.walk(tree): @@ -356,7 +356,7 @@ def _list_function_definitions( def _list_module_level_function_definitions( self, tree: ast.AST - ) -> List[Union[ast.FunctionDef, ast.AsyncFunctionDef]]: + ) -> list[Union[ast.FunctionDef, ast.AsyncFunctionDef]]: """List function definitions that appear outside of classes.""" function_definitions = [] @@ -367,7 +367,7 @@ def _list_module_level_function_definitions( return function_definitions -def parse_docs_contents_for_class_names(file_contents: Set[FileContents]) -> Set[str]: +def parse_docs_contents_for_class_names(file_contents: set[FileContents]) -> set[str]: """Parse contents of documentation for class names. Parses based on class names used in yaml examples e.g. Datasource and @@ -445,7 +445,7 @@ def get_shortest_dotted_path( return shortest_path -def _get_import_names(code: str) -> List[str]: +def _get_import_names(code: str) -> list[str]: """Get import names from import statements. Args: @@ -475,9 +475,9 @@ def __init__( ) -> None: self.code_parser = code_parser - def get_all_public_api_definitions(self) -> Set[Definition]: + def get_all_public_api_definitions(self) -> set[Definition]: """Get definitions that are marked with the public api decorator.""" - definitions: List[Definition] = [] + definitions: list[Definition] = [] for ( definition @@ -487,9 +487,9 @@ def get_all_public_api_definitions(self) -> Set[Definition]: return set(definitions) - def get_module_level_function_public_api_definitions(self) -> Set[Definition]: + def get_module_level_function_public_api_definitions(self) -> set[Definition]: """Get module level function definitions that are marked with the public api decorator.""" - definitions: List[Definition] = [] + definitions: list[Definition] = [] for definition in self.code_parser.get_module_level_function_definitions(): if self.is_definition_marked_public_api(definition): @@ -512,7 +512,7 @@ def is_definition_marked_public_api(self, definition: Definition) -> bool: def _get_decorator_names( self, ast_definition: Union[ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef] - ) -> Set[str]: + ) -> set[str]: """Get all decorator names for a single definition from an ast tree.""" def flatten_attr(node): @@ -549,8 +549,8 @@ def __init__( # noqa: PLR0913 code_parser: CodeParser, public_api_checker: PublicAPIChecker, references_from_docs_content: set[str] | None = None, - excludes: Union[List[IncludeExcludeDefinition], None] = None, - includes: Union[List[IncludeExcludeDefinition], None] = None, + excludes: Union[list[IncludeExcludeDefinition], None] = None, + includes: Union[list[IncludeExcludeDefinition], None] = None, ) -> None: """Create a CodeReferenceFilter. @@ -586,7 +586,7 @@ def __init__( # noqa: PLR0913 else: self.includes = includes - def filter_definitions(self) -> Set[Definition]: + def filter_definitions(self) -> set[Definition]: """Main method to perform all filtering. Filters Definitions of entities (class, method and function). @@ -599,21 +599,21 @@ def filter_definitions(self) -> Set[Definition]: Returns: Definitions that pass all filters. """ - usages_in_docs_examples_and_docs_content: Set[str] = ( + usages_in_docs_examples_and_docs_content: set[str] = ( self._docs_examples_usages() | self.references_from_docs_content ) - gx_definitions_used_in_docs_examples: Set[Definition] = ( + gx_definitions_used_in_docs_examples: set[Definition] = ( self._filter_gx_definitions_from_docs_examples( gx_usages_in_docs_examples=usages_in_docs_examples_and_docs_content ) ) - non_private_definitions: Set[Definition] = self._filter_private_entities( + non_private_definitions: set[Definition] = self._filter_private_entities( definitions=gx_definitions_used_in_docs_examples ) - included_definitions: Set[Definition] = self._filter_or_include( + included_definitions: set[Definition] = self._filter_or_include( definitions=non_private_definitions ) - definitions_not_marked_public_api: Set[Definition] = ( + definitions_not_marked_public_api: set[Definition] = ( self._filter_for_definitions_not_marked_public_api( definitions=included_definitions ) @@ -621,13 +621,13 @@ def filter_definitions(self) -> Set[Definition]: return definitions_not_marked_public_api - def _docs_examples_usages(self) -> Set[str]: + def _docs_examples_usages(self) -> set[str]: """Filter list of classes & methods from docs examples to only those found in the GX codebase (e.g. filter out print() or other python or 3rd party classes/methods). """ - doc_example_usages: Set[str] = ( + doc_example_usages: set[str] = ( self.docs_example_parser.get_names_from_usage_in_docs_examples() ) gx_code_definitions = self.code_parser.get_all_class_method_and_function_names() @@ -638,8 +638,8 @@ def _docs_examples_usages(self) -> Set[str]: return doc_example_usages_of_gx_code def _filter_gx_definitions_from_docs_examples( - self, gx_usages_in_docs_examples: Set[str] - ) -> Set[Definition]: + self, gx_usages_in_docs_examples: set[str] + ) -> set[Definition]: """Filter the list of GX definitions except those used in docs examples. Use the docs examples filtered list against the list of class and method @@ -657,17 +657,17 @@ def _filter_gx_definitions_from_docs_examples( } return gx_code_definitions_appearing_in_docs_examples - def _filter_private_entities(self, definitions: Set[Definition]) -> Set[Definition]: + def _filter_private_entities(self, definitions: set[Definition]) -> set[Definition]: """Filter out private entities (classes, methods and functions with leading underscore).""" return {d for d in definitions if not self._is_definition_private(definition=d)} - def _filter_or_include(self, definitions: Set[Definition]) -> Set[Definition]: + def _filter_or_include(self, definitions: set[Definition]) -> set[Definition]: """Filter definitions per all IncludeExcludeDefinition directives. Includes override excludes, and also don't require the included entity to be used in docs examples. """ - included_definitions: List[Definition] = [] + included_definitions: list[Definition] = [] all_gx_code_definitions = ( self.code_parser.get_all_class_method_and_function_definitions() ) @@ -714,8 +714,8 @@ def _repo_relative_filepath_comparison( ) def _filter_for_definitions_not_marked_public_api( - self, definitions: Set[Definition] - ) -> Set[Definition]: + self, definitions: set[Definition] + ) -> set[Definition]: """Return only those Definitions that are not marked with the public api decorator.""" return { d @@ -767,7 +767,7 @@ def _is_definition_private(self, definition: Definition) -> bool: class PublicAPIReport: """Generate a report from entity definitions (class, method and function).""" - def __init__(self, definitions: Set[Definition], repo_root: pathlib.Path) -> None: + def __init__(self, definitions: set[Definition], repo_root: pathlib.Path) -> None: """Create a PublicAPIReport object. Args: @@ -794,7 +794,7 @@ def write_printable_definitions_to_file( def generate_printable_definitions( self, - ) -> List[str]: + ) -> list[str]: """Generate a printable (human readable) definition. Returns: @@ -803,7 +803,7 @@ def generate_printable_definitions( sorted_definitions_list = sorted( list(self.definitions), key=operator.attrgetter("filepath", "name") ) - sorted_definitions_strings: List[str] = [] + sorted_definitions_strings: list[str] = [] for definition in sorted_definitions_list: if definition.filepath.is_absolute(): filepath = str(definition.filepath.relative_to(self.repo_root)) @@ -819,7 +819,7 @@ def generate_printable_definitions( return sorted_definitions_strings_no_dupes - def _deduplicate_strings(self, strings: List[str]) -> List[str]: + def _deduplicate_strings(self, strings: list[str]) -> list[str]: """Deduplicate a list of strings, keeping order intact.""" seen = set() no_duplicates = [] @@ -836,21 +836,21 @@ def _repo_root() -> pathlib.Path: return repo_root_path -def _default_doc_example_absolute_paths() -> Set[pathlib.Path]: +def _default_doc_example_absolute_paths() -> set[pathlib.Path]: """Get all paths of doc examples (docs examples).""" base_directory = _repo_root() / "docs" / "docusaurus" / "docs" paths = base_directory.rglob("*.py") return set(paths) -def _default_code_absolute_paths() -> Set[pathlib.Path]: +def _default_code_absolute_paths() -> set[pathlib.Path]: """All Great Expectations modules related to the main library.""" base_directory = _repo_root() / "great_expectations" paths = base_directory.rglob("**/*.py") return set(paths) -def _default_docs_absolute_paths() -> Set[pathlib.Path]: +def _default_docs_absolute_paths() -> set[pathlib.Path]: """All Great Expectations modules related to the main library.""" base_directory = _repo_root() / "docs" paths: list[pathlib.Path] = [] diff --git a/great_expectations/alias_types.py b/great_expectations/alias_types.py index a072935bf264..fa0794ad3e11 100644 --- a/great_expectations/alias_types.py +++ b/great_expectations/alias_types.py @@ -3,12 +3,12 @@ """This module contains shared TypeAliases""" import pathlib -from typing import TYPE_CHECKING, Dict, List, Union +from typing import TYPE_CHECKING, Union if TYPE_CHECKING: from typing_extensions import TypeAlias PathStr: TypeAlias = Union[str, pathlib.Path] JSONValues: TypeAlias = Union[ - Dict[str, "JSONValues"], List["JSONValues"], str, int, float, bool, None + dict[str, "JSONValues"], list["JSONValues"], str, int, float, bool, None ] diff --git a/great_expectations/analytics/base_event.py b/great_expectations/analytics/base_event.py index b2e9bdb1ae22..4261711e8f04 100644 --- a/great_expectations/analytics/base_event.py +++ b/great_expectations/analytics/base_event.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import ClassVar, List, Optional +from typing import ClassVar, Optional from uuid import UUID from great_expectations import __version__ as gx_version @@ -58,7 +58,7 @@ def distinct_id(self) -> UUID | None: """ return self.user_id or self.oss_id - _allowed_actions: ClassVar[Optional[List[Action]]] = None + _allowed_actions: ClassVar[Optional[list[Action]]] = None def __post_init__(self): allowed_actions = self.get_allowed_actions() diff --git a/great_expectations/analytics/events.py b/great_expectations/analytics/events.py index 74eec3811440..c667ac9a1d1f 100644 --- a/great_expectations/analytics/events.py +++ b/great_expectations/analytics/events.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import ClassVar, List +from typing import ClassVar from great_expectations.analytics.actions import ( CHECKPOINT_CREATED, @@ -23,7 +23,7 @@ @dataclass class DataContextInitializedEvent(Event): - _allowed_actions: ClassVar[List[Action]] = [DATA_CONTEXT_INITIALIZED] + _allowed_actions: ClassVar[list[Action]] = [DATA_CONTEXT_INITIALIZED] def __init__(self): super().__init__(action=DATA_CONTEXT_INITIALIZED) @@ -47,7 +47,7 @@ class ExpectationSuiteExpectationCreatedEvent(_ExpectationSuiteExpectationEvent) expectation_type: str = "UNKNOWN" custom_exp_type: bool = False - _allowed_actions: ClassVar[List[Action]] = [ + _allowed_actions: ClassVar[list[Action]] = [ EXPECTATION_SUITE_EXPECTATION_CREATED, ] @@ -77,7 +77,7 @@ def _properties(self) -> dict: @dataclass class ExpectationSuiteExpectationUpdatedEvent(_ExpectationSuiteExpectationEvent): - _allowed_actions: ClassVar[List[Action]] = [ + _allowed_actions: ClassVar[list[Action]] = [ EXPECTATION_SUITE_EXPECTATION_UPDATED, ] @@ -95,7 +95,7 @@ def __init__( @dataclass class ExpectationSuiteExpectationDeletedEvent(_ExpectationSuiteExpectationEvent): - _allowed_actions: ClassVar[List[Action]] = [ + _allowed_actions: ClassVar[list[Action]] = [ EXPECTATION_SUITE_EXPECTATION_DELETED, ] @@ -124,7 +124,7 @@ def _properties(self) -> dict: @dataclass class ExpectationSuiteCreatedEvent(_ExpectationSuiteEvent): - _allowed_actions: ClassVar[List[Action]] = [EXPECTATION_SUITE_CREATED] + _allowed_actions: ClassVar[list[Action]] = [EXPECTATION_SUITE_CREATED] def __init__(self, expectation_suite_id: str | None = None): super().__init__( @@ -135,7 +135,7 @@ def __init__(self, expectation_suite_id: str | None = None): @dataclass class ExpectationSuiteDeletedEvent(_ExpectationSuiteEvent): - _allowed_actions: ClassVar[List[Action]] = [EXPECTATION_SUITE_DELETED] + _allowed_actions: ClassVar[list[Action]] = [EXPECTATION_SUITE_DELETED] def __init__(self, expectation_suite_id: str | None = None): super().__init__( @@ -157,7 +157,7 @@ def _properties(self) -> dict: @dataclass class CheckpointCreatedEvent(_CheckpointEvent): - _allowed_actions: ClassVar[List[Action]] = [CHECKPOINT_CREATED] + _allowed_actions: ClassVar[list[Action]] = [CHECKPOINT_CREATED] def __init__( self, @@ -180,7 +180,7 @@ def _properties(self) -> dict: @dataclass class CheckpointDeletedEvent(_CheckpointEvent): - _allowed_actions: ClassVar[List[Action]] = [CHECKPOINT_DELETED] + _allowed_actions: ClassVar[list[Action]] = [CHECKPOINT_DELETED] def __init__(self, checkpoint_id: str | None = None): super().__init__( @@ -191,7 +191,7 @@ def __init__(self, checkpoint_id: str | None = None): @dataclass class CheckpointRanEvent(_CheckpointEvent): - _allowed_actions: ClassVar[List[Action]] = [CHECKPOINT_RAN] + _allowed_actions: ClassVar[list[Action]] = [CHECKPOINT_RAN] def __init__( self, @@ -225,7 +225,7 @@ def _properties(self) -> dict: @dataclass class ValidationDefinitionCreatedEvent(_ValidationDefinitionEvent): - _allowed_actions: ClassVar[List[Action]] = [VALIDATION_DEFINITION_CREATED] + _allowed_actions: ClassVar[list[Action]] = [VALIDATION_DEFINITION_CREATED] def __init__(self, validation_definition_id: str | None = None): super().__init__( @@ -236,7 +236,7 @@ def __init__(self, validation_definition_id: str | None = None): @dataclass class ValidationDefinitionDeletedEvent(_ValidationDefinitionEvent): - _allowed_actions: ClassVar[List[Action]] = [VALIDATION_DEFINITION_DELETED] + _allowed_actions: ClassVar[list[Action]] = [VALIDATION_DEFINITION_DELETED] def __init__(self, validation_definition_id: str | None = None): super().__init__( @@ -247,7 +247,7 @@ def __init__(self, validation_definition_id: str | None = None): @dataclass class DomainObjectAllDeserializationEvent(Event): - _allowed_actions: ClassVar[List[Action]] = [DOMAIN_OBJECT_ALL_DESERIALIZE_ERROR] + _allowed_actions: ClassVar[list[Action]] = [DOMAIN_OBJECT_ALL_DESERIALIZE_ERROR] store_name: str error_type: str diff --git a/great_expectations/checkpoint/actions.py b/great_expectations/checkpoint/actions.py index 3fedec301032..c80528dc2e5c 100644 --- a/great_expectations/checkpoint/actions.py +++ b/great_expectations/checkpoint/actions.py @@ -11,10 +11,8 @@ from typing import ( TYPE_CHECKING, Any, - List, Literal, Optional, - Type, Union, ) @@ -98,7 +96,7 @@ def data(self) -> list[tuple[ValidationAction, dict]]: def update(self, action: ValidationAction, action_result: dict) -> None: self._data.append((action, action_result)) - def filter_results(self, class_: Type[ValidationAction]) -> list[dict]: + def filter_results(self, class_: type[ValidationAction]) -> list[dict]: return [action_result for action, action_result in self._data if isinstance(action, class_)] @@ -229,7 +227,7 @@ class SlackNotificationAction(DataDocsAction): slack_token: Optional[Union[ConfigStr, str]] = None slack_channel: Optional[Union[ConfigStr, str]] = None notify_on: Literal["all", "failure", "success"] = "all" - notify_with: Optional[List[str]] = None + notify_with: Optional[list[str]] = None show_failed_expectations: bool = False renderer: SlackRenderer = Field(default_factory=SlackRenderer) @@ -518,7 +516,7 @@ class OpsgenieAlertAction(ValidationAction): region: Optional[str] = None priority: Literal["P1", "P2", "P3", "P4", "P5"] = "P3" notify_on: Literal["all", "failure", "success"] = "failure" - tags: Optional[List[str]] = None + tags: Optional[list[str]] = None renderer: OpsgenieRenderer = Field(default_factory=OpsgenieRenderer) @validator("renderer", pre=True) @@ -623,7 +621,7 @@ class EmailAction(ValidationAction): use_tls: Optional[bool] = None use_ssl: Optional[bool] = None notify_on: Literal["all", "failure", "success"] = "all" - notify_with: Optional[List[str]] = None + notify_with: Optional[list[str]] = None renderer: EmailRenderer = Field(default_factory=EmailRenderer) @validator("renderer", pre=True) @@ -721,7 +719,7 @@ class UpdateDataDocsAction(DataDocsAction): type: Literal["update_data_docs"] = "update_data_docs" - site_names: List[str] = [] + site_names: list[str] = [] @override def run( diff --git a/great_expectations/checkpoint/checkpoint.py b/great_expectations/checkpoint/checkpoint.py index 68ccb08bddc1..f05b90b22b5a 100644 --- a/great_expectations/checkpoint/checkpoint.py +++ b/great_expectations/checkpoint/checkpoint.py @@ -7,8 +7,6 @@ AbstractSet, Any, Callable, - Dict, - List, Mapping, Optional, TypedDict, @@ -84,8 +82,8 @@ class Checkpoint(BaseModel): """ # noqa: E501 name: str - validation_definitions: List[ValidationDefinition] - actions: List[CheckpointAction] = Field(default_factory=list) + validation_definitions: list[ValidationDefinition] + actions: list[CheckpointAction] = Field(default_factory=list) result_format: ResultFormatUnion = DEFAULT_RESULT_FORMAT id: Union[str, None] = None @@ -180,7 +178,7 @@ def dict( # noqa: PLR0913 exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Override the default dict method to enable proper diagnostics around validation_definitions. @@ -212,7 +210,7 @@ def _determine_exclude( return exclude - def _serialize_validation_definitions(self, data: Dict[str, Any]) -> Dict[str, Any]: + def _serialize_validation_definitions(self, data: dict[str, Any]) -> dict[str, Any]: """ Manually serialize the validation_definitions field to avoid Pydantic's default serialization. @@ -235,9 +233,9 @@ def _serialize_validation_definitions(self, data: Dict[str, Any]) -> Dict[str, A @validator("validation_definitions", pre=True) def _validate_validation_definitions( - cls, validation_definitions: list[ValidationDefinition] | list[Dict[str, Any]] + cls, validation_definitions: list[ValidationDefinition] | list[dict[str, Any]] ) -> list[ValidationDefinition]: - if validation_definitions and isinstance(validation_definitions[0], Dict): + if validation_definitions and isinstance(validation_definitions[0], dict): validation_definition_store = project_manager.get_validation_definition_store() identifier_bundles = [ _IdentifierBundle(**v) # type: ignore[arg-type] # All validation configs are dicts if the first one is @@ -247,7 +245,7 @@ def _validate_validation_definitions( identifier_bundles=identifier_bundles, store=validation_definition_store ) - return cast(List[ValidationDefinition], validation_definitions) + return cast(list[ValidationDefinition], validation_definitions) @classmethod def _deserialize_identifier_bundles_to_validation_definitions( @@ -273,7 +271,7 @@ def _deserialize_identifier_bundles_to_validation_definitions( @public_api def run( self, - batch_parameters: Dict[str, Any] | None = None, + batch_parameters: dict[str, Any] | None = None, expectation_parameters: SuiteParameterDict | None = None, run_id: RunIdentifier | None = None, ) -> CheckpointResult: @@ -312,12 +310,12 @@ def _submit_analytics_event(self): def _run_validation_definitions( self, - batch_parameters: Dict[str, Any] | None, + batch_parameters: dict[str, Any] | None, expectation_parameters: SuiteParameterDict | None, result_format: ResultFormatUnion, run_id: RunIdentifier, - ) -> Dict[ValidationResultIdentifier, ExpectationSuiteValidationResult]: - run_results: Dict[ValidationResultIdentifier, ExpectationSuiteValidationResult] = {} + ) -> dict[ValidationResultIdentifier, ExpectationSuiteValidationResult]: + run_results: dict[ValidationResultIdentifier, ExpectationSuiteValidationResult] = {} for validation_definition in self.validation_definitions: validation_result = validation_definition.run( checkpoint_id=self.id, @@ -352,7 +350,7 @@ def _build_result_key( def _construct_result( self, run_id: RunIdentifier, - run_results: Dict[ValidationResultIdentifier, ExpectationSuiteValidationResult], + run_results: dict[ValidationResultIdentifier, ExpectationSuiteValidationResult], ) -> CheckpointResult: for result in run_results.values(): result.meta["checkpoint_id"] = self.id @@ -376,15 +374,15 @@ def _run_actions( ) action_context.update(action=action, action_result=action_result) - def _sort_actions(self) -> List[CheckpointAction]: + def _sort_actions(self) -> list[CheckpointAction]: """ UpdateDataDocsActions are prioritized to run first, followed by all other actions. This is due to the fact that certain actions reference data docs sites, which must be updated first. """ - priority_actions: List[CheckpointAction] = [] - secondary_actions: List[CheckpointAction] = [] + priority_actions: list[CheckpointAction] = [] + secondary_actions: list[CheckpointAction] = [] for action in self.actions: if isinstance(action, UpdateDataDocsAction): priority_actions.append(action) @@ -440,7 +438,7 @@ def _add_to_store(self) -> None: @public_api class CheckpointResult(BaseModel): run_id: RunIdentifier - run_results: Dict[ValidationResultIdentifier, ExpectationSuiteValidationResult] + run_results: dict[ValidationResultIdentifier, ExpectationSuiteValidationResult] checkpoint_config: Checkpoint success: Optional[bool] = None @@ -491,7 +489,7 @@ def describe(self) -> str: class CheckpointDescriptionDict(TypedDict): success: bool statistics: CheckpointDescriptionStatistics - validation_results: List[Dict[str, Any]] + validation_results: list[dict[str, Any]] class CheckpointDescriptionStatistics(TypedDict): diff --git a/great_expectations/compatibility/bigquery.py b/great_expectations/compatibility/bigquery.py index 6d44299f1238..c15872303922 100644 --- a/great_expectations/compatibility/bigquery.py +++ b/great_expectations/compatibility/bigquery.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any from great_expectations.compatibility.not_imported import NotImported @@ -18,7 +18,7 @@ except (ImportError, AttributeError): sqlalchemy_bigquery = SQLALCHEMY_BIGQUERY_NOT_IMPORTED -BIGQUERY_TYPES: Dict[str, Any] = ( +BIGQUERY_TYPES: dict[str, Any] = ( { "INTEGER": sqlalchemy_bigquery.INTEGER, "NUMERIC": sqlalchemy_bigquery.NUMERIC, diff --git a/great_expectations/core/batch.py b/great_expectations/core/batch.py index cb9e09c74ff7..b218674adfa6 100644 --- a/great_expectations/core/batch.py +++ b/great_expectations/core/batch.py @@ -9,7 +9,6 @@ Any, Callable, ClassVar, - Type, TypedDict, Union, overload, @@ -60,21 +59,21 @@ class BlockConfigBatchRequestTypedDict(TypedDict): limit: NotRequired[BatchSlice] -def _get_fluent_batch_request_class() -> Type[FluentBatchRequest]: +def _get_fluent_batch_request_class() -> type[FluentBatchRequest]: """Using this function helps work around circular import dependncies.""" module_name = "great_expectations.datasource.fluent.batch_request" class_name = "BatchRequest" return load_class(class_name=class_name, module_name=module_name) -def _get_fluent_batch_class() -> Type[FluentBatch]: +def _get_fluent_batch_class() -> type[FluentBatch]: """Using this function helps work around circular import dependncies.""" module_name = "great_expectations.datasource.fluent.interfaces" class_name = "Batch" return load_class(class_name=class_name, module_name=module_name) -def _get_metrics_calculator_class() -> Type[MetricsCalculator]: +def _get_metrics_calculator_class() -> type[MetricsCalculator]: """Using this function helps work around circular import dependncies.""" module_name = "great_expectations.validator.metrics_calculator" class_name = "MetricsCalculator" @@ -1382,7 +1381,7 @@ def standardize_batch_request_display_ordering( if pyspark.DataFrame: # type: ignore[truthy-function] # False if NotImported BatchDataUnion: TypeAlias = Union[BatchData, pd.DataFrame, pyspark.DataFrame] - BatchDataType: TypeAlias = Union[Type[BatchData], Type[pd.DataFrame], Type[pyspark.DataFrame]] + BatchDataType: TypeAlias = Union[type[BatchData], type[pd.DataFrame], type[pyspark.DataFrame]] else: - BatchDataType = Union[Type[BatchData], Type[pd.DataFrame]] # type: ignore[misc] # Cannot assign multiple types + BatchDataType = Union[type[BatchData], type[pd.DataFrame]] # type: ignore[misc] # Cannot assign multiple types BatchDataUnion = Union[BatchData, pd.DataFrame] # type: ignore[misc] # Cannot assign multiple types diff --git a/great_expectations/core/batch_definition.py b/great_expectations/core/batch_definition.py index 3618377ca573..3e38b0243b0e 100644 --- a/great_expectations/core/batch_definition.py +++ b/great_expectations/core/batch_definition.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, TypeVar +from typing import TYPE_CHECKING, Any, Generic, Optional, TypeVar from great_expectations._docs_decorators import public_api from great_expectations.compatibility import pydantic @@ -84,7 +84,7 @@ def get_batch(self, batch_parameters: Optional[BatchParameters] = None) -> Batch @public_api def get_batch_identifiers_list( self, batch_parameters: Optional[BatchParameters] = None - ) -> List[Dict]: + ) -> list[dict]: """ Retrieves a list of available batch identifiers. These identifiers can be used to fetch specific batches via batch_options. diff --git a/great_expectations/core/batch_manager.py b/great_expectations/core/batch_manager.py index 20bb8ff923a4..41d5c81a349b 100644 --- a/great_expectations/core/batch_manager.py +++ b/great_expectations/core/batch_manager.py @@ -2,7 +2,7 @@ import logging from collections import OrderedDict -from typing import TYPE_CHECKING, Dict, List, Optional, Sequence +from typing import TYPE_CHECKING, Optional, Sequence from great_expectations.core.batch import ( Batch, @@ -25,7 +25,7 @@ class BatchManager: def __init__( self, execution_engine: ExecutionEngine, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, ) -> None: """ Args: @@ -37,19 +37,19 @@ def __init__( self._active_batch_id: Optional[str] = None self._active_batch_data_id: Optional[str] = None - self._batch_cache: Dict[str, AnyBatch] = OrderedDict() - self._batch_data_cache: Dict[str, BatchDataUnion] = {} + self._batch_cache: dict[str, AnyBatch] = OrderedDict() + self._batch_data_cache: dict[str, BatchDataUnion] = {} if batch_list: self.load_batch_list(batch_list=batch_list) @property - def batch_data_cache(self) -> Dict[str, BatchDataUnion]: + def batch_data_cache(self) -> dict[str, BatchDataUnion]: """Dictionary of loaded BatchData objects.""" return self._batch_data_cache @property - def loaded_batch_ids(self) -> List[str]: + def loaded_batch_ids(self) -> list[str]: """IDs of loaded BatchData objects.""" return list(self._batch_data_cache.keys()) @@ -82,7 +82,7 @@ def active_batch_data(self) -> Optional[BatchDataUnion]: return self._batch_data_cache.get(self.active_batch_data_id) @property - def batch_cache(self) -> Dict[str, AnyBatch]: + def batch_cache(self) -> dict[str, AnyBatch]: """Getter for ordered dictionary (cache) of "Batch" objects in use (with batch_id as key).""" # noqa: E501 return self._batch_cache diff --git a/great_expectations/core/batch_spec.py b/great_expectations/core/batch_spec.py index acc7bffc6503..46feb4e4e69f 100644 --- a/great_expectations/core/batch_spec.py +++ b/great_expectations/core/batch_spec.py @@ -2,7 +2,7 @@ import logging from abc import ABCMeta -from typing import TYPE_CHECKING, Any, Callable, List, Literal, Protocol +from typing import TYPE_CHECKING, Any, Callable, Literal, Protocol from great_expectations.compatibility.typing_extensions import override from great_expectations.core.id_dict import BatchSpec @@ -237,5 +237,5 @@ def reader_options(self) -> dict: return self.get("reader_options", {}) @property - def partitions(self) -> List[str]: + def partitions(self) -> list[str]: return self.get("partitions", []) diff --git a/great_expectations/core/config_provider.py b/great_expectations/core/config_provider.py index 1bd89a1e0303..0492aead2993 100644 --- a/great_expectations/core/config_provider.py +++ b/great_expectations/core/config_provider.py @@ -4,7 +4,7 @@ import os from abc import ABC, abstractmethod from collections import OrderedDict -from typing import Any, Dict, Optional, Type, cast +from typing import Any, Optional, cast from great_expectations.compatibility.typing_extensions import override from great_expectations.core.config_substitutor import _ConfigurationSubstitutor @@ -19,13 +19,13 @@ def __init__(self) -> None: self._substitutor = _ConfigurationSubstitutor() @abstractmethod - def get_values(self) -> Dict[str, str]: + def get_values(self) -> dict[str, str]: """ Retrieve any configuration variables relevant to the provider's environment. """ pass - def substitute_config(self, config: Any, config_values: Optional[Dict[str, str]] = None) -> Any: + def substitute_config(self, config: Any, config_values: Optional[dict[str, str]] = None) -> Any: """ Utilizes the underlying ConfigurationSubstitutor instance to substitute any $VARIABLES with their corresponding config variable value. @@ -56,7 +56,7 @@ class _ConfigurationProvider(_AbstractConfigurationProvider): def __init__(self) -> None: self._providers: OrderedDict[ - Type[_AbstractConfigurationProvider], _AbstractConfigurationProvider + type[_AbstractConfigurationProvider], _AbstractConfigurationProvider ] = OrderedDict() super().__init__() @@ -74,7 +74,7 @@ def register_provider(self, provider: _AbstractConfigurationProvider) -> None: self._providers[type_] = provider def get_provider( - self, type_: Type[_AbstractConfigurationProvider] + self, type_: type[_AbstractConfigurationProvider] ) -> Optional[_AbstractConfigurationProvider]: """ Retrieves a registered configuration provider (if available). @@ -89,14 +89,14 @@ def get_provider( return self._providers.get(type_) @override - def get_values(self) -> Dict[str, str]: + def get_values(self) -> dict[str, str]: """ Iterates through all registered providers to aggregate a list of configuration values. Values are generated based on the order of registration; if there is a conflict, subsequent providers will overwrite existing values. """ - values: Dict[str, str] = {} + values: dict[str, str] = {} for provider in self._providers.values(): values.update(provider.get_values()) return values @@ -107,12 +107,12 @@ class _RuntimeEnvironmentConfigurationProvider(_AbstractConfigurationProvider): Responsible for the management of the runtime_environment dictionary provided at runtime. """ - def __init__(self, runtime_environment: Dict[str, str]) -> None: + def __init__(self, runtime_environment: dict[str, str]) -> None: self._runtime_environment = runtime_environment super().__init__() @override - def get_values(self) -> Dict[str, str]: + def get_values(self) -> dict[str, str]: return self._runtime_environment @@ -125,7 +125,7 @@ def __init__(self) -> None: super().__init__() @override - def get_values(self) -> Dict[str, str]: + def get_values(self) -> dict[str, str]: return dict(os.environ) # noqa: TID251 # os.environ allowed in config files @@ -144,7 +144,7 @@ def __init__( super().__init__() @override - def get_values(self) -> Dict[str, str]: + def get_values(self) -> dict[str, str]: env_vars = dict(os.environ) # noqa: TID251 # os.environ allowed in config files try: # If the user specifies the config variable path with an environment variable, we want to substitute it # noqa: E501 @@ -162,7 +162,7 @@ def get_values(self) -> Dict[str, str]: variables = dict(yaml.load(contents)) or {} return cast( - Dict[str, str], + dict[str, str], self._substitutor.substitute_all_config_variables(variables, env_vars), ) @@ -184,7 +184,7 @@ def __init__(self, cloud_config: GXCloudConfig) -> None: self._cloud_config = cloud_config @override - def get_values(self) -> Dict[str, str]: + def get_values(self) -> dict[str, str]: from great_expectations.data_context.cloud_constants import ( GXCloudEnvironmentVariable, ) @@ -193,7 +193,7 @@ def get_values(self) -> Dict[str, str]: access_token = self._cloud_config.access_token organization_id = self._cloud_config.organization_id - cloud_values: Dict[str, str] = { + cloud_values: dict[str, str] = { GXCloudEnvironmentVariable.BASE_URL: base_url, GXCloudEnvironmentVariable.ACCESS_TOKEN: access_token, } diff --git a/great_expectations/core/config_substitutor.py b/great_expectations/core/config_substitutor.py index edce5f77735b..8411f971d794 100644 --- a/great_expectations/core/config_substitutor.py +++ b/great_expectations/core/config_substitutor.py @@ -6,7 +6,7 @@ import re from collections import OrderedDict from functools import lru_cache -from typing import Any, Dict, Final, Optional +from typing import Any, Final, Optional import great_expectations.exceptions as gx_exceptions from great_expectations.compatibility import aws, azure, google @@ -46,7 +46,7 @@ def __init__(self) -> None: def substitute_all_config_variables( self, data: Any, - replace_variables_dict: Dict[str, str], + replace_variables_dict: dict[str, str], dollar_sign_escape_string: str = r"\$", ) -> Any: """ @@ -77,7 +77,7 @@ def substitute_all_config_variables( def substitute_config_variable( self, template_str: str, - config_variables_dict: Dict[str, str], + config_variables_dict: dict[str, str], dollar_sign_escape_string: str = r"\$", ) -> Optional[str]: """ diff --git a/great_expectations/core/data_context_key.py b/great_expectations/core/data_context_key.py index 60ac1778a3b9..eb92117fbd52 100644 --- a/great_expectations/core/data_context_key.py +++ b/great_expectations/core/data_context_key.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABCMeta, abstractmethod -from typing import Optional, Tuple, Union +from typing import Optional, Union from great_expectations.compatibility.typing_extensions import override @@ -99,14 +99,14 @@ def resource_name(self) -> Union[str, None]: return self._resource_name @override - def to_tuple(self) -> Tuple[str]: + def to_tuple(self) -> tuple[str]: """ See parent `DataContextKey.to_tuple` for more information. """ return (self._resource_name or "",) @override - def to_fixed_length_tuple(self) -> Tuple[str]: + def to_fixed_length_tuple(self) -> tuple[str]: """ See parent `DataContextKey.to_fixed_length_tuple` for more information. """ diff --git a/great_expectations/core/domain.py b/great_expectations/core/domain.py index bf198157cf63..9089179b677e 100644 --- a/great_expectations/core/domain.py +++ b/great_expectations/core/domain.py @@ -3,7 +3,7 @@ import json from dataclasses import asdict, dataclass from enum import Enum -from typing import Any, Dict, Optional, TypeVar, Union +from typing import Any, Optional, TypeVar, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.id_dict import IDDict @@ -35,7 +35,7 @@ class SemanticDomainTypes(Enum): @dataclass class InferredSemanticDomainType(SerializableDictDot): semantic_domain_type: Optional[Union[str, SemanticDomainTypes]] = None - details: Optional[Dict[str, Any]] = None + details: Optional[dict[str, Any]] = None @override def to_dict(self) -> dict: @@ -60,8 +60,8 @@ class Domain(SerializableDotDict): def __init__( # noqa: C901 - too complex self, domain_type: Union[str, MetricDomainTypes], - domain_kwargs: Optional[Union[Dict[str, Any], DomainKwargs]] = None, - details: Optional[Dict[str, Any]] = None, + domain_kwargs: Optional[Union[dict[str, Any], DomainKwargs]] = None, + details: Optional[dict[str, Any]] = None, rule_name: Optional[str] = None, ) -> None: if isinstance(domain_type, str): @@ -92,7 +92,7 @@ def __init__( # noqa: C901 - too complex if details is None: details = {} - inferred_semantic_domain_type: Optional[Dict[str, Union[str, SemanticDomainTypes]]] = ( + inferred_semantic_domain_type: Optional[dict[str, Union[str, SemanticDomainTypes]]] = ( details.get(INFERRED_SEMANTIC_TYPE_KEY) ) if inferred_semantic_domain_type: diff --git a/great_expectations/core/expectation_diagnostics/expectation_diagnostics.py b/great_expectations/core/expectation_diagnostics/expectation_diagnostics.py index 79400cdea232..a8cc3beaa1f8 100644 --- a/great_expectations/core/expectation_diagnostics/expectation_diagnostics.py +++ b/great_expectations/core/expectation_diagnostics/expectation_diagnostics.py @@ -4,7 +4,7 @@ import re from collections import defaultdict from dataclasses import asdict, dataclass -from typing import List, Sequence, Tuple, Union +from typing import Sequence, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.expectation_diagnostics.expectation_test_data_cases import ( @@ -43,8 +43,8 @@ class ExpectationDiagnostics(SerializableDictDot): """ # noqa: E501 # This object is taken directly from the Expectation class, without modification - examples: List[ExpectationTestDataCases] - gallery_examples: List[ExpectationTestDataCases] + examples: list[ExpectationTestDataCases] + gallery_examples: list[ExpectationTestDataCases] # These objects are derived from the Expectation class # They're a combination of direct introspection of existing properties, @@ -58,11 +58,11 @@ class ExpectationDiagnostics(SerializableDictDot): description: ExpectationDescriptionDiagnostics execution_engines: ExpectationExecutionEngineDiagnostics - renderers: List[ExpectationRendererDiagnostics] - metrics: List[ExpectationMetricDiagnostics] - tests: List[ExpectationTestDiagnostics] - backend_test_result_counts: List[ExpectationBackendTestResultCounts] - errors: List[ExpectationErrorDiagnostics] + renderers: list[ExpectationRendererDiagnostics] + metrics: list[ExpectationMetricDiagnostics] + tests: list[ExpectationTestDiagnostics] + backend_test_result_counts: list[ExpectationBackendTestResultCounts] + errors: list[ExpectationErrorDiagnostics] maturity_checklist: ExpectationDiagnosticMaturityMessages coverage_score: float @@ -145,8 +145,8 @@ def _check_docstring( @classmethod def _check_example_cases( cls, - examples: List[ExpectationTestDataCases], - tests: List[ExpectationTestDiagnostics], + examples: list[ExpectationTestDataCases], + tests: list[ExpectationTestDiagnostics], ) -> ExpectationDiagnosticCheckMessage: """Check whether this Expectation has at least one positive and negative example case (and all test cases return the expected output)""" # noqa: E501 @@ -167,11 +167,11 @@ def _check_example_cases( @staticmethod def _check_core_logic_for_at_least_one_execution_engine( - backend_test_result_counts: List[ExpectationBackendTestResultCounts], + backend_test_result_counts: list[ExpectationBackendTestResultCounts], ) -> ExpectationDiagnosticCheckMessage: """Check whether core logic for this Expectation exists and passes tests on at least one Execution Engine""" # noqa: E501 - sub_messages: List[ExpectationDiagnosticCheckMessageDict] = [] + sub_messages: list[ExpectationDiagnosticCheckMessageDict] = [] passed = False message = "Has core logic and passes tests on at least one Execution Engine" all_passing = [ @@ -206,12 +206,12 @@ def _check_core_logic_for_at_least_one_execution_engine( @staticmethod def _get_backends_from_test_results( - test_results: List[ExpectationTestDiagnostics], - ) -> List[ExpectationBackendTestResultCounts]: + test_results: list[ExpectationTestDiagnostics], + ) -> list[ExpectationBackendTestResultCounts]: """Has each tested backend and the number of passing/failing tests""" backend_results = defaultdict(list) backend_failing_names = defaultdict(list) - results: List[ExpectationBackendTestResultCounts] = [] + results: list[ExpectationBackendTestResultCounts] = [] for test_result in test_results: backend_results[test_result.backend].append(test_result.test_passed) @@ -231,7 +231,7 @@ def _get_backends_from_test_results( @staticmethod def _check_core_logic_for_all_applicable_execution_engines( - backend_test_result_counts: List[ExpectationBackendTestResultCounts], + backend_test_result_counts: list[ExpectationBackendTestResultCounts], ) -> ExpectationDiagnosticCheckMessage: """Check whether core logic for this Expectation exists and passes tests on all applicable Execution Engines""" # noqa: E501 @@ -292,8 +292,8 @@ def _check_core_logic_for_all_applicable_execution_engines( @staticmethod def _count_positive_and_negative_example_cases( - examples: List[ExpectationTestDataCases], - ) -> Tuple[int, int]: + examples: list[ExpectationTestDataCases], + ) -> tuple[int, int]: """Scans examples and returns a 2-ple with the numbers of cases with success == True and success == False""" # noqa: E501 positive_cases: int = 0 @@ -356,7 +356,7 @@ def _convert_checks_into_output_message( @staticmethod def _check_input_validation( expectation_instance, - examples: List[ExpectationTestDataCases], + examples: list[ExpectationTestDataCases], ) -> ExpectationDiagnosticCheckMessage: """Check that the validate_configuration exists and doesn't raise a config error""" passed = False diff --git a/great_expectations/core/expectation_diagnostics/expectation_doctor.py b/great_expectations/core/expectation_diagnostics/expectation_doctor.py index ea177bd9638d..9f9b0a52fd33 100644 --- a/great_expectations/core/expectation_diagnostics/expectation_doctor.py +++ b/great_expectations/core/expectation_diagnostics/expectation_doctor.py @@ -8,7 +8,7 @@ import time import traceback from collections import defaultdict -from typing import TYPE_CHECKING, Final, List, Optional, Union +from typing import TYPE_CHECKING, Final, Optional, Union from great_expectations.core.expectation_diagnostics.expectation_diagnostics import ( ExpectationDiagnostics, @@ -76,7 +76,7 @@ def print_diagnostic_checklist( self, diagnostics: Optional[ExpectationDiagnostics] = None, show_failed_tests: bool = False, - backends: Optional[List[str]] = None, + backends: Optional[list[str]] = None, show_debug_messages: bool = False, ) -> str: if diagnostics is None: @@ -114,7 +114,7 @@ def run_diagnostics( # noqa: PLR0913 ignore_only_for: bool = False, for_gallery: bool = False, debug_logger: Optional[logging.Logger] = None, - only_consider_these_backends: Optional[List[str]] = None, + only_consider_these_backends: Optional[list[str]] = None, context: Optional[AbstractDataContext] = None, ) -> ExpectationDiagnostics: if debug_logger is not None: @@ -129,10 +129,10 @@ def run_diagnostics( # noqa: PLR0913 _error = lambda x: x # noqa: E731 library_metadata: AugmentedLibraryMetadata = self._get_augmented_library_metadata() - examples: List[ExpectationTestDataCases] = self._get_examples( + examples: list[ExpectationTestDataCases] = self._get_examples( return_only_gallery_examples=False ) - gallery_examples: List[ExpectationTestDataCases] = [] + gallery_examples: list[ExpectationTestDataCases] = [] for example in examples: _tests_to_include = [test for test in example.tests if test.include_in_gallery] example = copy.deepcopy(example) # noqa: PLW2901 @@ -152,7 +152,7 @@ def run_diagnostics( # noqa: PLR0913 f"Was NOT able to get Expectation configuration for {self._expectation.expectation_type}. " # noqa: E501 "Is there at least one sample test where 'success' is True?" ) - metric_diagnostics_list: List[ExpectationMetricDiagnostics] = ( + metric_diagnostics_list: list[ExpectationMetricDiagnostics] = ( self._get_metric_diagnostics_list( expectation_config=_expectation_config, ) @@ -174,7 +174,7 @@ def run_diagnostics( # noqa: PLR0913 ) _debug("Getting test results") - test_results: List[ExpectationTestDiagnostics] = self._get_test_results( + test_results: list[ExpectationTestDiagnostics] = self._get_test_results( expectation_type=description_diagnostics.snake_name, test_data_cases=examples, execution_engine_diagnostics=introspected_execution_engines, @@ -186,11 +186,11 @@ def run_diagnostics( # noqa: PLR0913 context=context, ) - backend_test_result_counts: List[ExpectationBackendTestResultCounts] = ( + backend_test_result_counts: list[ExpectationBackendTestResultCounts] = ( ExpectationDiagnostics._get_backends_from_test_results(test_results) ) - renderers: List[ExpectationRendererDiagnostics] = self._get_renderer_diagnostics( + renderers: list[ExpectationRendererDiagnostics] = self._get_renderer_diagnostics( expectation_type=description_diagnostics.snake_name, test_diagnostics=test_results, registered_renderers=_registered_renderers, # type: ignore[arg-type] @@ -295,9 +295,9 @@ def _get_maturity_checklist( self, library_metadata: Union[AugmentedLibraryMetadata, ExpectationDescriptionDiagnostics], description: ExpectationDescriptionDiagnostics, - examples: List[ExpectationTestDataCases], - tests: List[ExpectationTestDiagnostics], - backend_test_result_counts: List[ExpectationBackendTestResultCounts], + examples: list[ExpectationTestDataCases], + tests: list[ExpectationTestDiagnostics], + backend_test_result_counts: list[ExpectationBackendTestResultCounts], ) -> ExpectationDiagnosticMaturityMessages: """Generate maturity checklist messages""" experimental_checks = [] @@ -333,7 +333,7 @@ def _get_maturity_checklist( @staticmethod def _get_coverage_score( - backend_test_result_counts: List[ExpectationBackendTestResultCounts], + backend_test_result_counts: list[ExpectationBackendTestResultCounts], execution_engines: ExpectationExecutionEngineDiagnostics, ) -> float: """Generate coverage score""" @@ -380,7 +380,7 @@ def _get_examples_from_json(self): def _get_examples( # noqa: C901 - too complex self, return_only_gallery_examples: bool = True - ) -> List[ExpectationTestDataCases]: + ) -> list[ExpectationTestDataCases]: """ Get a list of examples from the object's `examples` member variable. @@ -390,7 +390,7 @@ def _get_examples( # noqa: C901 - too complex :return: list of examples or [], if no examples exist """ # noqa: E501 # Currently, only community contrib expectations have an examples attribute - all_examples: List[dict] = self._expectation.examples or self._get_examples_from_json() + all_examples: list[dict] = self._expectation.examples or self._get_examples_from_json() included_examples = [] for i, example in enumerate(all_examples, 1): @@ -474,7 +474,7 @@ def _get_description_diagnostics(self) -> ExpectationDescriptionDiagnostics: def _get_expectation_configuration_from_examples( # noqa: C901 - too complex self, - examples: List[ExpectationTestDataCases], + examples: list[ExpectationTestDataCases], ) -> Optional[ExpectationConfiguration]: """Return an ExpectationConfiguration instance using test input expected to succeed""" if examples: @@ -502,9 +502,9 @@ def _get_expectation_configuration_from_examples( # noqa: C901 - too complex @staticmethod def _get_execution_engine_diagnostics( - metric_diagnostics_list: List[ExpectationMetricDiagnostics], + metric_diagnostics_list: list[ExpectationMetricDiagnostics], registered_metrics: dict, - execution_engine_names: Optional[List[str]] = None, + execution_engine_names: Optional[list[str]] = None, ) -> ExpectationExecutionEngineDiagnostics: """Check to see which execution_engines are fully supported for this Expectation. @@ -544,7 +544,7 @@ def _get_execution_engine_diagnostics( def _get_metric_diagnostics_list( self, expectation_config: Optional[ExpectationConfiguration], - ) -> List[ExpectationMetricDiagnostics]: + ) -> list[ExpectationMetricDiagnostics]: """Check to see which Metrics are upstream validation_dependencies for this Expectation.""" # NOTE: Abe 20210102: Strictly speaking, identifying upstream metrics shouldn't need to rely on an expectation config. # noqa: E501 @@ -558,7 +558,7 @@ def _get_metric_diagnostics_list( ) metric_name: str - metric_diagnostics_list: List[ExpectationMetricDiagnostics] = [ + metric_diagnostics_list: list[ExpectationMetricDiagnostics] = [ ExpectationMetricDiagnostics( name=metric_name, has_question_renderer=False, @@ -572,15 +572,15 @@ def _get_metric_diagnostics_list( def _get_test_results( # noqa: PLR0913 cls, expectation_type: str, - test_data_cases: List[ExpectationTestDataCases], + test_data_cases: list[ExpectationTestDataCases], execution_engine_diagnostics: ExpectationExecutionEngineDiagnostics, raise_exceptions_for_backends: bool = False, ignore_suppress: bool = False, ignore_only_for: bool = False, debug_logger: Optional[logging.Logger] = None, - only_consider_these_backends: Optional[List[str]] = None, + only_consider_these_backends: Optional[list[str]] = None, context: Optional[AbstractDataContext] = None, - ) -> List[ExpectationTestDiagnostics]: + ) -> list[ExpectationTestDiagnostics]: """Generate test results. This is an internal method for run_diagnostics.""" if debug_logger is not None: @@ -695,12 +695,12 @@ def _get_test_results( # noqa: PLR0913 def _get_renderer_diagnostics( self, expectation_type: str, - test_diagnostics: List[ExpectationTestDiagnostics], - registered_renderers: List[str], + test_diagnostics: list[ExpectationTestDiagnostics], + registered_renderers: list[str], standard_renderers: Optional[ - List[Union[str, LegacyRendererType, LegacyDiagnosticRendererType]] + list[Union[str, LegacyRendererType, LegacyDiagnosticRendererType]] ] = None, - ) -> List[ExpectationRendererDiagnostics]: + ) -> list[ExpectationRendererDiagnostics]: """Generate Renderer diagnostics for this Expectation, based primarily on a list of ExpectationTestDiagnostics.""" # noqa: E501 if not standard_renderers: @@ -773,7 +773,7 @@ def _get_renderer_diagnostics( def _get_registered_renderers( expectation_type: str, registered_renderers: dict, - ) -> List[str]: + ) -> list[str]: """Get a list of supported renderers for this Expectation, in sorted order.""" supported_renderers = list(registered_renderers[expectation_type].keys()) supported_renderers.sort() diff --git a/great_expectations/core/expectation_diagnostics/expectation_test_data_cases.py b/great_expectations/core/expectation_diagnostics/expectation_test_data_cases.py index c72535aeadeb..0131b0d0d8a7 100644 --- a/great_expectations/core/expectation_diagnostics/expectation_test_data_cases.py +++ b/great_expectations/core/expectation_diagnostics/expectation_test_data_cases.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Optional from great_expectations.types import SerializableDictDot @@ -24,7 +24,7 @@ class Backend(Enum): @dataclass class TestBackend: backend: str - dialects: List[str] + dialects: list[str] __test__ = False # Tell pytest not to try to collect this class as a test @@ -53,12 +53,12 @@ class ExpectationTestCase(SerializableDictDot): """A single test case, with input arguments and output""" title: str - input: Dict[str, Any] - output: Dict[str, Any] + input: dict[str, Any] + output: dict[str, Any] exact_match_out: bool - suppress_test_for: List[str] = field(default_factory=list) + suppress_test_for: list[str] = field(default_factory=list) include_in_gallery: bool = False - only_for: Optional[List[str]] = None + only_for: Optional[list[str]] = None class ExpectationLegacyTestCaseAdapter(ExpectationTestCase): @@ -98,7 +98,7 @@ class ExpectationTestDataCases(SerializableDictDot): data: TestData dataset_name: str - tests: List[ExpectationTestCase] - schemas: Dict[Backend, Dict[str, str]] = field(default_factory=dict) - test_backends: Optional[List[TestBackend]] = None + tests: list[ExpectationTestCase] + schemas: dict[Backend, dict[str, str]] = field(default_factory=dict) + test_backends: Optional[list[TestBackend]] = None data_alt: Optional[TestData] = None diff --git a/great_expectations/core/expectation_diagnostics/supporting_types.py b/great_expectations/core/expectation_diagnostics/supporting_types.py index fd1e0b990be8..6b2c5a83c601 100644 --- a/great_expectations/core/expectation_diagnostics/supporting_types.py +++ b/great_expectations/core/expectation_diagnostics/supporting_types.py @@ -4,7 +4,7 @@ import logging from dataclasses import dataclass, field from enum import Enum -from typing import List, Optional, Sequence, Union +from typing import Optional, Sequence, Union from typing_extensions import TypedDict @@ -28,13 +28,13 @@ class AugmentedLibraryMetadata(SerializableDictDot): """An augmented version of the Expectation.library_metadata object, used within ExpectationDiagnostics""" # noqa: E501 maturity: Maturity - tags: List[str] - contributors: List[str] - requirements: List[str] + tags: list[str] + contributors: list[str] + requirements: list[str] library_metadata_passed_checks: bool has_full_test_suite: bool manually_reviewed_code: bool - problems: List[str] = field(default_factory=list) + problems: list[str] = field(default_factory=list) legacy_maturity_level_substitutions = { "experimental": "EXPERIMENTAL", @@ -97,7 +97,7 @@ class ExpectationRendererDiagnostics(SerializableDictDot): name: str is_supported: bool is_standard: bool - samples: List[RendererTestDiagnostics] + samples: list[RendererTestDiagnostics] @dataclass @@ -144,7 +144,7 @@ class ExpectationBackendTestResultCounts(SerializableDictDot): backend: str num_passed: int num_failed: int - failing_names: Optional[List[str]] + failing_names: Optional[list[str]] class ExpectationDiagnosticCheckMessageDict(TypedDict): @@ -168,6 +168,6 @@ class ExpectationDiagnosticCheckMessage(SerializableDictDot): class ExpectationDiagnosticMaturityMessages(SerializableDictDot): """A holder for ExpectationDiagnosticCheckMessages, grouping them by maturity level. Used within the ExpectationDiagnostic object.""" # noqa: E501 - experimental: List[ExpectationDiagnosticCheckMessage] - beta: List[ExpectationDiagnosticCheckMessage] - production: List[ExpectationDiagnosticCheckMessage] + experimental: list[ExpectationDiagnosticCheckMessage] + beta: list[ExpectationDiagnosticCheckMessage] + production: list[ExpectationDiagnosticCheckMessage] diff --git a/great_expectations/core/expectation_suite.py b/great_expectations/core/expectation_suite.py index beaab79a59af..178fbce22a59 100644 --- a/great_expectations/core/expectation_suite.py +++ b/great_expectations/core/expectation_suite.py @@ -6,8 +6,6 @@ from copy import deepcopy from typing import ( TYPE_CHECKING, - Dict, - List, Optional, Sequence, TypeVar, @@ -362,7 +360,7 @@ def __deepcopy__(self, memo: dict): @public_api @override - def to_json_dict(self) -> Dict[str, JSONValues]: + def to_json_dict(self) -> dict[str, JSONValues]: """Returns a JSON-serializable dict representation of this ExpectationSuite. Returns: @@ -386,7 +384,7 @@ def remove_expectation( match_type: str = "domain", remove_multiple_matches: bool = False, id: Optional[Union[str, uuid.UUID]] = None, - ) -> List[ExpectationConfiguration]: + ) -> list[ExpectationConfiguration]: """Remove an ExpectationConfiguration from the ExpectationSuite. Args: @@ -444,7 +442,7 @@ def _find_expectation_indexes( expectation_configuration: Optional[ExpectationConfiguration] = None, match_type: str = "domain", id: Optional[str] = None, - ) -> List[int]: + ) -> list[int]: """ Find indexes of Expectations matching the given ExpectationConfiguration on the given match_type. If a id is provided, match_type is ignored and only indexes of Expectations @@ -555,10 +553,10 @@ def _add_expectation( def add_expectation_configurations( self, - expectation_configurations: List[ExpectationConfiguration], + expectation_configurations: list[ExpectationConfiguration], match_type: str = "domain", overwrite_existing: bool = True, - ) -> List[ExpectationConfiguration]: + ) -> list[ExpectationConfiguration]: """Upsert a list of ExpectationConfigurations into this ExpectationSuite. Args: @@ -577,7 +575,7 @@ def add_expectation_configurations( One match if overwrite_existing = False """ # noqa: E501 expectation_configuration: ExpectationConfiguration - expectation_configurations_attempted_to_be_added: List[ExpectationConfiguration] = [ + expectation_configurations_attempted_to_be_added: list[ExpectationConfiguration] = [ self.add_expectation_configuration( expectation_configuration=expectation_configuration, match_type=match_type, diff --git a/great_expectations/core/expectation_validation_result.py b/great_expectations/core/expectation_validation_result.py index de8c15b62752..4369f4d54639 100644 --- a/great_expectations/core/expectation_validation_result.py +++ b/great_expectations/core/expectation_validation_result.py @@ -3,7 +3,7 @@ import json import logging from copy import deepcopy -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union from marshmallow import Schema, fields, post_dump, post_load, pre_dump from typing_extensions import TypedDict @@ -89,7 +89,7 @@ def __init__( # noqa: PLR0913 result: Optional[dict] = None, meta: Optional[dict] = None, exception_info: Optional[dict] = None, - rendered_content: Union[RenderedAtomicContent, List[RenderedAtomicContent], None] = None, + rendered_content: Union[RenderedAtomicContent, list[RenderedAtomicContent], None] = None, **kwargs: dict, ) -> None: if result and not self.validate_result_dict(result): @@ -225,7 +225,7 @@ def render(self) -> None: class_name=inline_renderer_config["class_name"], ) - rendered_content: List[RenderedAtomicContent] = inline_renderer.get_rendered_content() + rendered_content: list[RenderedAtomicContent] = inline_renderer.get_rendered_content() self.rendered_content = [ content_block diff --git a/great_expectations/core/freshness_diagnostics.py b/great_expectations/core/freshness_diagnostics.py index 147fdbfc5b02..819d36406b01 100644 --- a/great_expectations/core/freshness_diagnostics.py +++ b/great_expectations/core/freshness_diagnostics.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import ClassVar, Tuple, Type +from typing import ClassVar from great_expectations.compatibility.typing_extensions import override from great_expectations.exceptions import ( @@ -31,7 +31,7 @@ class FreshnessDiagnostics: unexpected behavior. """ - raise_for_error_class: ClassVar[Type[ResourceFreshnessAggregateError]] = ( + raise_for_error_class: ClassVar[type[ResourceFreshnessAggregateError]] = ( ResourceFreshnessAggregateError ) errors: list[GreatExpectationsError] @@ -68,8 +68,8 @@ class _ParentFreshnessDiagnostics(FreshnessDiagnostics): All errors throughout the hierarchy should be collected in the parent diagnostics object. """ - parent_error_class: ClassVar[Type[GreatExpectationsError]] - children_error_classes: ClassVar[Tuple[Type[GreatExpectationsError], ...]] + parent_error_class: ClassVar[type[GreatExpectationsError]] + children_error_classes: ClassVar[tuple[type[GreatExpectationsError], ...]] def update_with_children(self, *children_diagnostics: FreshnessDiagnostics) -> None: for diagnostics in children_diagnostics: @@ -92,22 +92,22 @@ def raise_for_error(self) -> None: @dataclass class ValidationDefinitionFreshnessDiagnostics(_ParentFreshnessDiagnostics): - parent_error_class: ClassVar[Type[GreatExpectationsError]] = ValidationDefinitionNotAddedError - children_error_classes: ClassVar[Tuple[Type[GreatExpectationsError], ...]] = ( + parent_error_class: ClassVar[type[GreatExpectationsError]] = ValidationDefinitionNotAddedError + children_error_classes: ClassVar[tuple[type[GreatExpectationsError], ...]] = ( ExpectationSuiteNotAddedError, BatchDefinitionNotAddedError, ) - raise_for_error_class: ClassVar[Type[ResourceFreshnessAggregateError]] = ( + raise_for_error_class: ClassVar[type[ResourceFreshnessAggregateError]] = ( ValidationDefinitionRelatedResourcesFreshnessError ) @dataclass class CheckpointFreshnessDiagnostics(_ParentFreshnessDiagnostics): - parent_error_class: ClassVar[Type[GreatExpectationsError]] = CheckpointNotAddedError - children_error_classes: ClassVar[Tuple[Type[GreatExpectationsError], ...]] = ( + parent_error_class: ClassVar[type[GreatExpectationsError]] = CheckpointNotAddedError + children_error_classes: ClassVar[tuple[type[GreatExpectationsError], ...]] = ( ValidationDefinitionNotAddedError, ) - raise_for_error_class: ClassVar[Type[ResourceFreshnessAggregateError]] = ( + raise_for_error_class: ClassVar[type[ResourceFreshnessAggregateError]] = ( CheckpointRelatedResourcesFreshnessError ) diff --git a/great_expectations/core/id_dict.py b/great_expectations/core/id_dict.py index e5f7e078428d..0a80d9e74c25 100644 --- a/great_expectations/core/id_dict.py +++ b/great_expectations/core/id_dict.py @@ -2,7 +2,7 @@ import hashlib import json -from typing import Any, Set, TypeVar, Union +from typing import Any, TypeVar, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.util import convert_to_json_serializable # noqa: TID251 @@ -11,7 +11,7 @@ class IDDict(dict): - _id_ignore_keys: Set[str] = set() + _id_ignore_keys: set[str] = set() def to_id(self, id_keys=None, id_ignore_keys=None): if id_keys is None: diff --git a/great_expectations/core/partitioners.py b/great_expectations/core/partitioners.py index a5a53a27ea2b..3a8049fc6820 100644 --- a/great_expectations/core/partitioners.py +++ b/great_expectations/core/partitioners.py @@ -1,7 +1,7 @@ from __future__ import annotations import re -from typing import List, Literal, Tuple, Union +from typing import Literal, Union from great_expectations.compatibility import pydantic @@ -27,7 +27,7 @@ class ColumnPartitionerDaily(pydantic.BaseModel): class PartitionerDatetimePart(pydantic.BaseModel): - datetime_parts: List[str] + datetime_parts: list[str] column_name: str sort_ascending: bool = True method_name: Literal["partition_on_date_parts"] = "partition_on_date_parts" @@ -54,7 +54,7 @@ class PartitionerColumnValue(pydantic.BaseModel): class PartitionerMultiColumnValue(pydantic.BaseModel): - column_names: List[str] + column_names: list[str] sort_ascending: bool = True method_name: Literal["partition_on_multi_column_values"] = "partition_on_multi_column_values" @@ -81,25 +81,25 @@ class PartitionerConvertedDatetime(pydantic.BaseModel): class FileNamePartitionerYearly(pydantic.BaseModel): regex: re.Pattern - param_names: Tuple[Literal["year"]] = ("year",) + param_names: tuple[Literal["year"]] = ("year",) sort_ascending: bool = True class FileNamePartitionerMonthly(pydantic.BaseModel): regex: re.Pattern - param_names: Tuple[Literal["year"], Literal["month"]] = ("year", "month") + param_names: tuple[Literal["year"], Literal["month"]] = ("year", "month") sort_ascending: bool = True class FileNamePartitionerDaily(pydantic.BaseModel): regex: re.Pattern - param_names: Tuple[Literal["year"], Literal["month"], Literal["day"]] = ("year", "month", "day") + param_names: tuple[Literal["year"], Literal["month"], Literal["day"]] = ("year", "month", "day") sort_ascending: bool = True class FileNamePartitionerPath(pydantic.BaseModel): regex: re.Pattern - param_names: Tuple[()] = () + param_names: tuple[()] = () sort_ascending: bool = True diff --git a/great_expectations/core/run_identifier.py b/great_expectations/core/run_identifier.py index e985cb97cfb8..88bd33f4c4d6 100644 --- a/great_expectations/core/run_identifier.py +++ b/great_expectations/core/run_identifier.py @@ -4,7 +4,7 @@ import json import warnings from copy import deepcopy -from typing import Dict, Optional, Union +from typing import Optional, Union from dateutil.parser import parse from marshmallow import Schema, fields, post_load, pre_dump @@ -88,7 +88,7 @@ def __str__(self): return json.dumps(self.to_json_dict(), indent=2) @public_api - def to_json_dict(self) -> Dict[str, JSONValues]: + def to_json_dict(self) -> dict[str, JSONValues]: """Returns a JSON-serializable dict representation of this RunIdentifier. Returns: diff --git a/great_expectations/core/suite_parameters.py b/great_expectations/core/suite_parameters.py index ead7ca380189..695d7805a1a0 100644 --- a/great_expectations/core/suite_parameters.py +++ b/great_expectations/core/suite_parameters.py @@ -7,7 +7,7 @@ import operator import traceback from collections import namedtuple -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union import dateutil from pyparsing import ( @@ -239,7 +239,7 @@ def build_suite_parameters( suite_parameters: Optional[dict] = None, interactive_evaluation: bool = True, data_context=None, -) -> Tuple[dict, dict]: +) -> tuple[dict, dict]: """Build a dictionary of parameters to evaluate, using the provided suite_parameters, AND mutate expectation_args by removing any parameter values passed in as temporary values during exploratory work. @@ -283,7 +283,7 @@ def build_suite_parameters( def parse_suite_parameter( # noqa: C901 parameter_expression: str, - suite_parameters: Optional[Dict[str, Any]] = None, + suite_parameters: Optional[dict[str, Any]] = None, data_context: Optional[AbstractDataContext] = None, ) -> Any: """Use the provided suite_parameters dict to parse a given parameter expression. diff --git a/great_expectations/data_context/data_context/abstract_data_context.py b/great_expectations/data_context/data_context/abstract_data_context.py index 1cd57b3e6a9b..650a8e3902dd 100644 --- a/great_expectations/data_context/data_context/abstract_data_context.py +++ b/great_expectations/data_context/data_context/abstract_data_context.py @@ -15,12 +15,9 @@ TYPE_CHECKING, Any, Callable, - Dict, - List, Mapping, Optional, Sequence, - Tuple, TypeVar, Union, overload, @@ -122,8 +119,8 @@ from great_expectations.execution_engine import ExecutionEngine from great_expectations.render.renderer.site_builder import SiteBuilder -BlockConfigDataAssetNames: TypeAlias = Dict[str, List[str]] -FluentDataAssetNames: TypeAlias = List[str] +BlockConfigDataAssetNames: TypeAlias = dict[str, list[str]] +FluentDataAssetNames: TypeAlias = list[str] logger = logging.getLogger(__name__) yaml = YAMLHandler() @@ -373,7 +370,7 @@ def instance_id(self) -> str: return instance_id @property - def config_variables(self) -> Dict: + def config_variables(self) -> dict: """Loads config variables into cache, by calling _load_config_variables() Returns: A dictionary containing config_variables from file or empty dictionary. @@ -800,7 +797,7 @@ def add_or_update_datasource( return return_datasource - def get_site_names(self) -> List[str]: + def get_site_names(self) -> list[str]: """Get a list of configured site names.""" return list(self.variables.data_docs_sites.keys()) # type: ignore[union-attr] @@ -817,7 +814,7 @@ def get_config_with_variables_substituted( config = self._project_config return DataContextConfig(**self.config_provider.substitute_config(config)) - def list_stores(self) -> List[Store]: + def list_stores(self) -> list[Store]: """List currently-configured Stores on this context""" stores = [] for ( @@ -830,14 +827,14 @@ def list_stores(self) -> List[Store]: stores.append(masked_config) return stores # type: ignore[return-value] - def list_active_stores(self) -> List[Store]: + def list_active_stores(self) -> list[Store]: """ List active Stores on this context. Active stores are identified by setting the following parameters: expectations_store_name, validation_results_store_name, checkpoint_store_name """ # noqa: E501 - active_store_names: List[str] = [ + active_store_names: list[str] = [ self.expectations_store_name, # type: ignore[list-item] self.validation_results_store_name, # type: ignore[list-item] ] @@ -993,7 +990,7 @@ def delete_store(self, name: str) -> None: self._save_project_config() - def list_datasources(self) -> List[dict]: + def list_datasources(self) -> list[dict]: """List the configurations of the datasources associated with this context. Note that any sensitive values are obfuscated before being returned. @@ -1028,9 +1025,9 @@ def get_validator( # noqa: PLR0913 data_connector_name: Optional[str] = None, data_asset_name: Optional[str] = None, batch: Optional[Batch] = None, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, FluentBatchRequest]] = None, - batch_request_list: Optional[List[BatchRequestBase]] = None, + batch_request_list: Optional[list[BatchRequestBase]] = None, batch_data: Optional[Any] = None, data_connector_query: Optional[Union[IDDict, dict]] = None, batch_identifiers: Optional[dict] = None, @@ -1147,9 +1144,9 @@ def _get_batch_list_from_inputs( # noqa: PLR0913 data_connector_name: str | None, data_asset_name: str | None, batch: Batch | None, - batch_list: List[Batch] | None, + batch_list: list[Batch] | None, batch_request: BatchRequestBase | FluentBatchRequest | None, - batch_request_list: List[BatchRequestBase] | None, + batch_request_list: list[BatchRequestBase] | None, batch_data: Any, data_connector_query: Union[IDDict, dict] | None, batch_identifiers: dict | None, @@ -1166,7 +1163,7 @@ def _get_batch_list_from_inputs( # noqa: PLR0913 batch_filter_parameters: dict | None, batch_spec_passthrough: dict | None, **kwargs, - ) -> List[Batch]: + ) -> list[Batch]: if ( sum( bool(x) @@ -1189,7 +1186,7 @@ def _get_batch_list_from_inputs( # noqa: PLR0913 if batch: return [batch] - computed_batch_list: List[Batch] = [] + computed_batch_list: list[Batch] = [] if not batch_request_list: # batch_request could actually be None here since we do explicit None checks in the # sum check above while here we do a truthy check. @@ -1621,8 +1618,8 @@ def get_docs_sites_urls( | None = None, site_name: Optional[str] = None, only_if_exists: bool = True, - site_names: Optional[List[str]] = None, - ) -> List[Dict[str, Optional[str]]]: + site_names: Optional[list[str]] = None, + ) -> list[dict[str, Optional[str]]]: """ Get URLs for a resource for all data docs sites. @@ -1669,7 +1666,7 @@ def get_docs_sites_urls( ) return [{"site_name": site_name, "site_url": url}] - site_urls: List[Dict[str, Optional[str]]] = [] + site_urls: list[dict[str, Optional[str]]] = [] for _site_name, site_config in sites.items(): site_builder = self._load_site_builder_from_site_config(site_config) url = site_builder.get_resource_url( @@ -1782,7 +1779,7 @@ def _get_global_config_value( @staticmethod def _get_metric_configuration_tuples( # noqa: C901 metric_configuration: Union[str, dict], base_kwargs: Optional[dict] = None - ) -> List[Tuple[str, Union[dict, Any]]]: + ) -> list[tuple[str, Union[dict, Any]]]: if base_kwargs is None: base_kwargs = {} @@ -1871,7 +1868,7 @@ def _normalize_absolute_or_relative_path(self, path: Optional[str]) -> Optional[ else: return os.path.join(self.root_directory, path) # type: ignore[arg-type] # noqa: PTH118 - def _load_config_variables(self) -> Dict: + def _load_config_variables(self) -> dict: config_var_provider = self.config_provider.get_provider( _ConfigurationVariablesConfigurationProvider ) @@ -1921,7 +1918,7 @@ def progress_bars(self) -> Optional[ProgressBarsConfig]: # TODO: All datasources should now be fluent so we should be able to delete this @property - def fluent_datasources(self) -> Dict[str, FluentDatasource]: + def fluent_datasources(self) -> dict[str, FluentDatasource]: return { name: ds for (name, ds) in self.data_sources.all().items() @@ -1932,7 +1929,7 @@ def fluent_datasources(self) -> Dict[str, FluentDatasource]: def data_context_id(self) -> uuid.UUID | None: return self.variables.data_context_id - def _init_primary_stores(self, store_configs: Dict[str, StoreConfigTypedDict]) -> None: + def _init_primary_stores(self, store_configs: dict[str, StoreConfigTypedDict]) -> None: """Initialize all Stores for this DataContext. Stores are a good fit for reading/writing objects that: @@ -2432,7 +2429,7 @@ def _attach_fluent_config_datasources_and_build_data_connectors(self, config: Gx # since we are loading the datasource from existing config, we do not need to save it self._add_fluent_datasource(datasource=datasource, save_changes=False) - def _synchronize_fluent_datasources(self) -> Dict[str, FluentDatasource]: + def _synchronize_fluent_datasources(self) -> dict[str, FluentDatasource]: """ Update `self.fluent_config.fluent_datasources` with any newly added datasources. Should be called before serializing `fluent_config`. diff --git a/great_expectations/data_context/data_context/cloud_data_context.py b/great_expectations/data_context/data_context/cloud_data_context.py index 041237806909..12f02d348b2b 100644 --- a/great_expectations/data_context/data_context/cloud_data_context.py +++ b/great_expectations/data_context/data_context/cloud_data_context.py @@ -5,7 +5,6 @@ import uuid from typing import ( TYPE_CHECKING, - Dict, Mapping, Optional, Union, @@ -413,7 +412,7 @@ def _get_cloud_config_dict( cloud_base_url: Optional[str] = None, cloud_access_token: Optional[str] = None, cloud_organization_id: Optional[str] = None, - ) -> Dict[GXCloudEnvironmentVariable, Optional[str]]: + ) -> dict[GXCloudEnvironmentVariable, Optional[str]]: cloud_base_url = ( cloud_base_url or cls._get_global_config_value( diff --git a/great_expectations/data_context/data_context/context_factory.py b/great_expectations/data_context/data_context/context_factory.py index 568a828fb56a..bc6758321b23 100644 --- a/great_expectations/data_context/data_context/context_factory.py +++ b/great_expectations/data_context/data_context/context_factory.py @@ -9,7 +9,6 @@ Literal, Mapping, Optional, - Type, overload, ) @@ -206,10 +205,7 @@ def _build_context( # noqa: PLR0913 expected_ctx_types: dict[ ContextModes | None, - Type[CloudDataContext] - | Type[EphemeralDataContext] - | Type[FileDataContext] - | Type[AbstractDataContext], + type[CloudDataContext | EphemeralDataContext | FileDataContext | AbstractDataContext], ] = { "ephemeral": EphemeralDataContext, "file": FileDataContext, diff --git a/great_expectations/data_context/data_context/file_data_context.py b/great_expectations/data_context/data_context/file_data_context.py index fbae32bb4273..b295ef8aa8b8 100644 --- a/great_expectations/data_context/data_context/file_data_context.py +++ b/great_expectations/data_context/data_context/file_data_context.py @@ -196,13 +196,7 @@ def _load_file_backed_project_config( except OSError: raise gx_exceptions.ConfigNotFoundError() - try: - return DataContextConfig.from_commented_map( - commented_map=config_commented_map_from_yaml - ) - except gx_exceptions.InvalidDataContextConfigError: # noqa: TRY203 - # Just to be explicit about what we intended to catch - raise + return DataContextConfig.from_commented_map(commented_map=config_commented_map_from_yaml) @override def _load_fluent_config(self, config_provider: _ConfigurationProvider) -> GxConfig: diff --git a/great_expectations/data_context/data_context_variables.py b/great_expectations/data_context/data_context_variables.py index 9820d59f32a6..ebe147aa2611 100644 --- a/great_expectations/data_context/data_context_variables.py +++ b/great_expectations/data_context/data_context_variables.py @@ -6,7 +6,7 @@ import uuid from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional +from typing import TYPE_CHECKING, Any, Generator, Optional from great_expectations._docs_decorators import public_api from great_expectations.compatibility.typing_extensions import override @@ -310,7 +310,7 @@ def _fluent_objects_stash( NOTE: This could be generalized into a stand-alone context manager function, but it would need to take in the data_context containing the fluent objects. """ - config_fluent_datasources_stash: Dict[str, FluentDatasource] = ( + config_fluent_datasources_stash: dict[str, FluentDatasource] = ( self.data_context._synchronize_fluent_datasources() ) try: diff --git a/great_expectations/data_context/store/_store_backend.py b/great_expectations/data_context/store/_store_backend.py index 5a5f8880512e..b8ad5f3a813c 100644 --- a/great_expectations/data_context/store/_store_backend.py +++ b/great_expectations/data_context/store/_store_backend.py @@ -4,7 +4,7 @@ import urllib import uuid from abc import ABCMeta, abstractmethod -from typing import Any, List, Optional, Union +from typing import Any, Optional, Union import pyparsing as pp @@ -228,7 +228,7 @@ def _move(self, source_key, dest_key, **kwargs) -> None: raise NotImplementedError @abstractmethod - def list_keys(self, prefix=()) -> Union[List[str], List[tuple]]: + def list_keys(self, prefix=()) -> Union[list[str], list[tuple]]: raise NotImplementedError @abstractmethod diff --git a/great_expectations/data_context/store/data_context_store.py b/great_expectations/data_context/store/data_context_store.py index fa021bd242c6..b00c13351deb 100644 --- a/great_expectations/data_context/store/data_context_store.py +++ b/great_expectations/data_context/store/data_context_store.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Set, Union +from typing import Union from great_expectations.compatibility.typing_extensions import override from great_expectations.data_context.data_context_variables import ( @@ -20,7 +20,7 @@ class DataContextStore(ConfigurationStore): _configuration_class = DataContextConfig - cloud_exclude_field_names: Set[DataContextVariableSchema] = { + cloud_exclude_field_names: set[DataContextVariableSchema] = { DataContextVariableSchema.CHECKPOINT_STORE_NAME, DataContextVariableSchema.DATASOURCES, DataContextVariableSchema.EXPECTATIONS_STORE_NAME, diff --git a/great_expectations/data_context/store/expectations_store.py b/great_expectations/data_context/store/expectations_store.py index c659f5ae3a7e..b25079a9d0b2 100644 --- a/great_expectations/data_context/store/expectations_store.py +++ b/great_expectations/data_context/store/expectations_store.py @@ -1,7 +1,7 @@ from __future__ import annotations import uuid -from typing import TYPE_CHECKING, Any, Dict, List, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union import great_expectations.exceptions as gx_exceptions from great_expectations.compatibility import pydantic @@ -38,7 +38,7 @@ class Config: id: str type: str - rendered_content: List[dict] = pydantic.Field(default_factory=list) + rendered_content: list[dict] = pydantic.Field(default_factory=list) kwargs: dict meta: Union[dict, None] expectation_context: Union[dict, None] @@ -52,7 +52,7 @@ class Config: name: str id: str - expectations: List[ExpectationConfigurationDTO] + expectations: list[ExpectationConfigurationDTO] meta: Union[dict, None] notes: Union[str, None] @@ -113,7 +113,7 @@ def gx_cloud_response_json_to_object_dict(cls, response_json: dict) -> dict: This method takes full json response from GX cloud and outputs a dict appropriate for deserialization into a GX object """ - suite_data: Dict + suite_data: dict # if only the expectation_suite_name is passed, a list will be returned if isinstance(response_json["data"], list): if len(response_json["data"]) == 1: diff --git a/great_expectations/data_context/store/gx_cloud_store_backend.py b/great_expectations/data_context/store/gx_cloud_store_backend.py index d96567f970a5..4b632e4d39a0 100644 --- a/great_expectations/data_context/store/gx_cloud_store_backend.py +++ b/great_expectations/data_context/store/gx_cloud_store_backend.py @@ -5,7 +5,7 @@ import weakref from abc import ABCMeta from enum import Enum -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Optional, Union from urllib.parse import urljoin import requests @@ -30,11 +30,11 @@ class ErrorDetail(TypedDict): code: Optional[str] detail: Optional[str] - source: Union[str, Dict[str, str], None] + source: Union[str, dict[str, str], None] class ErrorPayload(TypedDict): - errors: List[ErrorDetail] + errors: list[ErrorDetail] class EndpointVersion(str, Enum): @@ -72,7 +72,7 @@ def get_user_friendly_error_message( class GXCloudStoreBackend(StoreBackend, metaclass=ABCMeta): - PAYLOAD_ATTRIBUTES_KEYS: Dict[GXCloudRESTResource, str] = { + PAYLOAD_ATTRIBUTES_KEYS: dict[GXCloudRESTResource, str] = { GXCloudRESTResource.CHECKPOINT: "checkpoint_config", GXCloudRESTResource.DATASOURCE: "datasource_config", GXCloudRESTResource.DATA_CONTEXT: "data_context_config", @@ -82,7 +82,7 @@ class GXCloudStoreBackend(StoreBackend, metaclass=ABCMeta): GXCloudRESTResource.VALIDATION_DEFINITION: "validation_definition", } - ALLOWED_SET_KWARGS_BY_RESOURCE_TYPE: Dict[GXCloudRESTResource, Set[str]] = { + ALLOWED_SET_KWARGS_BY_RESOURCE_TYPE: dict[GXCloudRESTResource, set[str]] = { GXCloudRESTResource.EXPECTATION_SUITE: {"clause_id"}, GXCloudRESTResource.VALIDATION_RESULT: { "checkpoint_id", @@ -121,7 +121,7 @@ class GXCloudStoreBackend(StoreBackend, metaclass=ABCMeta): def __init__( # noqa: PLR0913 self, - ge_cloud_credentials: Dict, + ge_cloud_credentials: dict, ge_cloud_base_url: str = CLOUD_DEFAULT_BASE_URL, ge_cloud_resource_type: Optional[Union[str, GXCloudRESTResource]] = None, ge_cloud_resource_name: Optional[str] = None, @@ -183,7 +183,7 @@ def __init__( # noqa: PLR0913 @override def _get( # type: ignore[override] - self, key: Tuple[GXCloudRESTResource, str | None, str | None] + self, key: tuple[GXCloudRESTResource, str | None, str | None] ) -> dict: url = self.get_url_for_key(key=key) @@ -327,7 +327,7 @@ def _put(self, id: str, value: Any) -> GXCloudResourceRef | bool: ) from e @property - def allowed_set_kwargs(self) -> Set[str]: + def allowed_set_kwargs(self) -> set[str]: return self.ALLOWED_SET_KWARGS_BY_RESOURCE_TYPE.get(self.ge_cloud_resource_type, set()) def validate_set_kwargs(self, kwargs: dict) -> Union[bool, None]: @@ -346,7 +346,7 @@ def validate_set_kwargs(self, kwargs: dict) -> Union[bool, None]: @override def _set( # type: ignore[override] self, - key: Tuple[GXCloudRESTResource, ...], + key: tuple[GXCloudRESTResource, ...], value: Any, **kwargs, ) -> Union[bool, GXCloudResourceRef]: @@ -434,7 +434,7 @@ def ge_cloud_credentials(self) -> dict: return self._ge_cloud_credentials @override - def list_keys(self, prefix: Tuple = ()) -> List[Tuple[GXCloudRESTResource, str, str]]: + def list_keys(self, prefix: tuple = ()) -> list[tuple[GXCloudRESTResource, str, str]]: url = self.construct_versioned_url( base_url=self.ge_cloud_base_url, organization_id=self.ge_cloud_credentials["organization_id"], @@ -469,7 +469,7 @@ def list_keys(self, prefix: Tuple = ()) -> List[Tuple[GXCloudRESTResource, str, @override def get_url_for_key( self, - key: Tuple[GXCloudRESTResource, str | None, str | None], + key: tuple[GXCloudRESTResource, str | None, str | None], protocol: Optional[Any] = None, ) -> str: id = key[1] @@ -591,7 +591,7 @@ def _add_or_update(self, key, value, **kwargs): # type: ignore[explicit-overrid return self.add(key=key, value=value, **kwargs) @override - def _has_key(self, key: Tuple[GXCloudRESTResource, str | None, str | None]) -> bool: + def _has_key(self, key: tuple[GXCloudRESTResource, str | None, str | None]) -> bool: try: _ = self._get(key) return True diff --git a/great_expectations/data_context/store/json_site_store.py b/great_expectations/data_context/store/json_site_store.py index 44b8e7170b9b..5943c84bf896 100644 --- a/great_expectations/data_context/store/json_site_store.py +++ b/great_expectations/data_context/store/json_site_store.py @@ -1,7 +1,6 @@ from __future__ import annotations from json import loads -from typing import Dict from great_expectations.compatibility.typing_extensions import override from great_expectations.data_context.store.store import Store @@ -48,7 +47,7 @@ def __init__(self, store_backend=None, runtime_environment=None, store_name=None @override @staticmethod - def gx_cloud_response_json_to_object_dict(response_json: Dict) -> Dict: + def gx_cloud_response_json_to_object_dict(response_json: dict) -> dict: """ This method takes full json response from GX cloud and outputs a dict appropriate for deserialization into a GX object diff --git a/great_expectations/data_context/store/metric_store.py b/great_expectations/data_context/store/metric_store.py index b95c3ac5d354..04c400890b6e 100644 --- a/great_expectations/data_context/store/metric_store.py +++ b/great_expectations/data_context/store/metric_store.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from typing import ClassVar, Type +from typing import ClassVar from great_expectations.data_context.store.database_store_backend import ( DatabaseStoreBackend, @@ -21,7 +21,7 @@ class MetricStore(Store): A MetricStore stores ValidationMetric information to be used between runs. """ - _key_class: ClassVar[Type] = ValidationMetricIdentifier + _key_class: ClassVar[type] = ValidationMetricIdentifier def __init__(self, store_backend=None, store_name=None) -> None: if store_backend is not None: diff --git a/great_expectations/data_context/store/query_store.py b/great_expectations/data_context/store/query_store.py index ff4ca68eefce..e41fb600c32f 100644 --- a/great_expectations/data_context/store/query_store.py +++ b/great_expectations/data_context/store/query_store.py @@ -2,7 +2,7 @@ import logging from string import Template -from typing import ClassVar, Type +from typing import ClassVar import great_expectations.exceptions as gx_exceptions from great_expectations.compatibility import sqlalchemy @@ -27,7 +27,7 @@ class SqlAlchemyQueryStore(Store): """SqlAlchemyQueryStore stores queries by name, and makes it possible to retrieve the resulting value by query name.""" # noqa: E501 - _key_class: ClassVar[Type] = StringKey + _key_class: ClassVar[type] = StringKey def __init__( self, diff --git a/great_expectations/data_context/store/store.py b/great_expectations/data_context/store/store.py index 576460aea194..8406529a5d74 100644 --- a/great_expectations/data_context/store/store.py +++ b/great_expectations/data_context/store/store.py @@ -6,11 +6,7 @@ TYPE_CHECKING, Any, ClassVar, - Dict, - List, Optional, - Tuple, - Type, ) from marshmallow import ValidationError as MarshmallowValidationError @@ -74,7 +70,7 @@ class Store: All keys must have a to_tuple() method. """ - _key_class: ClassVar[Type] = DataContextKey + _key_class: ClassVar[type] = DataContextKey def __init__( self, @@ -123,7 +119,7 @@ def _determine_store_backend_class(store_backend: dict | None) -> type: return load_class(store_backend_class_name, store_backend_module_name) @classmethod - def gx_cloud_response_json_to_object_dict(cls, response_json: Dict) -> Dict: + def gx_cloud_response_json_to_object_dict(cls, response_json: dict) -> dict: """ This method takes full json response from GX cloud and outputs a dict appropriate for deserialization into a GX object @@ -131,7 +127,7 @@ def gx_cloud_response_json_to_object_dict(cls, response_json: Dict) -> Dict: return response_json @classmethod - def gx_cloud_response_json_to_object_collection(cls, response_json: Dict) -> List[Dict]: + def gx_cloud_response_json_to_object_collection(cls, response_json: dict) -> list[dict]: """ This method takes full json response from GX cloud and outputs a list of dicts appropriate for deserialization into a collection of GX objects @@ -187,7 +183,7 @@ def store_backend_id(self) -> str: return self._store_backend.store_backend_id @property - def key_class(self) -> Type[DataContextKey]: + def key_class(self) -> type[DataContextKey]: if self.cloud_mode: return GXCloudIdentifier return self._key_class @@ -210,12 +206,12 @@ def serialize(self, value: Any) -> Any: return value # noinspection PyMethodMayBeStatic - def key_to_tuple(self, key: DataContextKey) -> Tuple[str, ...]: + def key_to_tuple(self, key: DataContextKey) -> tuple[str, ...]: if self._use_fixed_length_key: return key.to_fixed_length_tuple() return key.to_tuple() - def tuple_to_key(self, tuple_: Tuple[str, ...]) -> DataContextKey: + def tuple_to_key(self, tuple_: tuple[str, ...]) -> DataContextKey: if tuple_ == StoreBackend.STORE_BACKEND_ID_KEY: return StoreBackend.STORE_BACKEND_ID_KEY[0] # type: ignore[return-value] if self._use_fixed_length_key: @@ -327,7 +323,7 @@ def _add_or_update(self, key: DataContextKey, value: Any, **kwargs) -> None | GX self.key_to_tuple(key), self.serialize(value), **kwargs ) - def list_keys(self) -> List[DataContextKey]: + def list_keys(self) -> list[DataContextKey]: keys_without_store_backend_id = [ key for key in self._store_backend.list_keys() diff --git a/great_expectations/data_context/store/tuple_store_backend.py b/great_expectations/data_context/store/tuple_store_backend.py index fac14102e597..69f78a473ace 100644 --- a/great_expectations/data_context/store/tuple_store_backend.py +++ b/great_expectations/data_context/store/tuple_store_backend.py @@ -9,7 +9,7 @@ import re import shutil from abc import ABCMeta -from typing import Any, List, Tuple +from typing import Any from great_expectations.compatibility import aws from great_expectations.compatibility.typing_extensions import override @@ -346,7 +346,7 @@ def _move(self, source_key, dest_key, **kwargs): # type: ignore[explicit-overri return False @override - def list_keys(self, prefix: Tuple = ()) -> List[Tuple]: + def list_keys(self, prefix: tuple = ()) -> list[tuple]: key_list = [] for root, dirs, files in os.walk( os.path.join(self.full_base_directory, *prefix) # noqa: PTH118 @@ -616,7 +616,7 @@ def _move(self, source_key, dest_key, **kwargs) -> None: s3.Object(self.bucket, source_filepath).delete() @override - def list_keys(self, prefix: Tuple = ()) -> List[Tuple]: # noqa: C901 - too complex + def list_keys(self, prefix: tuple = ()) -> list[tuple]: # noqa: C901 - too complex # Note that the prefix arg is only included to maintain consistency with the parent class signature # noqa: E501 s3r = self._create_resource() bucket = s3r.Bucket(self.bucket) @@ -897,7 +897,7 @@ def _move(self, source_key, dest_key, **kwargs) -> None: _ = bucket.rename_blob(blob, dest_filepath) @override - def list_keys(self, prefix: Tuple = ()) -> List[Tuple]: + def list_keys(self, prefix: tuple = ()) -> list[tuple]: # Note that the prefix arg is only included to maintain consistency with the parent class signature # noqa: E501 key_list = [] @@ -1106,7 +1106,7 @@ def _set(self, key, value, content_encoding="utf-8", **kwargs): # type: ignore[ return az_blob_key @override - def list_keys(self, prefix: Tuple = ()) -> List[Tuple]: + def list_keys(self, prefix: tuple = ()) -> list[tuple]: # Note that the prefix arg is only included to maintain consistency with the parent class signature # noqa: E501 key_list = [] diff --git a/great_expectations/data_context/store/validation_results_store.py b/great_expectations/data_context/store/validation_results_store.py index 9add5891de61..2b8cf01eb460 100644 --- a/great_expectations/data_context/store/validation_results_store.py +++ b/great_expectations/data_context/store/validation_results_store.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Dict, Optional, Type +from typing import TYPE_CHECKING, ClassVar, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.core.expectation_validation_result import ( @@ -96,7 +96,7 @@ class ValidationResultsStore(Store): --ge-feature-maturity-info-- """ # noqa: E501 - _key_class: ClassVar[Type] = ValidationResultIdentifier + _key_class: ClassVar[type] = ValidationResultIdentifier def __init__(self, store_backend=None, runtime_environment=None, store_name=None) -> None: self._expectationSuiteValidationResultSchema = ExpectationSuiteValidationResultSchema() @@ -146,7 +146,7 @@ def __init__(self, store_backend=None, runtime_environment=None, store_name=None @override @staticmethod - def gx_cloud_response_json_to_object_dict(response_json: Dict) -> Dict: + def gx_cloud_response_json_to_object_dict(response_json: dict) -> dict: """ This method takes full json response from GX cloud and outputs a dict appropriate for deserialization into a GX object diff --git a/great_expectations/data_context/types/base.py b/great_expectations/data_context/types/base.py index b1cc596fbf9f..8a3f17614f85 100644 --- a/great_expectations/data_context/types/base.py +++ b/great_expectations/data_context/types/base.py @@ -13,12 +13,8 @@ TYPE_CHECKING, Any, ClassVar, - Dict, - List, Mapping, Optional, - Set, - Type, TypeVar, Union, ) @@ -85,9 +81,9 @@ def object_to_yaml_str(obj): class BaseYamlConfig(SerializableDictDot): - _config_schema_class: ClassVar[Optional[Type[Schema]]] = None + _config_schema_class: ClassVar[Optional[type[Schema]]] = None - exclude_field_names: ClassVar[Set[str]] = { + exclude_field_names: ClassVar[set[str]] = { "commented_map", } @@ -97,7 +93,7 @@ def __init__(self, commented_map: Optional[CommentedMap] = None) -> None: self._commented_map = commented_map @classmethod - def _get_schema_instance(cls: Type[BYC]) -> Schema: + def _get_schema_instance(cls: type[BYC]) -> Schema: if not issubclass(cls.get_schema_class(), Schema): raise gx_exceptions.InvalidConfigError( # noqa: TRY003 "Invalid type: A configuration schema class needs to inherit from the Marshmallow Schema class." # noqa: E501 @@ -121,7 +117,7 @@ def _get_schema_instance(cls: Type[BYC]) -> Schema: return cls.get_config_class().schema_instance @classmethod - def from_commented_map(cls: Type[BYC], commented_map: Union[CommentedMap, Dict]) -> BYC: + def from_commented_map(cls: type[BYC], commented_map: Union[CommentedMap, dict]) -> BYC: try: schema_instance: Schema = cls._get_schema_instance() config: Union[dict, BYC] = schema_instance.load(commented_map) @@ -168,11 +164,11 @@ def commented_map(self) -> CommentedMap: return self._get_schema_validated_updated_commented_map() @classmethod - def get_config_class(cls: Type) -> Type: + def get_config_class(cls: type) -> type: raise NotImplementedError @classmethod - def get_schema_class(cls) -> Type[Schema]: + def get_schema_class(cls) -> type[Schema]: raise NotImplementedError @@ -305,14 +301,14 @@ def __init__( # noqa: C901, PLR0912, PLR0913 delimiter: Optional[str] = None, max_keys: Optional[int] = None, schema_name: Optional[str] = None, - batch_spec_passthrough: Optional[Dict[str, Any]] = None, - batch_identifiers: Optional[List[str]] = None, + batch_spec_passthrough: Optional[dict[str, Any]] = None, + batch_identifiers: Optional[list[str]] = None, partitioner_method: Optional[str] = None, - partitioner_kwargs: Optional[Dict[str, str]] = None, + partitioner_kwargs: Optional[dict[str, str]] = None, sorters: Optional[dict] = None, sampling_method: Optional[str] = None, - sampling_kwargs: Optional[Dict[str, str]] = None, - reader_options: Optional[Dict[str, Any]] = None, + sampling_kwargs: Optional[dict[str, str]] = None, + reader_options: Optional[dict[str, Any]] = None, **kwargs: Optional[dict], ) -> None: if name is not None: @@ -357,7 +353,7 @@ def module_name(self) -> Optional[str]: return self._module_name @override - def to_json_dict(self) -> Dict[str, JSONValues]: + def to_json_dict(self) -> dict[str, JSONValues]: """Returns a JSON-serializable dict representation of this AssetConfig. Returns: @@ -590,7 +586,7 @@ def module_name(self): return self._module_name @override - def to_json_dict(self) -> Dict[str, JSONValues]: + def to_json_dict(self) -> dict[str, JSONValues]: """Returns a JSON-serializable dict representation of this DataConnectorConfig. Returns: @@ -1075,7 +1071,7 @@ def __init__( self.organization_id = organization_id self.access_token = access_token - def to_json_dict(self) -> Dict[str, JSONValues]: + def to_json_dict(self) -> dict[str, JSONValues]: """Returns a JSON-serializable dict representation of this GXCloudConfig. Returns: @@ -1659,11 +1655,11 @@ class DatabaseStoreBackendDefaults(BaseStoreBackendDefaults): def __init__( # noqa: PLR0913 self, - default_credentials: Optional[Dict] = None, - expectations_store_credentials: Optional[Dict] = None, - validation_results_store_credentials: Optional[Dict] = None, - validation_definition_store_credentials: Optional[Dict] = None, - checkpoint_store_credentials: Optional[Dict] = None, + default_credentials: Optional[dict] = None, + expectations_store_credentials: Optional[dict] = None, + validation_results_store_credentials: Optional[dict] = None, + validation_definition_store_credentials: Optional[dict] = None, + checkpoint_store_credentials: Optional[dict] = None, expectations_store_name: str = "expectations_database_store", validation_results_store_name: str = "validation_results_database_store", checkpoint_store_name: str = "checkpoint_database_store", @@ -1758,8 +1754,8 @@ def __init__( # noqa: PLR0913 validation_results_store_name: Optional[str] = None, checkpoint_store_name: Optional[str] = None, plugins_directory: Optional[str] = None, - stores: Optional[Dict] = None, - data_docs_sites: Optional[Dict] = None, + stores: Optional[dict] = None, + data_docs_sites: Optional[dict] = None, config_variables_file_path: Optional[str] = None, analytics_enabled: Optional[bool] = None, data_context_id: Optional[uuid.UUID] = None, @@ -1837,7 +1833,7 @@ def config_version(self, config_version: float) -> None: @public_api @override - def to_json_dict(self) -> Dict[str, JSONValues]: + def to_json_dict(self) -> dict[str, JSONValues]: """Returns a JSON-serializable dict representation of this DataContextConfig. Returns: @@ -1881,7 +1877,7 @@ def __repr__(self) -> str: inplace=True, ) - keys: List[str] = sorted(list(json_dict.keys())) + keys: list[str] = sorted(list(json_dict.keys())) key: str sorted_json_dict: dict = {key: json_dict[key] for key in keys} diff --git a/great_expectations/datasource/fluent/batch_request.py b/great_expectations/datasource/fluent/batch_request.py index 19703fe52122..880cb0ec4408 100644 --- a/great_expectations/datasource/fluent/batch_request.py +++ b/great_expectations/datasource/fluent/batch_request.py @@ -5,7 +5,6 @@ AbstractSet, Any, Callable, - Dict, Generic, Mapping, Optional, @@ -43,7 +42,7 @@ # partitioner, and we want to query all months in the year 2020, the batch parameters # would look like: # options = { "year": 2020 } -BatchParameters: TypeAlias = Dict[StrictStr, Any] +BatchParameters: TypeAlias = dict[StrictStr, Any] class BatchRequest(pydantic.GenericModel, Generic[PartitionerT]): diff --git a/great_expectations/datasource/fluent/batch_request.pyi b/great_expectations/datasource/fluent/batch_request.pyi index 8ae7a217c0a3..29d667186c8c 100644 --- a/great_expectations/datasource/fluent/batch_request.pyi +++ b/great_expectations/datasource/fluent/batch_request.pyi @@ -1,4 +1,4 @@ -from typing import Any, Dict, Generic, Optional +from typing import Any, Generic, Optional from typing_extensions import TypeAlias @@ -7,7 +7,7 @@ from great_expectations.compatibility.pydantic import StrictStr from great_expectations.core.batch_definition import PartitionerT from great_expectations.datasource.fluent.data_connector.batch_filter import BatchSlice -BatchParameters: TypeAlias = Dict[StrictStr, Any] +BatchParameters: TypeAlias = dict[StrictStr, Any] class BatchRequest(pydantic.GenericModel, Generic[PartitionerT]): datasource_name: StrictStr diff --git a/great_expectations/datasource/fluent/config.py b/great_expectations/datasource/fluent/config.py index 0a0135046e11..7bacc62869f6 100644 --- a/great_expectations/datasource/fluent/config.py +++ b/great_expectations/datasource/fluent/config.py @@ -11,12 +11,7 @@ Any, Callable, ClassVar, - Dict, Final, - List, - Set, - Tuple, - Type, TypeVar, Union, overload, @@ -64,7 +59,7 @@ _FLUENT_STYLE_DESCRIPTION: Final[str] = "Fluent Datasources" -_MISSING_FLUENT_DATASOURCES_ERRORS: Final[List[PydanticErrorDict]] = [ +_MISSING_FLUENT_DATASOURCES_ERRORS: Final[list[PydanticErrorDict]] = [ { "loc": (_FLUENT_DATASOURCES_KEY,), "msg": "field required", @@ -75,7 +70,7 @@ # sentinel value to know if parameter was passed _MISSING: Final = object() -JSON_ENCODERS: dict[Type, Callable] = {} +JSON_ENCODERS: dict[type, Callable] = {} if TextClause: # type: ignore[truthy-function] JSON_ENCODERS[TextClause] = lambda v: str(v) @@ -85,17 +80,17 @@ class GxConfig(FluentBaseModel): """Represents the full fluent configuration file.""" - fluent_datasources: List[Datasource] = Field(..., description=_FLUENT_STYLE_DESCRIPTION) + fluent_datasources: list[Datasource] = Field(..., description=_FLUENT_STYLE_DESCRIPTION) - _EXCLUDE_FROM_DATASOURCE_SERIALIZATION: ClassVar[Set[str]] = { + _EXCLUDE_FROM_DATASOURCE_SERIALIZATION: ClassVar[set[str]] = { _DATASOURCE_NAME_KEY, # The "name" field is set in validation upon deserialization from configuration key; hence, it should not be serialized. # noqa: E501 } - _EXCLUDE_FROM_DATA_ASSET_SERIALIZATION: ClassVar[Set[str]] = { + _EXCLUDE_FROM_DATA_ASSET_SERIALIZATION: ClassVar[set[str]] = { _DATA_ASSET_NAME_KEY, # The "name" field is set in validation upon deserialization from configuration key; hence, it should not be serialized. # noqa: E501 } - _EXCLUDE_FROM_BATCH_DEFINITION_SERIALIZATION: ClassVar[Set[str]] = { + _EXCLUDE_FROM_BATCH_DEFINITION_SERIALIZATION: ClassVar[set[str]] = { _BATCH_DEFINITION_NAME_KEY, # The "name" field is set in validation upon deserialization from configuration key; hence, it should not be serialized. # noqa: E501 } @@ -104,24 +99,24 @@ class Config: json_encoders = JSON_ENCODERS @property - def datasources(self) -> List[Datasource]: + def datasources(self) -> list[Datasource]: """Returns available Fluent Datasources as list.""" return self.fluent_datasources - def get_datasources_as_dict(self) -> Dict[str, Datasource]: + def get_datasources_as_dict(self) -> dict[str, Datasource]: """Returns available Datasource objects as dictionary, with corresponding name as key. Returns: Dictionary of "Datasource" objects with "name" attribute serving as key. """ datasource: Datasource - datasources_as_dict: Dict[str, Datasource] = { + datasources_as_dict: dict[str, Datasource] = { datasource.name: datasource for datasource in self.fluent_datasources } return datasources_as_dict - def get_datasource_names(self) -> Set[str]: + def get_datasource_names(self) -> set[str]: """Returns the set of available Datasource names. Returns: @@ -152,14 +147,14 @@ def get_datasource(self, name: str) -> Datasource: f"'{name}' not found. Available datasources are {self.get_datasource_names()}" ) from exc - def update_datasources(self, datasources: Dict[str, Datasource]) -> None: + def update_datasources(self, datasources: dict[str, Datasource]) -> None: """ Updates internal list of datasources using supplied datasources dictionary. Args: datasources: Dictionary of datasources to use to update internal datasources. """ - datasources_as_dict: Dict[str, Datasource] = self.get_datasources_as_dict() + datasources_as_dict: dict[str, Datasource] = self.get_datasources_as_dict() datasources_as_dict.update(datasources) self.fluent_datasources = list(datasources_as_dict.values()) @@ -187,9 +182,9 @@ def pop_datasource(self, name: str, default: T = _MISSING) -> Datasource | T: # # noinspection PyNestedDecorators @validator(_FLUENT_DATASOURCES_KEY, pre=True) @classmethod - def _load_datasource_subtype(cls, v: List[dict]): # noqa: C901 - too complex + def _load_datasource_subtype(cls, v: list[dict]): # noqa: C901 - too complex logger.info(f"Loading 'datasources' ->\n{pf(v, depth=2)}") - loaded_datasources: List[Datasource] = [] + loaded_datasources: list[Datasource] = [] for config in v: ds_type_name: str = config.get("type", "") @@ -200,7 +195,7 @@ def _load_datasource_subtype(cls, v: List[dict]): # noqa: C901 - too complex raise ValueError(f"'{ds_name}' is missing a 'type' entry") # noqa: TRY003 try: - ds_type: Type[Datasource] = DataSourceManager.type_lookup[ds_type_name] + ds_type: type[Datasource] = DataSourceManager.type_lookup[ds_type_name] logger.debug(f"Instantiating '{ds_name}' as {ds_type}") except KeyError as type_lookup_err: raise ValueError( # noqa: TRY003 @@ -235,7 +230,7 @@ def _load_datasource_subtype(cls, v: List[dict]): # noqa: C901 - too complex @classmethod @override def parse_yaml( - cls: Type[GxConfig], f: Union[pathlib.Path, str], _allow_empty: bool = False + cls: type[GxConfig], f: Union[pathlib.Path, str], _allow_empty: bool = False ) -> GxConfig: """ Overriding base method to allow an empty/missing `fluent_datasources` field. @@ -332,12 +327,12 @@ def yaml( # noqa: PLR0913 return stream_or_path.getvalue() def _exclude_name_fields_from_fluent_datasources( - self, config: Dict[str, Any] - ) -> Dict[str, Any]: + self, config: dict[str, Any] + ) -> dict[str, Any]: if _FLUENT_DATASOURCES_KEY in config: fluent_datasources_config_as_dict = {} - fluent_datasources: List[dict] = config[_FLUENT_DATASOURCES_KEY] + fluent_datasources: list[dict] = config[_FLUENT_DATASOURCES_KEY] datasource_name: str datasource_config: dict @@ -348,7 +343,7 @@ def _exclude_name_fields_from_fluent_datasources( exclusions=self._EXCLUDE_FROM_DATASOURCE_SERIALIZATION, ) if "assets" in datasource_config: - data_assets: List[dict] = datasource_config["assets"] + data_assets: list[dict] = datasource_config["assets"] data_asset_config: dict data_assets_config_as_dict = { data_asset_config[_DATA_ASSET_NAME_KEY]: _exclude_fields_from_serialization( @@ -378,9 +373,9 @@ def _exclude_name_fields_from_fluent_datasources( def _exclude_fields_from_serialization( - source_dict: Dict[str, Any], exclusions: Set[str] -) -> Dict[str, Any]: - element: Tuple[str, Any] + source_dict: dict[str, Any], exclusions: set[str] +) -> dict[str, Any]: + element: tuple[str, Any] # noinspection PyTypeChecker return dict( filter( @@ -391,8 +386,8 @@ def _exclude_fields_from_serialization( def _convert_fluent_datasources_loaded_from_yaml_to_internal_object_representation( - config: Dict[str, Any], _allow_empty: bool = False -) -> Dict[str, Any]: + config: dict[str, Any], _allow_empty: bool = False +) -> dict[str, Any]: if _FLUENT_DATASOURCES_KEY in config: fluent_datasources: dict = config[_FLUENT_DATASOURCES_KEY] or {} @@ -424,8 +419,8 @@ def _convert_fluent_datasources_loaded_from_yaml_to_internal_object_representati def _convert_batch_definitions_from_yaml_to_internal_object_representation( - batch_definitions: Dict[str, Dict], -) -> List[Dict]: + batch_definitions: dict[str, dict], +) -> list[dict]: for ( batch_definition_name, batch_definition, diff --git a/great_expectations/datasource/fluent/data_asset/path/dataframe_partitioners.py b/great_expectations/datasource/fluent/data_asset/path/dataframe_partitioners.py index d1d0037eddad..42b7c7e67729 100644 --- a/great_expectations/datasource/fluent/data_asset/path/dataframe_partitioners.py +++ b/great_expectations/datasource/fluent/data_asset/path/dataframe_partitioners.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Literal, Union +from typing import TYPE_CHECKING, Literal, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.datasource.fluent.fluent_base_model import FluentBaseModel @@ -21,9 +21,9 @@ def columns(self) -> list[str]: def batch_parameters_to_batch_spec_kwarg_identifiers( self, parameters: BatchParameters - ) -> Dict[str, Dict[str, str]]: + ) -> dict[str, dict[str, str]]: """Validates all the datetime parameters for this partitioner exist in `parameters`.""" - identifiers: Dict = {} + identifiers: dict = {} for part in self.param_names: if part in parameters: identifiers[part] = parameters[part] @@ -36,7 +36,7 @@ def _get_concrete_values_from_batch(self, batch: Batch) -> tuple[int]: def param_names(self) -> list[str]: raise NotImplementedError - def partitioner_method_kwargs(self) -> Dict[str, str]: + def partitioner_method_kwargs(self) -> dict[str, str]: raise NotImplementedError @@ -47,11 +47,11 @@ class DataframePartitionerYearly(_PartitionerDatetime): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["year"] @override - def partitioner_method_kwargs(self) -> Dict[str, str]: + def partitioner_method_kwargs(self) -> dict[str, str]: return {"column_name": self.column_name} @@ -62,11 +62,11 @@ class DataframePartitionerMonthly(_PartitionerDatetime): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["year", "month"] @override - def partitioner_method_kwargs(self) -> Dict[str, str]: + def partitioner_method_kwargs(self) -> dict[str, str]: return {"column_name": self.column_name} @@ -79,11 +79,11 @@ class DataframePartitionerDaily(_PartitionerDatetime): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["year", "month", "day"] @override - def partitioner_method_kwargs(self) -> Dict[str, str]: + def partitioner_method_kwargs(self) -> dict[str, str]: return {"column_name": self.column_name} diff --git a/great_expectations/datasource/fluent/data_asset/path/pandas/generated_assets.py b/great_expectations/datasource/fluent/data_asset/path/pandas/generated_assets.py index 358c8a66f5ed..7123a0a68626 100644 --- a/great_expectations/datasource/fluent/data_asset/path/pandas/generated_assets.py +++ b/great_expectations/datasource/fluent/data_asset/path/pandas/generated_assets.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Type - from great_expectations.datasource.fluent.data_asset.path.file_asset import FileDataAsset from great_expectations.datasource.fluent.dynamic_pandas import _generate_pandas_data_asset_models @@ -32,9 +30,9 @@ use_docstring_from_method=True, skip_first_param=True, ) -CSVAsset: Type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("csv", FileDataAsset) -ExcelAsset: Type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("excel", FileDataAsset) -FWFAsset: Type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("fwf", FileDataAsset) -JSONAsset: Type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("json", FileDataAsset) -ORCAsset: Type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("orc", FileDataAsset) -ParquetAsset: Type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("parquet", FileDataAsset) +CSVAsset: type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("csv", FileDataAsset) +ExcelAsset: type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("excel", FileDataAsset) +FWFAsset: type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("fwf", FileDataAsset) +JSONAsset: type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("json", FileDataAsset) +ORCAsset: type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("orc", FileDataAsset) +ParquetAsset: type[FileDataAsset] = _FILE_PATH_ASSET_MODELS.get("parquet", FileDataAsset) diff --git a/great_expectations/datasource/fluent/data_asset/path/path_data_asset.py b/great_expectations/datasource/fluent/data_asset/path/path_data_asset.py index 2220e0d08759..7aaadd10e28f 100644 --- a/great_expectations/datasource/fluent/data_asset/path/path_data_asset.py +++ b/great_expectations/datasource/fluent/data_asset/path/path_data_asset.py @@ -8,10 +8,8 @@ TYPE_CHECKING, ClassVar, Generic, - List, Mapping, Optional, - Set, ) import great_expectations.exceptions as gx_exceptions @@ -45,7 +43,7 @@ class PathDataAsset(DataAsset, Generic[DatasourceT, PartitionerT], ABC): - _EXCLUDE_FROM_READER_OPTIONS: ClassVar[Set[str]] = { + _EXCLUDE_FROM_READER_OPTIONS: ClassVar[set[str]] = { "batch_definitions", "type", "name", @@ -116,9 +114,9 @@ def _validate_batch_request(self, batch_request: BatchRequest) -> None: ) @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: batch_definition_list = self._get_batch_definition_list(batch_request) - batch_identifiers_list: List[dict] = [ + batch_identifiers_list: list[dict] = [ batch_definition_list.batch_identifiers for batch_definition_list in batch_definition_list ] diff --git a/great_expectations/datasource/fluent/data_asset/path/spark/spark_asset.py b/great_expectations/datasource/fluent/data_asset/path/spark/spark_asset.py index 1659d618b8a1..f7abc47a9a5c 100644 --- a/great_expectations/datasource/fluent/data_asset/path/spark/spark_asset.py +++ b/great_expectations/datasource/fluent/data_asset/path/spark/spark_asset.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Sequence, Type, Union +from typing import TYPE_CHECKING, Sequence, Union from great_expectations.datasource.fluent.data_asset.path.spark.csv_asset import ( CSVAsset, @@ -35,7 +35,7 @@ # so that the schemas are generated and the assets are registered. -SPARK_PATH_ASSET_TYPES: Sequence[Type[DataAsset]] = ( +SPARK_PATH_ASSET_TYPES: Sequence[type[DataAsset]] = ( CSVAsset, DirectoryCSVAsset, ParquetAsset, diff --git a/great_expectations/datasource/fluent/data_connector/azure_blob_storage_data_connector.py b/great_expectations/datasource/fluent/data_connector/azure_blob_storage_data_connector.py index cd699ac2244f..77a9d9eea120 100644 --- a/great_expectations/datasource/fluent/data_connector/azure_blob_storage_data_connector.py +++ b/great_expectations/datasource/fluent/data_connector/azure_blob_storage_data_connector.py @@ -3,7 +3,7 @@ import logging import os import re -from typing import TYPE_CHECKING, Callable, ClassVar, List, Optional, Type +from typing import TYPE_CHECKING, Callable, ClassVar, Optional from great_expectations.compatibility import azure, pydantic from great_expectations.compatibility.typing_extensions import override @@ -48,7 +48,7 @@ class AzureBlobStorageDataConnector(FilePathDataConnector): "abs_delimiter", "abs_recursive_file_discovery", ) - asset_options_type: ClassVar[Type[_AzureOptions]] = _AzureOptions + asset_options_type: ClassVar[type[_AzureOptions]] = _AzureOptions def __init__( # noqa: PLR0913 self, @@ -172,13 +172,13 @@ def build_batch_spec(self, batch_definition: LegacyBatchDefinition) -> AzureBatc # Interface Method @override - def get_data_references(self) -> List[str]: + def get_data_references(self) -> list[str]: query_options: dict = { "container": self._container, "name_starts_with": self._sanitized_prefix, "delimiter": self._delimiter, } - path_list: List[str] = list_azure_keys( + path_list: list[str] = list_azure_keys( azure_client=self._azure_client, query_options=query_options, recursive=self._recursive_file_discovery, @@ -227,7 +227,7 @@ def list_azure_keys( azure_client: azure.BlobServiceClient, query_options: dict, recursive: bool = False, -) -> List[str]: +) -> list[str]: """ Utilizes the Azure Blob Storage connection object to retrieve blob names based on user-provided criteria. @@ -250,7 +250,7 @@ def list_azure_keys( container: str = query_options["container"] container_client: azure.ContainerClient = azure_client.get_container_client(container=container) - path_list: List[str] = [] + path_list: list[str] = [] def _walk_blob_hierarchy(name_starts_with: str) -> None: for item in container_client.walk_blobs(name_starts_with=name_starts_with): diff --git a/great_expectations/datasource/fluent/data_connector/batch_filter.py b/great_expectations/datasource/fluent/data_connector/batch_filter.py index dc83ef53163d..31396ce1ccc0 100644 --- a/great_expectations/datasource/fluent/data_connector/batch_filter.py +++ b/great_expectations/datasource/fluent/data_connector/batch_filter.py @@ -2,7 +2,7 @@ import itertools import logging -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Sequence, Union +from typing import TYPE_CHECKING, Callable, Optional, Sequence, Union import great_expectations.exceptions as gx_exceptions from great_expectations.compatibility.pydantic import StrictInt, StrictStr @@ -76,7 +76,7 @@ def validate(cls, v): def build_batch_filter( # noqa: C901 - too complex data_connector_query_dict: Optional[ - Dict[ + dict[ str, Optional[ Union[ @@ -85,7 +85,7 @@ def build_batch_filter( # noqa: C901 - too complex tuple, Union[slice, SliceValidator], str, - Union[Dict, IDDict], + Union[dict, IDDict], Callable, ] ], @@ -291,8 +291,8 @@ def __repr__(self) -> str: return str(doc_fields_dict) def select_from_data_connector_query( - self, batch_definition_list: Optional[List[LegacyBatchDefinition]] = None - ) -> List[LegacyBatchDefinition]: + self, batch_definition_list: Optional[list[LegacyBatchDefinition]] = None + ) -> list[LegacyBatchDefinition]: if batch_definition_list is None: return [] filter_function: Callable @@ -300,7 +300,7 @@ def select_from_data_connector_query( filter_function = self.custom_filter_function else: filter_function = self.best_effort_batch_definition_matcher() - selected_batch_definitions: List[LegacyBatchDefinition] + selected_batch_definitions: list[LegacyBatchDefinition] selected_batch_definitions = list( filter( lambda batch_definition: filter_function( diff --git a/great_expectations/datasource/fluent/data_connector/data_connector.py b/great_expectations/datasource/fluent/data_connector/data_connector.py index f49922046a8e..5e12c9b182fa 100644 --- a/great_expectations/datasource/fluent/data_connector/data_connector.py +++ b/great_expectations/datasource/fluent/data_connector/data_connector.py @@ -2,7 +2,7 @@ import logging from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, ClassVar, List, Type +from typing import TYPE_CHECKING, Any, ClassVar from great_expectations.core.id_dict import BatchSpec @@ -42,7 +42,7 @@ class DataConnector(ABC): # needed to select the asset level kwargs needed to build the DataConnector asset_level_option_keys: ClassVar[tuple[str, ...]] = () - asset_options_type: ClassVar[Type] = dict + asset_options_type: ClassVar[type] = dict def __init__( self, @@ -61,7 +61,7 @@ def datasource_name(self) -> str: return self._datasource_name @abstractmethod - def get_batch_definition_list(self, batch_request: BatchRequest) -> List[LegacyBatchDefinition]: + def get_batch_definition_list(self, batch_request: BatchRequest) -> list[LegacyBatchDefinition]: """ This interface method, implemented by subclasses, examines "BatchRequest" and converts it to one or more "BatchDefinition" objects, each of which can be later converted to ExecutionEngine-specific "BatchSpec" object @@ -99,7 +99,7 @@ def test_connection(self) -> bool: return self.get_unmatched_data_reference_count() < self.get_data_reference_count() @abstractmethod - def get_data_references(self) -> List[Any]: + def get_data_references(self) -> list[Any]: """ This interface method lists objects in the underlying data store used to create a list of data_references (type depends on cloud storage environment, SQL DBMS, etc.). """ # noqa: E501 @@ -116,7 +116,7 @@ def get_data_reference_count(self) -> int: pass @abstractmethod - def get_matched_data_references(self) -> List[Any]: + def get_matched_data_references(self) -> list[Any]: """ This interface method returns (e.g., cached) data references that were successfully matched based on "BatchRequest" options. @@ -136,7 +136,7 @@ def get_matched_data_reference_count(self) -> int: pass @abstractmethod - def get_unmatched_data_references(self) -> List[Any]: + def get_unmatched_data_references(self) -> list[Any]: """ This interface method returns (e.g., cached) data references that could not be matched based on "BatchRequest" options. diff --git a/great_expectations/datasource/fluent/data_connector/file_path_data_connector.py b/great_expectations/datasource/fluent/data_connector/file_path_data_connector.py index b1228db4b701..93f87daf5a00 100644 --- a/great_expectations/datasource/fluent/data_connector/file_path_data_connector.py +++ b/great_expectations/datasource/fluent/data_connector/file_path_data_connector.py @@ -7,7 +7,7 @@ import sre_parse from abc import abstractmethod from collections import defaultdict -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Callable, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core import IDDict @@ -27,8 +27,6 @@ ) if TYPE_CHECKING: - from typing import DefaultDict - from great_expectations.alias_types import PathStr from great_expectations.core.partitioners import FileNamePartitioner from great_expectations.datasource.fluent import BatchRequest @@ -73,13 +71,13 @@ def __init__( self._whole_directory_path_override = whole_directory_path_override # This is a dictionary which maps data_references onto batch_requests. - self._data_references_cache: DefaultDict[ - re.Pattern, Dict[str, List[LegacyBatchDefinition] | None] + self._data_references_cache: defaultdict[ + re.Pattern, dict[str, list[LegacyBatchDefinition] | None] ] = defaultdict(dict) # Interface Method @override - def get_batch_definition_list(self, batch_request: BatchRequest) -> List[LegacyBatchDefinition]: + def get_batch_definition_list(self, batch_request: BatchRequest) -> list[LegacyBatchDefinition]: """ Retrieve batch_definitions and that match batch_request. @@ -93,7 +91,7 @@ def get_batch_definition_list(self, batch_request: BatchRequest) -> List[LegacyB A list of BatchDefinition objects that match BatchRequest """ # noqa: E501 - legacy_batch_definition_list: List[LegacyBatchDefinition] = ( + legacy_batch_definition_list: list[LegacyBatchDefinition] = ( self._get_unfiltered_batch_definition_list(batch_request=batch_request) ) @@ -145,7 +143,7 @@ def get_data_reference_count(self) -> int: # Interface Method @override - def get_matched_data_references(self, regex: re.Pattern | None = None) -> List[str]: + def get_matched_data_references(self, regex: re.Pattern | None = None) -> list[str]: """ Returns the list of data_references matched by configuration by looping through items in _data_references_cache and returning data_references that have an associated data_asset. @@ -170,7 +168,7 @@ def get_matched_data_reference_count(self) -> int: # Interface Method @override - def get_unmatched_data_references(self) -> List[str]: + def get_unmatched_data_references(self) -> list[str]: """ Returns the list of data_references unmatched by configuration by looping through items in _data_references_cache and returning data_references that do not have an associated data_asset. @@ -245,7 +243,7 @@ def _get_directory_batch_definition_list( ) return [batch_definition] - def _get_data_references(self, matched: bool, regex: re.Pattern | None = None) -> List[str]: + def _get_data_references(self, matched: bool, regex: re.Pattern | None = None) -> list[str]: """ Returns the list of data_references unmatched by configuration by looping through items in _data_references_cache and returning data_references that do not have an associated data_asset. @@ -257,15 +255,15 @@ def _get_data_references(self, matched: bool, regex: re.Pattern | None = None) - regex = self._preprocess_batching_regex(MATCH_ALL_PATTERN) def _matching_criterion( - batch_definition_list: Union[List[LegacyBatchDefinition], None], + batch_definition_list: Union[list[LegacyBatchDefinition], None], ) -> bool: return ( (batch_definition_list is not None) if matched else (batch_definition_list is None) ) - data_reference_mapped_element: Tuple[str, Union[List[LegacyBatchDefinition], None]] + data_reference_mapped_element: tuple[str, Union[list[LegacyBatchDefinition], None]] data_references = self._get_data_references_cache(batching_regex=regex) - unmatched_data_references: List[str] = list( + unmatched_data_references: list[str] = list( dict( filter( lambda data_reference_mapped_element: _matching_criterion( @@ -311,7 +309,7 @@ def _get_batch_spec_params_file(self, batch_definition: LegacyBatchDefinition) - regex_pattern=batching_regex, unnamed_regex_group_prefix=self._unnamed_regex_group_prefix, ) - group_names: List[str] = regex_parser.group_names() + group_names: list[str] = regex_parser.group_names() path: str = map_batch_definition_to_data_reference_string_using_regex( batch_definition=batch_definition, regex_pattern=batching_regex, @@ -340,7 +338,7 @@ def _preprocess_batching_regex(self, regex: re.Pattern) -> re.Pattern: regex_pattern=regex, unnamed_regex_group_prefix=self._unnamed_regex_group_prefix, ) - group_names: List[str] = regex_parser.group_names() + group_names: list[str] = regex_parser.group_names() if FilePathDataConnector.FILE_PATH_BATCH_SPEC_KEY not in group_names: pattern: str = regex.pattern pattern = f"(?P<{FilePathDataConnector.FILE_PATH_BATCH_SPEC_KEY}>{pattern})" @@ -350,7 +348,7 @@ def _preprocess_batching_regex(self, regex: re.Pattern) -> re.Pattern: def _get_data_references_cache( self, batching_regex: re.Pattern - ) -> Dict[str, List[LegacyBatchDefinition] | None]: + ) -> dict[str, list[LegacyBatchDefinition] | None]: """Access a map where keys are data references and values are LegacyBatchDefinitions.""" batch_definitions = self._data_references_cache[batching_regex] @@ -371,7 +369,7 @@ def _get_data_references_cache( return batch_definitions - def _get_batch_definitions(self, batching_regex: re.Pattern) -> List[LegacyBatchDefinition]: + def _get_batch_definitions(self, batching_regex: re.Pattern) -> list[LegacyBatchDefinition]: batch_definition_map = self._get_data_references_cache(batching_regex=batching_regex) batch_definitions = [ batch_definitions[0] @@ -413,19 +411,19 @@ def _build_batch_identifiers( num_all_matched_group_values: int = regex_parser.get_num_all_matched_group_values() # Check for `(?P)` named group syntax - defined_group_name_to_group_index_mapping: Dict[str, int] = ( + defined_group_name_to_group_index_mapping: dict[str, int] = ( regex_parser.get_named_group_name_to_group_index_mapping() ) - defined_group_name_indexes: Set[int] = set( + defined_group_name_indexes: set[int] = set( defined_group_name_to_group_index_mapping.values() ) - defined_group_name_to_group_value_mapping: Dict[str, str] = matches.groupdict() + defined_group_name_to_group_value_mapping: dict[str, str] = matches.groupdict() - all_matched_group_values: List[str] = list(matches.groups()) + all_matched_group_values: list[str] = list(matches.groups()) assert len(all_matched_group_values) == num_all_matched_group_values - group_name_to_group_value_mapping: Dict[str, str] = copy.deepcopy( + group_name_to_group_value_mapping: dict[str, str] = copy.deepcopy( defined_group_name_to_group_value_mapping ) @@ -450,7 +448,7 @@ def _get_full_file_path(self, path: str) -> str: def map_batch_definition_to_data_reference_string_using_regex( batch_definition: LegacyBatchDefinition, regex_pattern: re.Pattern, - group_names: List[str], + group_names: list[str], ) -> str: if not isinstance(batch_definition, LegacyBatchDefinition): raise TypeError("batch_definition is not of an instance of type BatchDefinition") # noqa: TRY003 @@ -469,7 +467,7 @@ def map_batch_definition_to_data_reference_string_using_regex( def convert_batch_identifiers_to_data_reference_string_using_regex( batch_identifiers: IDDict, regex_pattern: re.Pattern, - group_names: List[str], + group_names: list[str], data_asset_name: Optional[str] = None, ) -> str: if not isinstance(batch_identifiers, IDDict): @@ -490,7 +488,7 @@ def convert_batch_identifiers_to_data_reference_string_using_regex( def _invert_regex_to_data_reference_template( # noqa: C901 - too complex regex_pattern: re.Pattern | str, - group_names: List[str], + group_names: list[str], ) -> str: r"""Create a string template based on a regex and corresponding list of group names. diff --git a/great_expectations/datasource/fluent/data_connector/filesystem_data_connector.py b/great_expectations/datasource/fluent/data_connector/filesystem_data_connector.py index 256fb2e8d564..7270915379e9 100644 --- a/great_expectations/datasource/fluent/data_connector/filesystem_data_connector.py +++ b/great_expectations/datasource/fluent/data_connector/filesystem_data_connector.py @@ -3,7 +3,7 @@ import logging import os import pathlib -from typing import TYPE_CHECKING, Callable, ClassVar, List, Optional, Type, Union +from typing import TYPE_CHECKING, Callable, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -34,7 +34,7 @@ class FilesystemDataConnector(FilePathDataConnector): """ # noqa: E501 asset_level_option_keys: ClassVar[tuple[str, ...]] = ("glob_directive",) - asset_options_type: ClassVar[Type[FilesystemOptions]] = FilesystemOptions + asset_options_type: ClassVar[type[FilesystemOptions]] = FilesystemOptions def __init__( # noqa: PLR0913 self, @@ -134,10 +134,10 @@ def build_test_connection_error_message( # Interface Method @override - def get_data_references(self) -> List[str]: + def get_data_references(self) -> list[str]: base_directory: pathlib.Path = self.base_directory glob_directive: str = self._glob_directive - path_list: List[str] = get_filesystem_one_level_directory_glob_path_list( + path_list: list[str] = get_filesystem_one_level_directory_glob_path_list( base_directory_path=base_directory, glob_directive=glob_directive ) return sorted(path_list) @@ -165,7 +165,7 @@ def normalize_directory_path( def get_filesystem_one_level_directory_glob_path_list( base_directory_path: Union[PathStr], glob_directive: str -) -> List[str]: +) -> list[str]: """ List file names, relative to base_directory_path one level deep, with expansion specified by glob_directive. :param base_directory_path -- base directory path, relative to which file paths will be collected @@ -177,7 +177,7 @@ def get_filesystem_one_level_directory_glob_path_list( globbed_paths = base_directory_path.glob(glob_directive) - path_list: List[str] = [ + path_list: list[str] = [ os.path.relpath(str(posix_path), base_directory_path) for posix_path in globbed_paths ] diff --git a/great_expectations/datasource/fluent/data_connector/google_cloud_storage_data_connector.py b/great_expectations/datasource/fluent/data_connector/google_cloud_storage_data_connector.py index 58937e885bf9..b0223317b307 100644 --- a/great_expectations/datasource/fluent/data_connector/google_cloud_storage_data_connector.py +++ b/great_expectations/datasource/fluent/data_connector/google_cloud_storage_data_connector.py @@ -3,7 +3,7 @@ import logging import re import warnings -from typing import TYPE_CHECKING, Callable, ClassVar, List, Optional, Type +from typing import TYPE_CHECKING, Callable, ClassVar, Optional from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -51,7 +51,7 @@ class GoogleCloudStorageDataConnector(FilePathDataConnector): "gcs_max_results", "gcs_recursive_file_discovery", ) - asset_options_type: ClassVar[Type[_GCSOptions]] = _GCSOptions + asset_options_type: ClassVar[type[_GCSOptions]] = _GCSOptions def __init__( # noqa: PLR0913 self, @@ -172,14 +172,14 @@ def build_batch_spec(self, batch_definition: LegacyBatchDefinition) -> GCSBatchS # Interface Method @override - def get_data_references(self) -> List[str]: + def get_data_references(self) -> list[str]: query_options: dict = { "bucket_or_name": self._bucket_or_name, "prefix": self._sanitized_prefix, "delimiter": self._delimiter, "max_results": self._max_results, } - path_list: List[str] = list_gcs_keys( + path_list: list[str] = list_gcs_keys( gcs_client=self._gcs_client, query_options=query_options, recursive=self._recursive_file_discovery, @@ -213,7 +213,7 @@ def list_gcs_keys( gcs_client, query_options: dict, recursive: bool = False, -) -> List[str]: +) -> list[str]: """ Utilizes the GCS connection object to retrieve blob names based on user-provided criteria. @@ -262,7 +262,7 @@ def list_gcs_keys( ) query_options["delimiter"] = None - keys: List[str] = [] + keys: list[str] = [] for blob in gcs_client.list_blobs(**query_options): name: str = blob.name if name.endswith("/"): # GCS includes directories in blob output diff --git a/great_expectations/datasource/fluent/data_connector/regex_parser.py b/great_expectations/datasource/fluent/data_connector/regex_parser.py index 2c193f211d5d..5f092ed2a3fb 100644 --- a/great_expectations/datasource/fluent/data_connector/regex_parser.py +++ b/great_expectations/datasource/fluent/data_connector/regex_parser.py @@ -2,7 +2,7 @@ import logging import re -from typing import Dict, List, Match, Optional, Tuple +from typing import Match, Optional logger = logging.getLogger(__name__) @@ -16,7 +16,7 @@ def __init__( self._num_all_matched_group_values: int = regex_pattern.groups # Check for `(?P)` named group syntax - self._group_name_to_index_dict: Dict[str, int] = dict(regex_pattern.groupindex) + self._group_name_to_index_dict: dict[str, int] = dict(regex_pattern.groupindex) self._regex_pattern: re.Pattern = regex_pattern self._unnamed_regex_group_prefix: str = unnamed_regex_group_prefix @@ -24,7 +24,7 @@ def __init__( def get_num_all_matched_group_values(self) -> int: return self._num_all_matched_group_values - def get_named_group_name_to_group_index_mapping(self) -> Dict[str, int]: + def get_named_group_name_to_group_index_mapping(self) -> dict[str, int]: return self._group_name_to_index_dict def get_matches(self, target: str) -> Optional[Match[str]]: @@ -32,8 +32,8 @@ def get_matches(self, target: str) -> Optional[Match[str]]: def get_all_group_names_to_group_indexes_bidirectional_mappings( self, - ) -> Tuple[Dict[str, int], Dict[int, str]]: - named_group_index_to_group_name_mapping: Dict[int, str] = dict( + ) -> tuple[dict[str, int], dict[int, str]]: + named_group_index_to_group_name_mapping: dict[int, str] = dict( zip( self._group_name_to_index_dict.values(), self._group_name_to_index_dict.keys(), @@ -41,7 +41,7 @@ def get_all_group_names_to_group_indexes_bidirectional_mappings( ) idx: int - common_group_indexes: List[int] = list( + common_group_indexes: list[int] = list( filter( lambda idx: idx not in self._group_name_to_index_dict.values(), range(1, self._num_all_matched_group_values + 1), @@ -49,17 +49,17 @@ def get_all_group_names_to_group_indexes_bidirectional_mappings( ) group_idx: int - common_group_index_to_group_name_mapping: Dict[int, str] = { + common_group_index_to_group_name_mapping: dict[int, str] = { group_idx: f"{self._unnamed_regex_group_prefix}{group_idx}" for group_idx in common_group_indexes } - all_group_index_to_group_name_mapping: Dict[int, str] = { + all_group_index_to_group_name_mapping: dict[int, str] = { **named_group_index_to_group_name_mapping, **common_group_index_to_group_name_mapping, } - element: Tuple[int, str] + element: tuple[int, str] # noinspection PyTypeChecker all_group_index_to_group_name_mapping = dict( sorted( @@ -69,7 +69,7 @@ def get_all_group_names_to_group_indexes_bidirectional_mappings( ) ) - all_group_name_to_group_index_mapping: Dict[str, int] = dict( + all_group_name_to_group_index_mapping: dict[str, int] = dict( zip( all_group_index_to_group_name_mapping.values(), all_group_index_to_group_name_mapping.keys(), @@ -81,27 +81,27 @@ def get_all_group_names_to_group_indexes_bidirectional_mappings( all_group_index_to_group_name_mapping, ) - def get_all_group_name_to_group_index_mapping(self) -> Dict[str, int]: - all_group_names_to_group_indexes_bidirectional_mappings: Tuple[ - Dict[str, int], Dict[int, str] + def get_all_group_name_to_group_index_mapping(self) -> dict[str, int]: + all_group_names_to_group_indexes_bidirectional_mappings: tuple[ + dict[str, int], dict[int, str] ] = self.get_all_group_names_to_group_indexes_bidirectional_mappings() - all_group_name_to_group_index_mapping: Dict[str, int] = ( + all_group_name_to_group_index_mapping: dict[str, int] = ( all_group_names_to_group_indexes_bidirectional_mappings[0] ) return all_group_name_to_group_index_mapping - def get_all_group_index_to_group_name_mapping(self) -> Dict[int, str]: - all_group_names_to_group_indexes_bidirectional_mappings: Tuple[ - Dict[str, int], Dict[int, str] + def get_all_group_index_to_group_name_mapping(self) -> dict[int, str]: + all_group_names_to_group_indexes_bidirectional_mappings: tuple[ + dict[str, int], dict[int, str] ] = self.get_all_group_names_to_group_indexes_bidirectional_mappings() - all_group_index_to_group_name_mapping: Dict[int, str] = ( + all_group_index_to_group_name_mapping: dict[int, str] = ( all_group_names_to_group_indexes_bidirectional_mappings[1] ) return all_group_index_to_group_name_mapping - def group_names(self) -> List[str]: - all_group_name_to_group_index_mapping: Dict[str, int] = ( + def group_names(self) -> list[str]: + all_group_name_to_group_index_mapping: dict[str, int] = ( self.get_all_group_name_to_group_index_mapping() ) - all_group_names: List[str] = list(all_group_name_to_group_index_mapping.keys()) + all_group_names: list[str] = list(all_group_name_to_group_index_mapping.keys()) return all_group_names diff --git a/great_expectations/datasource/fluent/data_connector/s3_data_connector.py b/great_expectations/datasource/fluent/data_connector/s3_data_connector.py index 1df4f4ea77b0..4e7383b3339b 100644 --- a/great_expectations/datasource/fluent/data_connector/s3_data_connector.py +++ b/great_expectations/datasource/fluent/data_connector/s3_data_connector.py @@ -3,7 +3,7 @@ import copy import logging import re -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Generator, List, Optional, Type +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generator, Optional from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -53,7 +53,7 @@ class S3DataConnector(FilePathDataConnector): "s3_max_keys", "s3_recursive_file_discovery", ) - asset_options_type: ClassVar[Type[_S3Options]] = _S3Options + asset_options_type: ClassVar[type[_S3Options]] = _S3Options def __init__( # noqa: PLR0913 self, @@ -174,14 +174,14 @@ def build_batch_spec(self, batch_definition: LegacyBatchDefinition) -> S3BatchSp # Interface Method @override - def get_data_references(self) -> List[str]: + def get_data_references(self) -> list[str]: query_options: dict = { "Bucket": self._bucket, "Prefix": self._sanitized_prefix, "Delimiter": self._delimiter, "MaxKeys": self._max_keys, } - path_list: List[str] = list( + path_list: list[str] = list( list_s3_keys( s3=self._s3_client, query_options=query_options, @@ -244,11 +244,11 @@ def list_s3_keys( # noqa: C901 - too complex raise ValueError("S3 query may not have been configured correctly.") # noqa: TRY003 if "Contents" in s3_objects_info: - keys: List[str] = [item["Key"] for item in s3_objects_info["Contents"] if item["Size"] > 0] + keys: list[str] = [item["Key"] for item in s3_objects_info["Contents"] if item["Size"] > 0] yield from keys if recursive and "CommonPrefixes" in s3_objects_info: - common_prefixes: List[Dict[str, Any]] = s3_objects_info["CommonPrefixes"] + common_prefixes: list[dict[str, Any]] = s3_objects_info["CommonPrefixes"] for prefix_info in common_prefixes: query_options_tmp: dict = copy.deepcopy(query_options) query_options_tmp.update({"Prefix": prefix_info["Prefix"]}) diff --git a/great_expectations/datasource/fluent/databricks_sql_datasource.py b/great_expectations/datasource/fluent/databricks_sql_datasource.py index 88e300aa2eba..6d8c7633aaab 100644 --- a/great_expectations/datasource/fluent/databricks_sql_datasource.py +++ b/great_expectations/datasource/fluent/databricks_sql_datasource.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, List, Literal, Type, Union, overload +import builtins +from typing import TYPE_CHECKING, ClassVar, Literal, Union, overload from urllib import parse from great_expectations._docs_decorators import public_api @@ -187,14 +188,14 @@ class DatabricksSQLDatasource(SQLDatasource): """ # noqa: E501 # class var definitions - asset_types: ClassVar[List[Type[DataAsset]]] = [DatabricksTableAsset, SqlQueryAsset] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [DatabricksTableAsset, SqlQueryAsset] type: Literal["databricks_sql"] = "databricks_sql" # type: ignore[assignment] connection_string: Union[ConfigStr, DatabricksDsn] # These are instance var because ClassVars can't contain Type variables. See # https://peps.python.org/pep-0526/#class-and-instance-variable-annotations - _TableAsset: Type[SqlTableAsset] = pydantic.PrivateAttr(DatabricksTableAsset) + _TableAsset: builtins.type[SqlTableAsset] = pydantic.PrivateAttr(DatabricksTableAsset) @override def test_connection(self, test_assets: bool = True) -> None: diff --git a/great_expectations/datasource/fluent/dynamic_pandas.py b/great_expectations/datasource/fluent/dynamic_pandas.py index cbd3d10ec985..4a28f8c821c5 100644 --- a/great_expectations/datasource/fluent/dynamic_pandas.py +++ b/great_expectations/datasource/fluent/dynamic_pandas.py @@ -11,20 +11,15 @@ TYPE_CHECKING, Any, Callable, - Dict, Final, Hashable, Iterable, Iterator, - List, Literal, NamedTuple, Optional, Pattern, # must use typing.Pattern for pydantic < v1.10 Sequence, - Set, - Tuple, - Type, TypeVar, Union, ) @@ -50,12 +45,12 @@ except ImportError: # Types may not exist on earlier version of pandas (current min ver is v.1.1.0) # https://github.com/pandas-dev/pandas/blob/v1.1.0/pandas/_typing.py - CompressionDict = Dict[str, Any] + CompressionDict = dict[str, Any] CompressionOptions = Optional[ # type: ignore[misc] Union[Literal["infer", "gzip", "bz2", "zip", "xz", "zstd", "tar"], CompressionDict] ] CSVEngine = Literal["c", "python", "pyarrow", "python-fwf"] # type: ignore[misc] - StorageOptions = Optional[Dict[str, Any]] # type: ignore[misc] + StorageOptions = Optional[dict[str, Any]] # type: ignore[misc] try: from pandas._libs.lib import _NoDefault @@ -82,7 +77,7 @@ class _NoDefault(enum.Enum): # type: ignore[no-redef] # sentinel values UNSUPPORTED_TYPE: Final = object() -CAN_HANDLE: Final[Set[str]] = { +CAN_HANDLE: Final[set[str]] = { # builtins "str", "int", @@ -126,7 +121,7 @@ class _NoDefault(enum.Enum): # type: ignore[no-redef] "DtypeBackend", } -TYPE_SUBSTITUTIONS: Final[Dict[str, str]] = { +TYPE_SUBSTITUTIONS: Final[dict[str, str]] = { # Hashable causes `TypeError:issubclass() arg 1 must be a class` on some versions of pydantic "Hashable": "str", "Sequence[Hashable]": "Sequence[str]", @@ -138,9 +133,9 @@ class _NoDefault(enum.Enum): # type: ignore[no-redef] "list[IntStrT]": "List[Union[int, str]]", } -NEED_SPECIAL_HANDLING: Dict[str, Set[str]] = defaultdict(set) -FIELD_SKIPPED_UNSUPPORTED_TYPE: Set[str] = set() -FIELD_SKIPPED_NO_ANNOTATION: Set[str] = set() +NEED_SPECIAL_HANDLING: dict[str, set[str]] = defaultdict(set) +FIELD_SKIPPED_UNSUPPORTED_TYPE: set[str] = set() +FIELD_SKIPPED_NO_ANNOTATION: set[str] = set() class DynamicAssetError(Exception): @@ -155,7 +150,7 @@ class _SignatureTuple(NamedTuple): class _FieldSpec(NamedTuple): # mypy doesn't consider Optional[SOMETHING] or Union[SOMETHING] a type. So what is it? - type: Type | str + type: type | str default_value: object # ... for required value @@ -166,7 +161,7 @@ def _replace_builtins(input_: str | type) -> str | type: return input_.replace("list", "List").replace("dict", "Dict") -FIELD_SUBSTITUTIONS: Final[Dict[str, Dict[str, _FieldSpec]]] = { +FIELD_SUBSTITUTIONS: Final[dict[str, dict[str, _FieldSpec]]] = { # SQLTable "schema": { "schema_name": _FieldSpec( @@ -213,7 +208,7 @@ def _replace_builtins(input_: str | type) -> str | type: }, } -_METHOD_TO_CLASS_NAME_MAPPINGS: Final[Dict[str, str]] = { +_METHOD_TO_CLASS_NAME_MAPPINGS: Final[dict[str, str]] = { "csv": "CSVAsset", "fwf": "FWFAsset", "gbq": "GBQAsset", @@ -228,7 +223,7 @@ def _replace_builtins(input_: str | type) -> str | type: "xml": "XMLAsset", } -_TYPE_REF_LOCALS: Final[Dict[str, Type | Any]] = { +_TYPE_REF_LOCALS: Final[dict[str, type | Any]] = { "Literal": Literal, "Sequence": Sequence, "Hashable": Hashable, @@ -248,7 +243,7 @@ def _replace_builtins(input_: str | type) -> str | type: def _extract_io_methods( blacklist: Optional[Sequence[str]] = None, -) -> List[Tuple[str, DataFrameFactoryFn]]: +) -> list[tuple[str, DataFrameFactoryFn]]: # suppress pandas future warnings that may be emitted by collecting # pandas io methods # Once the context manager exits, the warning filter is removed. @@ -265,8 +260,8 @@ def _extract_io_methods( def _extract_io_signatures( - io_methods: List[Tuple[str, DataFrameFactoryFn]], -) -> List[_SignatureTuple]: + io_methods: list[tuple[str, DataFrameFactoryFn]], +) -> list[_SignatureTuple]: signatures = [] for name, method in io_methods: sig = inspect.signature(method) @@ -289,7 +284,7 @@ def _get_default_value( return default -def _get_annotation_type(param: inspect.Parameter) -> Union[Type, str, object]: +def _get_annotation_type(param: inspect.Parameter) -> Union[type, str, object]: """ https://docs.python.org/3/howto/annotations.html#manually-un-stringizing-stringized-annotations """ @@ -333,12 +328,12 @@ def _get_annotation_type(param: inspect.Parameter) -> Union[Type, str, object]: def _to_pydantic_fields( sig_tuple: _SignatureTuple, skip_first_param: bool -) -> Dict[str, _FieldSpec]: +) -> dict[str, _FieldSpec]: """ Extract the parameter details in a structure that can be easily unpacked to `pydantic.create_model()` as field arguments """ - fields_dict: Dict[str, _FieldSpec] = {} + fields_dict: dict[str, _FieldSpec] = {} all_parameters: Iterator[tuple[str, inspect.Parameter]] = iter( sig_tuple.signature.parameters.items() ) @@ -370,14 +365,14 @@ def _to_pydantic_fields( return fields_dict -M = TypeVar("M", bound=Type[DataAsset]) +M = TypeVar("M", bound=type[DataAsset]) def _create_pandas_asset_model( # noqa: PLR0913 model_name: str, model_base: M, - type_field: Tuple[Union[Type, str], str], - fields_dict: Dict[str, _FieldSpec], + type_field: tuple[Union[type, str], str], + fields_dict: dict[str, _FieldSpec], extra: pydantic.Extra, model_docstring: str = "", ) -> M: @@ -410,11 +405,11 @@ def _generate_pandas_data_asset_models( blacklist: Optional[Sequence[str]] = None, use_docstring_from_method: bool = False, skip_first_param: bool = False, -) -> Dict[str, M]: +) -> dict[str, M]: io_methods = _extract_io_methods(blacklist) io_method_sigs = _extract_io_signatures(io_methods) - data_asset_models: Dict[str, M] = {} + data_asset_models: dict[str, M] = {} for signature_tuple in io_method_sigs: # skip the first parameter as this corresponds to the path/buffer/io field # paths to specific files are provided by the batch building logic diff --git a/great_expectations/datasource/fluent/fabric.py b/great_expectations/datasource/fluent/fabric.py index 4485087a6c23..4409cd83f6fa 100644 --- a/great_expectations/datasource/fluent/fabric.py +++ b/great_expectations/datasource/fluent/fabric.py @@ -4,6 +4,7 @@ from __future__ import annotations +import builtins import logging import os import uuid @@ -11,13 +12,9 @@ from typing import ( TYPE_CHECKING, ClassVar, - Dict, Final, - List, Literal, Optional, - Set, - Type, Union, ) @@ -54,7 +51,7 @@ LOGGER = logging.getLogger(__name__) -SortersDefinition: TypeAlias = List[Union[Sorter, str, dict]] +SortersDefinition: TypeAlias = list[Union[Sorter, str, dict]] _REQUIRED_FABRIC_SERVICE: Final[str] = "Microsoft.ProjectArcadia" Mode: TypeAlias = Literal["xmla", "rest", "onelake"] @@ -64,7 +61,7 @@ class _PowerBIAsset(DataAsset): """Microsoft PowerBI Asset base class.""" _reader_method: ClassVar[FabricReaderMethods] - _EXCLUDE_FROM_READER_OPTIONS: ClassVar[Set[str]] = { + _EXCLUDE_FROM_READER_OPTIONS: ClassVar[set[str]] = { "batch_definitions", "batch_metadata", "name", @@ -82,7 +79,7 @@ def test_connection(self) -> None: LOGGER.debug(f"Testing connection to {self.__class__.__name__} has not been implemented") @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: return [IDDict(batch_request.options)] @override @@ -219,9 +216,9 @@ class PowerBIMeasure(_PowerBIAsset): _reader_method: ClassVar[FabricReaderMethods] = "evaluate_measure" type: Literal["powerbi_measure"] = "powerbi_measure" - measure: Union[str, List[str]] - groupby_columns: Optional[List[str]] = None - filters: Optional[Dict[str, List[str]]] = None + measure: Union[str, list[str]] + groupby_columns: Optional[list[str]] = None + filters: Optional[dict[str, list[str]]] = None fully_qualified_columns: Optional[bool] = None num_rows: Optional[int] = None use_xmla: bool = False @@ -259,7 +256,7 @@ class FabricPowerBIDatasource(Datasource): """ # class var definitions - asset_types: ClassVar[List[Type[DataAsset]]] = [ + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [ PowerBIDax, PowerBIMeasure, PowerBITable, @@ -271,7 +268,7 @@ class FabricPowerBIDatasource(Datasource): # right side of the operator determines the type name # left side enforces the names on instance creation type: Literal["fabric_powerbi"] = "fabric_powerbi" - assets: List[AssetTypes] = [] + assets: list[AssetTypes] = [] # fabric datasource specific fields workspace: Optional[Union[uuid.UUID, str]] = None @@ -279,7 +276,7 @@ class FabricPowerBIDatasource(Datasource): @property @override - def execution_engine_type(self) -> Type[PandasExecutionEngine]: + def execution_engine_type(self) -> type[PandasExecutionEngine]: """Return the PandasExecutionEngine unless the override is set""" from great_expectations.execution_engine.pandas_execution_engine import ( PandasExecutionEngine, @@ -340,10 +337,10 @@ def add_powerbi_dax_asset( def add_powerbi_measure_asset( # noqa: PLR0913 self, name: str, - measure: Union[str, List[str]], + measure: Union[str, list[str]], batch_metadata: Optional[BatchMetadata] = None, - groupby_columns: Optional[List[str]] = None, - filters: Optional[Dict[str, List[str]]] = None, + groupby_columns: Optional[list[str]] = None, + filters: Optional[dict[str, list[str]]] = None, fully_qualified_columns: Optional[bool] = None, num_rows: Optional[int] = None, use_xmla: bool = False, diff --git a/great_expectations/datasource/fluent/fluent_base_model.py b/great_expectations/datasource/fluent/fluent_base_model.py index 82b6f863d096..a27741f62e18 100644 --- a/great_expectations/datasource/fluent/fluent_base_model.py +++ b/great_expectations/datasource/fluent/fluent_base_model.py @@ -11,9 +11,7 @@ AbstractSet, Any, Callable, - Dict, Mapping, - Type, Union, overload, ) @@ -66,7 +64,7 @@ class Config: extra = pydantic.Extra.forbid @classmethod - def parse_yaml(cls: Type[Self], f: Union[pathlib.Path, str]) -> Self: + def parse_yaml(cls: type[Self], f: Union[pathlib.Path, str]) -> Self: loaded = yaml.load(f) logger.debug(f"loaded from yaml ->\n{pf(loaded, depth=3)}\n") # noinspection PyArgumentList @@ -277,7 +275,7 @@ def dict( # noqa: PLR0913 @staticmethod def _include_exclude_to_dict( include_exclude: AbstractSetIntStr | MappingIntStrAny | None, - ) -> Dict[int | str, Any]: + ) -> dict[int | str, Any]: """ Takes the mapping or abstract set passed to pydantic model export include or exclude and makes it a mutable dictionary that can be altered for nested include/exclude operations. diff --git a/great_expectations/datasource/fluent/interfaces.py b/great_expectations/datasource/fluent/interfaces.py index b276e280cdf2..b738fd9e8483 100644 --- a/great_expectations/datasource/fluent/interfaces.py +++ b/great_expectations/datasource/fluent/interfaces.py @@ -14,18 +14,14 @@ Any, Callable, ClassVar, - Dict, Final, Generic, - List, Mapping, MutableMapping, MutableSequence, Optional, Protocol, Sequence, - Set, - Type, TypeVar, Union, overload, @@ -134,7 +130,7 @@ def method_name(self) -> str: """ ... - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: """A shim to our execution engine partitioner methods We translate any internal Partitioner state and what is passed in from @@ -146,7 +142,7 @@ def partitioner_method_kwargs(self) -> Dict[str, Any]: def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """Translates `options` to the execution engine batch spec kwarg identifiers Arguments: @@ -224,7 +220,7 @@ class GxSerializationWarning(GxDatasourceWarning): pass -BatchMetadata: TypeAlias = Dict[str, Any] +BatchMetadata: TypeAlias = dict[str, Any] @pydantic_dc.dataclass(frozen=True) @@ -233,7 +229,7 @@ class Sorter: reverse: bool = False -SortersDefinition: TypeAlias = List[Union[Sorter, str, dict]] +SortersDefinition: TypeAlias = list[Union[Sorter, str, dict]] def _is_sorter_list( @@ -300,9 +296,9 @@ class DataAsset(GenericBaseModel, Generic[DatasourceT, PartitionerT], ABC): id: Optional[uuid.UUID] = Field(default=None, description="DataAsset id") # TODO: order_by should no longer be used and should be removed - order_by: List[Sorter] = Field(default_factory=list) + order_by: list[Sorter] = Field(default_factory=list) batch_metadata: BatchMetadata = pydantic.Field(default_factory=dict) - batch_definitions: List[BatchDefinition] = Field(default_factory=list) + batch_definitions: list[BatchDefinition] = Field(default_factory=list) # non-field private attributes _datasource: DatasourceT = pydantic.PrivateAttr() @@ -355,7 +351,7 @@ def build_batch_request( ) @abstractmethod - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: ... + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: ... @abstractmethod def get_batch(self, batch_request: BatchRequest) -> Batch: ... @@ -515,8 +511,8 @@ def _get_batch_metadata_from_batch_request( # Sorter methods @pydantic.validator("order_by", pre=True) def _order_by_validator( - cls, order_by: Optional[List[Union[Sorter, str, dict]]] = None - ) -> List[Sorter]: + cls, order_by: Optional[list[Union[Sorter, str, dict]]] = None + ) -> list[Sorter]: if order_by: raise DataAssetInitializationError( message="'order_by' is no longer a valid argument. " @@ -525,8 +521,8 @@ def _order_by_validator( return [] def sort_batches( - self, batch_list: List[Batch], partitioner: PartitionerSortingProtocol - ) -> List[Batch]: + self, batch_list: list[Batch], partitioner: PartitionerSortingProtocol + ) -> list[Batch]: """Sorts batch_list in place in the order configured in this DataAsset. Args: batch_list: The list of batches to sort in place. @@ -540,9 +536,9 @@ def get_value(key: str) -> Callable[[Batch], Any]: def sort_legacy_batch_definitions( self, - legacy_batch_definition_list: List[LegacyBatchDefinition], + legacy_batch_definition_list: list[LegacyBatchDefinition], partitioner: PartitionerSortingProtocol, - ) -> List[LegacyBatchDefinition]: + ) -> list[LegacyBatchDefinition]: """Sorts batch_definition_list in the order configured by the partitioner.""" def get_value(key: str) -> Callable[[LegacyBatchDefinition], Any]: @@ -551,8 +547,8 @@ def get_value(key: str) -> Callable[[LegacyBatchDefinition], Any]: return self._sort_batch_data_list(legacy_batch_definition_list, partitioner, get_value) def sort_batch_identifiers_list( - self, batch_identfiers_list: List[dict], partitioner: PartitionerSortingProtocol - ) -> List[dict]: + self, batch_identfiers_list: list[dict], partitioner: PartitionerSortingProtocol + ) -> list[dict]: """Sorts batch_identfiers_list in the order configured by the partitioner.""" def get_value(key: str) -> Callable[[dict], Any]: @@ -562,10 +558,10 @@ def get_value(key: str) -> Callable[[dict], Any]: def _sort_batch_data_list( self, - batch_data_list: List[_T], + batch_data_list: list[_T], partitioner: PartitionerSortingProtocol, get_value: Callable[[str], Any], - ) -> List[_T]: + ) -> list[_T]: """Sorts batch_data_list in the order configured by the partitioner.""" reverse = not partitioner.sort_ascending for key in reversed(partitioner.param_names): @@ -630,15 +626,15 @@ class Datasource( # data context method `data_context.data_sources.add_my_datasource` method. # class attrs - asset_types: ClassVar[Sequence[Type[DataAsset]]] = [] + asset_types: ClassVar[Sequence[type[DataAsset]]] = [] # Not all Datasources require a DataConnector - data_connector_type: ClassVar[Optional[Type[DataConnector]]] = None + data_connector_type: ClassVar[Optional[type[DataConnector]]] = None # Datasource sublcasses should update this set if the field should not be passed to the execution engine # noqa: E501 - _EXTRA_EXCLUDED_EXEC_ENG_ARGS: ClassVar[Set[str]] = set() + _EXTRA_EXCLUDED_EXEC_ENG_ARGS: ClassVar[set[str]] = set() _type_lookup: ClassVar[TypeLookup] # This attribute is set in `MetaDatasource.__new__` # Setting this in a Datasource subclass will override the execution engine type. # The primary use case is to inject an execution engine for testing. - execution_engine_override: ClassVar[Optional[Type[_ExecutionEngineT]]] = None # type: ignore[misc] # ClassVar cannot contain type variables + execution_engine_override: ClassVar[Optional[type[_ExecutionEngineT]]] = None # type: ignore[misc] # ClassVar cannot contain type variables # instance attrs type: str @@ -648,7 +644,7 @@ class Datasource( # private attrs _data_context: Union[GXDataContext, None] = pydantic.PrivateAttr(None) - _cached_execution_engine_kwargs: Dict[str, Any] = pydantic.PrivateAttr({}) + _cached_execution_engine_kwargs: dict[str, Any] = pydantic.PrivateAttr({}) _execution_engine: Union[_ExecutionEngineT, None] = pydantic.PrivateAttr(None) @property @@ -666,7 +662,7 @@ def data_context(self) -> GXDataContext | None: @pydantic.validator("assets", each_item=True) @classmethod def _load_asset_subtype( - cls: Type[Datasource[_DataAssetT, _ExecutionEngineT]], data_asset: DataAsset + cls: type[Datasource[_DataAssetT, _ExecutionEngineT]], data_asset: DataAsset ) -> _DataAssetT: """ Some `data_asset` may be loaded as a less specific asset subtype different than @@ -676,7 +672,7 @@ def _load_asset_subtype( """ logger.debug(f"Loading '{data_asset.name}' asset ->\n{pf(data_asset, depth=4)}") asset_type_name: str = data_asset.type - asset_type: Type[_DataAssetT] = cls._type_lookup[asset_type_name] + asset_type: type[_DataAssetT] = cls._type_lookup[asset_type_name] if asset_type is type(data_asset): # asset is already the intended type @@ -698,7 +694,7 @@ def _update_batch_definitions(cls, data_asset: DataAsset) -> DataAsset: batch_definition.set_data_asset(data_asset) return data_asset - def _execution_engine_type(self) -> Type[_ExecutionEngineT]: + def _execution_engine_type(self) -> type[_ExecutionEngineT]: """Returns the execution engine to be used""" return self.execution_engine_override or self.execution_engine_type @@ -774,7 +770,7 @@ def get_batch(self, batch_request: BatchRequest) -> Batch: data_asset = self.get_asset(batch_request.data_asset_name) return data_asset.get_batch(batch_request) - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: data_asset = self.get_asset(batch_request.data_asset_name) return data_asset.get_batch_identifiers_list(batch_request) @@ -791,7 +787,7 @@ def get_assets_as_dict(self) -> MutableMapping[str, _DataAssetT]: return assets_as_dict - def get_asset_names(self) -> Set[str]: + def get_asset_names(self) -> set[str]: """Returns the set of available DataAsset names Returns: @@ -857,7 +853,7 @@ def _add_asset(self, asset: _DataAssetT, connect_options: dict | None = None) -> asset.test_connection() - asset_names: Set[str] = self.get_asset_names() + asset_names: set[str] = self.get_asset_names() if asset.name in asset_names: raise ValueError( # noqa: TRY003 f'"{asset.name}" already exists (all existing assets are {", ".join(asset_names)})' @@ -910,7 +906,7 @@ def _rebuild_asset_data_connectors(self) -> None: ) if asset_build_failure_direct_cause: # TODO: allow users to opt out of these warnings - names_and_error: List[str] = [ + names_and_error: list[str] = [ f"{name}:{type(exc).__name__}" for (name, exc) in asset_build_failure_direct_cause.items() ] @@ -920,7 +916,7 @@ def _rebuild_asset_data_connectors(self) -> None: ) @staticmethod - def _update_asset_forward_refs(asset_type: Type[_DataAssetT]) -> None: + def _update_asset_forward_refs(asset_type: type[_DataAssetT]) -> None: """Update forward refs of an asset_type if necessary. Note, this should be overridden in child datasource classes if forward @@ -938,7 +934,7 @@ def _update_asset_forward_refs(asset_type: Type[_DataAssetT]) -> None: # Abstract Methods @property - def execution_engine_type(self) -> Type[_ExecutionEngineT]: + def execution_engine_type(self) -> type[_ExecutionEngineT]: """Return the ExecutionEngine type use for this Datasource""" raise NotImplementedError( """One needs to implement "execution_engine_type" on a Datasource subclass.""" @@ -969,7 +965,7 @@ def _build_data_connector(self, data_asset: _DataAssetT, **kwargs) -> None: pass @classmethod - def _get_exec_engine_excludes(cls) -> Set[str]: + def _get_exec_engine_excludes(cls) -> set[str]: """ Return a set of field names to exclude from the execution engine. @@ -984,7 +980,7 @@ def _get_exec_engine_excludes(cls) -> Set[str]: # This is used to prevent passing things like `type`, `assets` etc. to the execution engine -_BASE_DATASOURCE_FIELD_NAMES: Final[Set[str]] = {name for name in Datasource.__fields__} +_BASE_DATASOURCE_FIELD_NAMES: Final[set[str]] = {name for name in Datasource.__fields__} @dataclasses.dataclass(frozen=True) @@ -1018,7 +1014,7 @@ def __init__( # noqa: PLR0913 batch_markers: BatchMarkers, batch_spec: BatchSpec, batch_definition: LegacyBatchDefinition, - metadata: Dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, ): # Immutable attributes self._datasource = datasource @@ -1081,7 +1077,7 @@ def id(self) -> str: @public_api @validate_arguments - def columns(self) -> List[str]: + def columns(self) -> list[str]: """Return column names of this Batch. Returns diff --git a/great_expectations/datasource/fluent/invalid_datasource.py b/great_expectations/datasource/fluent/invalid_datasource.py index c2c672f66e4d..c0e004db7f67 100644 --- a/great_expectations/datasource/fluent/invalid_datasource.py +++ b/great_expectations/datasource/fluent/invalid_datasource.py @@ -6,9 +6,7 @@ Any, ClassVar, Final, - List, NoReturn, - Type, Union, overload, ) @@ -93,7 +91,7 @@ def build_batch_request( self._raise_type_error() @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: self._raise_type_error() @override @@ -102,8 +100,8 @@ def get_batch(self, batch_request: BatchRequest) -> Batch: @override def sort_batches( - self, batch_list: List[Batch], partitioner: PartitionerSortingProtocol - ) -> List[Batch]: + self, batch_list: list[Batch], partitioner: PartitionerSortingProtocol + ) -> list[Batch]: self._raise_type_error() @override @@ -115,10 +113,10 @@ class InvalidAssetTypeLookup(TypeLookup): """A TypeLookup that always returns InvalidAsset for any type.""" @overload - def __getitem__(self, key: str) -> Type: ... + def __getitem__(self, key: str) -> type: ... @overload - def __getitem__(self, key: Type) -> str: ... + def __getitem__(self, key: type) -> str: ... @override def __getitem__(self, key: ValidTypes) -> ValidTypes: @@ -145,14 +143,14 @@ class InvalidDatasource(Datasource): """ # noqa: E501 # class var definitions - asset_types: ClassVar[List[Type[DataAsset]]] = [InvalidAsset] + asset_types: ClassVar[list[type[DataAsset]]] = [InvalidAsset] _type_lookup: ClassVar[TypeLookup] = InvalidAssetTypeLookup() type: str = "invalid" config_error: Union[pydantic.ValidationError, LookupError] = Field( ..., description="The error that caused the Datasource to be invalid." ) - assets: List[InvalidAsset] = [] + assets: list[InvalidAsset] = [] class Config: extra = "ignore" diff --git a/great_expectations/datasource/fluent/metadatasource.py b/great_expectations/datasource/fluent/metadatasource.py index 884fd8e52924..7846d5daa1a5 100644 --- a/great_expectations/datasource/fluent/metadatasource.py +++ b/great_expectations/datasource/fluent/metadatasource.py @@ -6,7 +6,6 @@ import logging from pprint import pformat as pf -from typing import Set, Type from great_expectations.compatibility.pydantic import ModelMetaclass from great_expectations.datasource.fluent.sources import DataSourceManager @@ -16,10 +15,10 @@ class MetaDatasource(ModelMetaclass): - __cls_set: Set[Type] = set() + __cls_set: set[type] = set() def __new__( # noqa: PYI034 # Self cannot be used with Metaclass - meta_cls: Type[MetaDatasource], cls_name: str, bases: tuple[type], cls_dict + meta_cls: type[MetaDatasource], cls_name: str, bases: tuple[type], cls_dict ) -> MetaDatasource: """ MetaDatasource hook that runs when a new `Datasource` is defined. diff --git a/great_expectations/datasource/fluent/pandas_azure_blob_storage_datasource.py b/great_expectations/datasource/fluent/pandas_azure_blob_storage_datasource.py index ec4f01e033c0..14163d686d50 100644 --- a/great_expectations/datasource/fluent/pandas_azure_blob_storage_datasource.py +++ b/great_expectations/datasource/fluent/pandas_azure_blob_storage_datasource.py @@ -2,7 +2,7 @@ import logging import re -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Final, Literal, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Union from great_expectations._docs_decorators import public_api from great_expectations.compatibility import azure, pydantic @@ -36,7 +36,7 @@ class PandasAzureBlobStorageDatasourceError(PandasDatasourceError): @public_api class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[AzureBlobStorageDataConnector]] = ( + data_connector_type: ClassVar[type[AzureBlobStorageDataConnector]] = ( AzureBlobStorageDataConnector ) @@ -44,7 +44,7 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): type: Literal["pandas_abs"] = "pandas_abs" # Azure Blob Storage specific attributes - azure_options: Dict[str, Union[ConfigStr, Any]] = {} + azure_options: dict[str, Union[ConfigStr, Any]] = {} _account_name: str = pydantic.PrivateAttr(default="") # on 3.11 the annotation must be type-checking import otherwise it will fail at import time diff --git a/great_expectations/datasource/fluent/pandas_azure_blob_storage_datasource.pyi b/great_expectations/datasource/fluent/pandas_azure_blob_storage_datasource.pyi index 1ec95b15a425..a4f97a152864 100644 --- a/great_expectations/datasource/fluent/pandas_azure_blob_storage_datasource.pyi +++ b/great_expectations/datasource/fluent/pandas_azure_blob_storage_datasource.pyi @@ -2,13 +2,11 @@ import typing from logging import Logger from typing import ( Any, - Dict, Hashable, Iterable, Literal, Optional, Sequence, - Tuple, Union, ) @@ -65,7 +63,7 @@ class PandasAzureBlobStorageDatasourceError(PandasDatasourceError): ... class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): type: Literal["pandas_abs"] - azure_options: Dict[str, ConfigStr | Any] + azure_options: dict[str, ConfigStr | Any] _account_name: str _azure_client: Any @@ -142,7 +140,7 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): abs_delimiter: str = "/", sheet_name: typing.Union[str, int, None] = 0, header: Union[int, Sequence[int], None] = 0, - names: typing.Union[typing.List[str], None] = ..., + names: typing.Union[list[str], None] = ..., index_col: Union[int, Sequence[int], None] = ..., usecols: typing.Union[int, str, typing.Sequence[int], None] = ..., squeeze: typing.Union[bool, None] = ..., @@ -155,7 +153,7 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): keep_default_na: bool = ..., na_filter: bool = ..., verbose: bool = ..., - parse_dates: typing.Union[typing.List, typing.Dict, bool] = ..., + parse_dates: typing.Union[list, dict, bool] = ..., thousands: typing.Union[str, None] = ..., decimal: str = ".", comment: typing.Union[str, None] = ..., @@ -183,7 +181,7 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., connect_options: typing.Mapping = ..., - colspecs: Union[Sequence[Tuple[int, int]], str, None] = ..., + colspecs: Union[Sequence[tuple[int, int]], str, None] = ..., widths: Union[Sequence[int], None] = ..., infer_nrows: int = ..., kwargs: Optional[dict] = ..., @@ -199,10 +197,10 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): key: typing.Any = ..., mode: str = "r", errors: str = "strict", - where: typing.Union[str, typing.List, None] = ..., + where: typing.Union[str, list, None] = ..., start: typing.Union[int, None] = ..., stop: typing.Union[int, None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., iterator: bool = ..., chunksize: typing.Union[int, None] = ..., kwargs: typing.Union[dict, None] = ..., @@ -220,12 +218,12 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): header: Union[int, Sequence[int], None] = ..., index_col: Union[int, Sequence[int], None] = ..., skiprows: typing.Union[typing.Sequence[int], int, None] = ..., - attrs: typing.Union[typing.Dict[str, str], None] = ..., + attrs: typing.Union[dict[str, str], None] = ..., parse_dates: bool = ..., thousands: typing.Union[str, None] = ",", encoding: typing.Union[str, None] = ..., decimal: str = ".", - converters: typing.Union[typing.Dict, None] = ..., + converters: typing.Union[dict, None] = ..., na_values: Union[Iterable[object], None] = ..., keep_default_na: bool = ..., displayed_only: bool = ..., @@ -241,7 +239,7 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): orient: typing.Union[str, None] = ..., dtype: typing.Union[dict, None] = ..., convert_axes: typing.Any = ..., - convert_dates: typing.Union[bool, typing.List[str]] = ..., + convert_dates: typing.Union[bool, list[str]] = ..., keep_default_dates: bool = ..., numpy: bool = ..., precise_float: bool = ..., @@ -262,7 +260,7 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): abs_container: str = ..., abs_name_starts_with: str = "", abs_delimiter: str = "/", - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., kwargs: typing.Union[dict, None] = ..., ) -> ORCAsset: ... def add_parquet_asset( # noqa: PLR0913 @@ -274,7 +272,7 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): abs_name_starts_with: str = "", abs_delimiter: str = "/", engine: str = "auto", - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., storage_options: StorageOptions = ..., use_nullable_dtypes: bool = ..., kwargs: typing.Union[dict, None] = ..., @@ -345,14 +343,14 @@ class PandasAzureBlobStorageDatasource(_PandasFilePathDatasource): abs_name_starts_with: str = "", abs_delimiter: str = "/", xpath: str = "./*", - namespaces: typing.Union[typing.Dict[str, str], None] = ..., + namespaces: typing.Union[dict[str, str], None] = ..., elems_only: bool = ..., attrs_only: bool = ..., names: Union[Sequence[str], None] = ..., dtype: typing.Union[dict, None] = ..., encoding: typing.Union[str, None] = "utf-8", stylesheet: Union[FilePath, None] = ..., - iterparse: typing.Union[typing.Dict[str, typing.List[str]], None] = ..., + iterparse: typing.Union[dict[str, list[str]], None] = ..., compression: CompressionOptions = "infer", storage_options: StorageOptions = ..., ) -> XMLAsset: ... diff --git a/great_expectations/datasource/fluent/pandas_datasource.py b/great_expectations/datasource/fluent/pandas_datasource.py index 88ee6e8059fb..0e050760c838 100644 --- a/great_expectations/datasource/fluent/pandas_datasource.py +++ b/great_expectations/datasource/fluent/pandas_datasource.py @@ -1,5 +1,6 @@ from __future__ import annotations +import builtins import logging import sqlite3 import uuid @@ -11,15 +12,11 @@ Callable, ClassVar, Generic, - List, Literal, Mapping, MutableSequence, Optional, Sequence, - Set, - Tuple, - Type, Union, ) @@ -54,7 +51,7 @@ from great_expectations.datasource.fluent.sources import DEFAULT_PANDAS_DATA_ASSET_NAME from great_expectations.exceptions.exceptions import BuildBatchRequestError -_EXCLUDE_TYPES_FROM_JSON: list[Type] = [sqlite3.Connection] +_EXCLUDE_TYPES_FROM_JSON: list[type] = [sqlite3.Connection] if sa: _EXCLUDE_TYPES_FROM_JSON = _EXCLUDE_TYPES_FROM_JSON + [sqlalchemy.Engine] @@ -85,7 +82,7 @@ class PandasDatasourceError(Exception): @public_api class _PandasDataAsset(DataAsset): - _EXCLUDE_FROM_READER_OPTIONS: ClassVar[Set[str]] = { + _EXCLUDE_FROM_READER_OPTIONS: ClassVar[set[str]] = { "batch_definitions", "batch_metadata", "name", @@ -116,13 +113,13 @@ def test_connection(self) -> None: ... @override def get_batch_parameters_keys( self, partitioner: Optional[ColumnPartitioner] = None - ) -> Tuple[str, ...]: + ) -> tuple[str, ...]: return tuple( "dataframe", ) @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: return [IDDict(batch_request.options)] @override @@ -322,26 +319,26 @@ def json( # noqa: PLR0913 ) -ClipboardAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("clipboard", _PandasDataAsset) -CSVAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("csv", _PandasDataAsset) -ExcelAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("excel", _PandasDataAsset) -FeatherAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("feather", _PandasDataAsset) -FWFAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("fwf", _PandasDataAsset) -GBQAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("gbq", _PandasDataAsset) -HDFAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("hdf", _PandasDataAsset) -HTMLAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("html", _PandasDataAsset) -JSONAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("json", _PandasDataAsset) -ORCAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("orc", _PandasDataAsset) -ParquetAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("parquet", _PandasDataAsset) -PickleAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("pickle", _PandasDataAsset) -SQLAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("sql", _PandasDataAsset) -SQLQueryAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("sql_query", _PandasDataAsset) -SQLTableAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("sql_table", _PandasDataAsset) -SASAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("sas", _PandasDataAsset) -SPSSAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("spss", _PandasDataAsset) -StataAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("stata", _PandasDataAsset) -TableAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("table", _PandasDataAsset) -XMLAsset: Type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get( +ClipboardAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("clipboard", _PandasDataAsset) +CSVAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("csv", _PandasDataAsset) +ExcelAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("excel", _PandasDataAsset) +FeatherAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("feather", _PandasDataAsset) +FWFAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("fwf", _PandasDataAsset) +GBQAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("gbq", _PandasDataAsset) +HDFAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("hdf", _PandasDataAsset) +HTMLAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("html", _PandasDataAsset) +JSONAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("json", _PandasDataAsset) +ORCAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("orc", _PandasDataAsset) +ParquetAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("parquet", _PandasDataAsset) +PickleAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("pickle", _PandasDataAsset) +SQLAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("sql", _PandasDataAsset) +SQLQueryAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("sql_query", _PandasDataAsset) +SQLTableAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("sql_table", _PandasDataAsset) +SASAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("sas", _PandasDataAsset) +SPSSAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("spss", _PandasDataAsset) +StataAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("stata", _PandasDataAsset) +TableAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get("table", _PandasDataAsset) +XMLAsset: type[_PandasDataAsset] = _PANDAS_ASSET_MODELS.get( "xml", _PandasDataAsset ) # read_xml doesn't exist for pandas < 1.3 @@ -445,7 +442,7 @@ def _validate_batch_request(self, batch_request: BatchRequest) -> None: ) @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: return [IDDict(batch_request.options)] @override @@ -485,7 +482,7 @@ def get_batch(self, batch_request: BatchRequest) -> Batch: class _PandasDatasource(Datasource, Generic[_DataAssetT]): # class attributes - asset_types: ClassVar[Sequence[Type[DataAsset]]] = [] + asset_types: ClassVar[Sequence[type[DataAsset]]] = [] # instance attributes assets: MutableSequence[_DataAssetT] = [] @@ -493,7 +490,7 @@ class _PandasDatasource(Datasource, Generic[_DataAssetT]): # Abstract Methods @property @override - def execution_engine_type(self) -> Type[PandasExecutionEngine]: + def execution_engine_type(self) -> type[PandasExecutionEngine]: """Return the PandasExecutionEngine unless the override is set""" from great_expectations.execution_engine.pandas_execution_engine import ( PandasExecutionEngine, @@ -583,7 +580,7 @@ def _add_asset(self, asset: _DataAssetT, connect_options: dict | None = None) -> """ # noqa: E501 asset_name: str = asset.name - asset_names: Set[str] = self.get_asset_names() + asset_names: set[str] = self.get_asset_names() in_cloud_context: bool = False if self._data_context: @@ -620,11 +617,13 @@ class PandasDatasource(_PandasDatasource): ADD_READER_METHODS: ClassVar[bool] = True # class attributes - asset_types: ClassVar[Sequence[Type[DataAsset]]] = _DYNAMIC_ASSET_TYPES + [DataFrameAsset] + asset_types: ClassVar[Sequence[builtins.type[DataAsset]]] = _DYNAMIC_ASSET_TYPES + [ + DataFrameAsset + ] # instance attributes type: Literal["pandas"] = "pandas" - assets: List[_PandasDataAsset] = [] + assets: list[_PandasDataAsset] = [] @override def dict(self, _exclude_default_asset_names: bool = True, **kwargs): diff --git a/great_expectations/datasource/fluent/pandas_datasource.pyi b/great_expectations/datasource/fluent/pandas_datasource.pyi index 1e434a9d2a25..fde442471fb1 100644 --- a/great_expectations/datasource/fluent/pandas_datasource.pyi +++ b/great_expectations/datasource/fluent/pandas_datasource.pyi @@ -9,15 +9,11 @@ from typing import ( ClassVar, Hashable, Iterable, - List, Literal, Mapping, MutableSequence, Optional, Sequence, - Set, - Tuple, - Type, TypeVar, Union, ) @@ -54,7 +50,7 @@ from great_expectations.datasource.fluent.interfaces import ( ) from great_expectations.execution_engine import PandasExecutionEngine -_EXCLUDE_TYPES_FROM_JSON: list[Type] +_EXCLUDE_TYPES_FROM_JSON: list[type] MappingIntStrAny: TypeAlias = Mapping[Union[int, str], Any] AbstractSetIntStr: TypeAlias = AbstractSet[Union[int, str]] @@ -63,7 +59,7 @@ logger: Logger class PandasDatasourceError(Exception): ... class _PandasDataAsset(DataAsset): - _EXCLUDE_FROM_READER_OPTIONS: ClassVar[Set[str]] + _EXCLUDE_FROM_READER_OPTIONS: ClassVar[set[str]] def _get_reader_method(self) -> str: ... @override @@ -133,16 +129,16 @@ class DataFrameAsset(_PandasDataAsset): @override def get_batch(self, batch_request: BatchRequest) -> Batch: ... @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: ... + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: ... _PandasDataAssetT = TypeVar("_PandasDataAssetT", bound=_PandasDataAsset) class _PandasDatasource(Datasource): - asset_types: ClassVar[Sequence[Type[DataAsset]]] + asset_types: ClassVar[Sequence[type[DataAsset]]] assets: MutableSequence[_PandasDataAssetT] # type: ignore[valid-type] @property @override - def execution_engine_type(self) -> Type[PandasExecutionEngine]: ... + def execution_engine_type(self) -> type[PandasExecutionEngine]: ... @override def test_connection(self, test_assets: bool = ...) -> None: ... @override @@ -161,12 +157,12 @@ class _PandasDatasource(Datasource): **dumps_kwargs: Any, ) -> str: ... -_DYNAMIC_ASSET_TYPES: list[Type[_PandasDataAsset]] +_DYNAMIC_ASSET_TYPES: list[type[_PandasDataAsset]] class PandasDatasource(_PandasDatasource): - asset_types: ClassVar[Sequence[Type[DataAsset]]] + asset_types: ClassVar[Sequence[type[DataAsset]]] type: Literal["pandas"] - assets: List[_PandasDataAsset] + assets: list[_PandasDataAsset] @override def test_connection(self, test_assets: bool = ...) -> None: ... @deprecated_argument( @@ -261,7 +257,7 @@ class PandasDatasource(_PandasDatasource): batch_metadata: Optional[BatchMetadata] = ..., sheet_name: typing.Union[str, int, None] = 0, header: Union[int, Sequence[int], None] = 0, - names: typing.Union[typing.List[str], None] = ..., + names: typing.Union[list[str], None] = ..., index_col: Union[int, Sequence[int], None] = ..., usecols: typing.Union[int, str, typing.Sequence[int], None] = ..., squeeze: typing.Union[bool, None] = ..., @@ -274,7 +270,7 @@ class PandasDatasource(_PandasDatasource): keep_default_na: bool = ..., na_filter: bool = ..., verbose: bool = ..., - parse_dates: typing.Union[typing.List, typing.Dict, bool] = ..., + parse_dates: typing.Union[list, dict, bool] = ..., thousands: typing.Union[str, None] = ..., decimal: str = ".", comment: typing.Union[str, None] = ..., @@ -299,7 +295,7 @@ class PandasDatasource(_PandasDatasource): filepath_or_buffer: pydantic.FilePath | pydantic.AnyUrl, *, batch_metadata: Optional[BatchMetadata] = ..., - colspecs: Union[Sequence[Tuple[int, int]], str, None] = ..., + colspecs: Union[Sequence[tuple[int, int]], str, None] = ..., widths: Union[Sequence[int], None] = ..., infer_nrows: int = ..., kwargs: Optional[dict] = ..., @@ -312,12 +308,12 @@ class PandasDatasource(_PandasDatasource): batch_metadata: Optional[BatchMetadata] = ..., project_id: typing.Union[str, None] = ..., index_col: typing.Union[str, None] = ..., - col_order: typing.Union[typing.List[str], None] = ..., + col_order: typing.Union[list[str], None] = ..., reauth: bool = ..., auth_local_webserver: bool = ..., dialect: typing.Union[str, None] = ..., location: typing.Union[str, None] = ..., - configuration: typing.Union[typing.Dict[str, typing.Any], None] = ..., + configuration: typing.Union[dict[str, typing.Any], None] = ..., credentials: typing.Any = ..., use_bqstorage_api: typing.Union[bool, None] = ..., max_results: typing.Union[int, None] = ..., @@ -332,10 +328,10 @@ class PandasDatasource(_PandasDatasource): key: typing.Any = ..., mode: str = "r", errors: str = "strict", - where: typing.Union[str, typing.List, None] = ..., + where: typing.Union[str, list, None] = ..., start: typing.Union[int, None] = ..., stop: typing.Union[int, None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., iterator: bool = ..., chunksize: typing.Union[int, None] = ..., kwargs: typing.Union[dict, None] = ..., @@ -351,12 +347,12 @@ class PandasDatasource(_PandasDatasource): header: Union[int, Sequence[int], None] = ..., index_col: Union[int, Sequence[int], None] = ..., skiprows: typing.Union[typing.Sequence[int], int, None] = ..., - attrs: typing.Union[typing.Dict[str, str], None] = ..., + attrs: typing.Union[dict[str, str], None] = ..., parse_dates: bool = ..., thousands: typing.Union[str, None] = ",", encoding: typing.Union[str, None] = ..., decimal: str = ".", - converters: typing.Union[typing.Dict, None] = ..., + converters: typing.Union[dict, None] = ..., na_values: Union[Iterable[object], None] = ..., keep_default_na: bool = ..., displayed_only: bool = ..., @@ -370,7 +366,7 @@ class PandasDatasource(_PandasDatasource): orient: typing.Union[str, None] = ..., dtype: typing.Union[dict, None] = ..., convert_axes: typing.Any = ..., - convert_dates: typing.Union[bool, typing.List[str]] = ..., + convert_dates: typing.Union[bool, list[str]] = ..., keep_default_dates: bool = ..., numpy: bool = ..., precise_float: bool = ..., @@ -389,7 +385,7 @@ class PandasDatasource(_PandasDatasource): path: pydantic.FilePath | pydantic.AnyUrl, *, batch_metadata: Optional[BatchMetadata] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., kwargs: typing.Union[dict, None] = ..., ) -> ORCAsset: ... def add_parquet_asset( # noqa: PLR0913 @@ -399,7 +395,7 @@ class PandasDatasource(_PandasDatasource): *, batch_metadata: Optional[BatchMetadata] = ..., engine: str = "auto", - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., storage_options: StorageOptions = ..., use_nullable_dtypes: bool = ..., kwargs: typing.Union[dict, None] = ..., @@ -442,11 +438,11 @@ class PandasDatasource(_PandasDatasource): con: sqlalchemy.Engine | sqlite3.Connection | str, *, batch_metadata: Optional[BatchMetadata] = ..., - index_col: typing.Union[str, typing.List[str], None] = ..., + index_col: typing.Union[str, list[str], None] = ..., coerce_float: bool = ..., params: typing.Any = ..., parse_dates: typing.Any = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., chunksize: typing.Union[int, None] = ..., ) -> SQLAsset: ... def add_sql_query_asset( # noqa: PLR0913 @@ -456,10 +452,10 @@ class PandasDatasource(_PandasDatasource): con: sqlalchemy.Engine | sqlite3.Connection | str, *, batch_metadata: Optional[BatchMetadata] = ..., - index_col: typing.Union[str, typing.List[str], None] = ..., + index_col: typing.Union[str, list[str], None] = ..., coerce_float: bool = ..., - params: typing.Union[typing.List[str], typing.Dict[str, str], None] = ..., - parse_dates: typing.Union[typing.List[str], typing.Dict[str, str], None] = ..., + params: typing.Union[list[str], dict[str, str], None] = ..., + parse_dates: typing.Union[list[str], dict[str, str], None] = ..., chunksize: typing.Union[int, None] = ..., dtype: typing.Union[dict, None] = ..., ) -> SQLQueryAsset: ... @@ -471,10 +467,10 @@ class PandasDatasource(_PandasDatasource): *, batch_metadata: Optional[BatchMetadata] = ..., schema: typing.Union[str, None] = ..., - index_col: typing.Union[str, typing.List[str], None] = ..., + index_col: typing.Union[str, list[str], None] = ..., coerce_float: bool = ..., - parse_dates: typing.Union[typing.List[str], typing.Dict[str, str], None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + parse_dates: typing.Union[list[str], dict[str, str], None] = ..., + columns: typing.Union[list[str], None] = ..., chunksize: typing.Union[int, None] = ..., ) -> SQLTableAsset: ... def add_stata_asset( # noqa: PLR0913 @@ -560,14 +556,14 @@ class PandasDatasource(_PandasDatasource): *, batch_metadata: Optional[BatchMetadata] = ..., xpath: str = "./*", - namespaces: typing.Union[typing.Dict[str, str], None] = ..., + namespaces: typing.Union[dict[str, str], None] = ..., elems_only: bool = ..., attrs_only: bool = ..., names: Union[Sequence[str], None] = ..., dtype: typing.Union[dict, None] = ..., encoding: typing.Union[str, None] = "utf-8", stylesheet: Union[FilePath, None] = ..., - iterparse: typing.Union[typing.Dict[str, typing.List[str]], None] = ..., + iterparse: typing.Union[dict[str, list[str]], None] = ..., compression: CompressionOptions = "infer", storage_options: StorageOptions = ..., ) -> XMLAsset: ... @@ -643,7 +639,7 @@ class PandasDatasource(_PandasDatasource): batch_metadata: Optional[BatchMetadata] = ..., sheet_name: typing.Union[str, int, None] = 0, header: Union[int, Sequence[int], None] = 0, - names: typing.Union[typing.List[str], None] = ..., + names: typing.Union[list[str], None] = ..., index_col: Union[int, Sequence[int], None] = ..., usecols: typing.Union[int, str, typing.Sequence[int], None] = ..., squeeze: typing.Union[bool, None] = ..., @@ -656,7 +652,7 @@ class PandasDatasource(_PandasDatasource): keep_default_na: bool = ..., na_filter: bool = ..., verbose: bool = ..., - parse_dates: typing.Union[typing.List, typing.Dict, bool] = ..., + parse_dates: typing.Union[list, dict, bool] = ..., thousands: typing.Union[str, None] = ..., decimal: str = ".", comment: typing.Union[str, None] = ..., @@ -680,7 +676,7 @@ class PandasDatasource(_PandasDatasource): filepath_or_buffer: pydantic.FilePath | pydantic.AnyUrl, *, batch_metadata: Optional[BatchMetadata] = ..., - colspecs: Union[Sequence[Tuple[int, int]], str, None] = ..., + colspecs: Union[Sequence[tuple[int, int]], str, None] = ..., widths: Union[Sequence[int], None] = ..., infer_nrows: int = ..., kwargs: Optional[dict] = ..., @@ -693,12 +689,12 @@ class PandasDatasource(_PandasDatasource): batch_metadata: Optional[BatchMetadata] = ..., project_id: typing.Union[str, None] = ..., index_col: typing.Union[str, None] = ..., - col_order: typing.Union[typing.List[str], None] = ..., + col_order: typing.Union[list[str], None] = ..., reauth: bool = ..., auth_local_webserver: bool = ..., dialect: typing.Union[str, None] = ..., location: typing.Union[str, None] = ..., - configuration: typing.Union[typing.Dict[str, typing.Any], None] = ..., + configuration: typing.Union[dict[str, typing.Any], None] = ..., credentials: typing.Any = ..., use_bqstorage_api: typing.Union[bool, None] = ..., max_results: typing.Union[int, None] = ..., @@ -713,10 +709,10 @@ class PandasDatasource(_PandasDatasource): key: typing.Any = ..., mode: str = "r", errors: str = "strict", - where: typing.Union[str, typing.List, None] = ..., + where: typing.Union[str, list, None] = ..., start: typing.Union[int, None] = ..., stop: typing.Union[int, None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., iterator: bool = ..., chunksize: typing.Union[int, None] = ..., kwargs: typing.Union[dict, None] = ..., @@ -732,12 +728,12 @@ class PandasDatasource(_PandasDatasource): header: Union[int, Sequence[int], None] = ..., index_col: Union[int, Sequence[int], None] = ..., skiprows: typing.Union[typing.Sequence[int], int, None] = ..., - attrs: typing.Union[typing.Dict[str, str], None] = ..., + attrs: typing.Union[dict[str, str], None] = ..., parse_dates: bool = ..., thousands: typing.Union[str, None] = ",", encoding: typing.Union[str, None] = ..., decimal: str = ".", - converters: typing.Union[typing.Dict, None] = ..., + converters: typing.Union[dict, None] = ..., na_values: Union[Iterable[object], None] = ..., keep_default_na: bool = ..., displayed_only: bool = ..., @@ -751,7 +747,7 @@ class PandasDatasource(_PandasDatasource): orient: typing.Union[str, None] = ..., dtype: typing.Union[dict, None] = ..., convert_axes: typing.Any = ..., - convert_dates: typing.Union[bool, typing.List[str]] = ..., + convert_dates: typing.Union[bool, list[str]] = ..., keep_default_dates: bool = ..., numpy: bool = ..., precise_float: bool = ..., @@ -770,7 +766,7 @@ class PandasDatasource(_PandasDatasource): *, asset_name: Optional[str] = ..., batch_metadata: Optional[BatchMetadata] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., kwargs: typing.Union[dict, None] = ..., ) -> Batch: ... def read_parquet( # noqa: PLR0913 @@ -780,7 +776,7 @@ class PandasDatasource(_PandasDatasource): asset_name: Optional[str] = ..., batch_metadata: Optional[BatchMetadata] = ..., engine: str = "auto", - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., storage_options: StorageOptions = ..., use_nullable_dtypes: bool = ..., kwargs: typing.Union[dict, None] = ..., @@ -823,11 +819,11 @@ class PandasDatasource(_PandasDatasource): *, asset_name: Optional[str] = ..., batch_metadata: Optional[BatchMetadata] = ..., - index_col: typing.Union[str, typing.List[str], None] = ..., + index_col: typing.Union[str, list[str], None] = ..., coerce_float: bool = ..., params: typing.Any = ..., parse_dates: typing.Any = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., chunksize: typing.Union[int, None] = ..., ) -> Batch: ... def read_sql_query( # noqa: PLR0913 @@ -837,10 +833,10 @@ class PandasDatasource(_PandasDatasource): *, asset_name: Optional[str] = ..., batch_metadata: Optional[BatchMetadata] = ..., - index_col: typing.Union[str, typing.List[str], None] = ..., + index_col: typing.Union[str, list[str], None] = ..., coerce_float: bool = ..., - params: typing.Union[typing.List[str], typing.Dict[str, str], None] = ..., - parse_dates: typing.Union[typing.List[str], typing.Dict[str, str], None] = ..., + params: typing.Union[list[str], dict[str, str], None] = ..., + parse_dates: typing.Union[list[str], dict[str, str], None] = ..., chunksize: typing.Union[int, None] = ..., dtype: typing.Union[dict, None] = ..., ) -> Batch: ... @@ -852,10 +848,10 @@ class PandasDatasource(_PandasDatasource): asset_name: Optional[str] = ..., batch_metadata: Optional[BatchMetadata] = ..., schema: typing.Union[str, None] = ..., - index_col: typing.Union[str, typing.List[str], None] = ..., + index_col: typing.Union[str, list[str], None] = ..., coerce_float: bool = ..., - parse_dates: typing.Union[typing.List[str], typing.Dict[str, str], None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + parse_dates: typing.Union[list[str], dict[str, str], None] = ..., + columns: typing.Union[list[str], None] = ..., chunksize: typing.Union[int, None] = ..., ) -> Batch: ... def read_stata( # noqa: PLR0913 @@ -941,14 +937,14 @@ class PandasDatasource(_PandasDatasource): asset_name: Optional[str] = ..., batch_metadata: Optional[BatchMetadata] = ..., xpath: str = "./*", - namespaces: typing.Union[typing.Dict[str, str], None] = ..., + namespaces: typing.Union[dict[str, str], None] = ..., elems_only: bool = ..., attrs_only: bool = ..., names: Union[Sequence[str], None] = ..., dtype: typing.Union[dict, None] = ..., encoding: typing.Union[str, None] = "utf-8", stylesheet: Union[FilePath, None] = ..., - iterparse: typing.Union[typing.Dict[str, typing.List[str]], None] = ..., + iterparse: typing.Union[dict[str, list[str]], None] = ..., compression: CompressionOptions = "infer", storage_options: StorageOptions = ..., ) -> Batch: ... diff --git a/great_expectations/datasource/fluent/pandas_dbfs_datasource.py b/great_expectations/datasource/fluent/pandas_dbfs_datasource.py index 373e128807cd..a0d69f04fade 100644 --- a/great_expectations/datasource/fluent/pandas_dbfs_datasource.py +++ b/great_expectations/datasource/fluent/pandas_dbfs_datasource.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, ClassVar, Literal, Type +from typing import TYPE_CHECKING, ClassVar, Literal from great_expectations._docs_decorators import public_api from great_expectations.compatibility.typing_extensions import override @@ -22,7 +22,7 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): """Pandas based Datasource for DataBricks File System (DBFS) based data assets.""" # class attributes - data_connector_type: ClassVar[Type[DBFSDataConnector]] = DBFSDataConnector + data_connector_type: ClassVar[type[DBFSDataConnector]] = DBFSDataConnector # instance attributes # overridden from base `Literal['pandas_filesystem']` diff --git a/great_expectations/datasource/fluent/pandas_dbfs_datasource.pyi b/great_expectations/datasource/fluent/pandas_dbfs_datasource.pyi index 3c3f8e12e66b..a8f9e0c4b58f 100644 --- a/great_expectations/datasource/fluent/pandas_dbfs_datasource.pyi +++ b/great_expectations/datasource/fluent/pandas_dbfs_datasource.pyi @@ -110,7 +110,7 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): batch_metadata: Optional[BatchMetadata] = ..., sheet_name: typing.Union[str, int, None] = 0, header: Union[int, Sequence[int], None] = 0, - names: typing.Union[typing.List[str], None] = ..., + names: typing.Union[list[str], None] = ..., index_col: Union[int, Sequence[int], None] = ..., usecols: typing.Union[int, str, typing.Sequence[int], None] = ..., squeeze: typing.Union[bool, None] = ..., @@ -123,7 +123,7 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): keep_default_na: bool = ..., na_filter: bool = ..., verbose: bool = ..., - parse_dates: typing.Union[typing.List, typing.Dict, bool] = ..., + parse_dates: typing.Union[list, dict, bool] = ..., thousands: typing.Union[str, None] = ..., decimal: str = ".", comment: typing.Union[str, None] = ..., @@ -153,10 +153,10 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): key: typing.Any = ..., mode: str = "r", errors: str = "strict", - where: typing.Union[str, typing.List, None] = ..., + where: typing.Union[str, list, None] = ..., start: typing.Union[int, None] = ..., stop: typing.Union[int, None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., iterator: bool = ..., chunksize: typing.Union[int, None] = ..., kwargs: typing.Union[dict, None] = ..., @@ -173,12 +173,12 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): header: Union[int, Sequence[int], None] = ..., index_col: Union[int, Sequence[int], None] = ..., skiprows: typing.Union[typing.Sequence[int], int, None] = ..., - attrs: typing.Union[typing.Dict[str, str], None] = ..., + attrs: typing.Union[dict[str, str], None] = ..., parse_dates: bool = ..., thousands: typing.Union[str, None] = ",", encoding: typing.Union[str, None] = ..., decimal: str = ".", - converters: typing.Union[typing.Dict, None] = ..., + converters: typing.Union[dict, None] = ..., na_values: Union[Iterable[object], None] = ..., keep_default_na: bool = ..., displayed_only: bool = ..., @@ -193,7 +193,7 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): orient: typing.Union[str, None] = ..., dtype: typing.Union[dict, None] = ..., convert_axes: typing.Any = ..., - convert_dates: typing.Union[bool, typing.List[str]] = ..., + convert_dates: typing.Union[bool, list[str]] = ..., keep_default_dates: bool = ..., numpy: bool = ..., precise_float: bool = ..., @@ -213,7 +213,7 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): *, glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., kwargs: typing.Union[dict, None] = ..., ) -> ORCAsset: ... @override @@ -224,7 +224,7 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., engine: str = "auto", - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., storage_options: StorageOptions = ..., use_nullable_dtypes: bool = ..., kwargs: typing.Union[dict, None] = ..., @@ -290,14 +290,14 @@ class PandasDBFSDatasource(PandasFilesystemDatasource): glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., xpath: str = "./*", - namespaces: typing.Union[typing.Dict[str, str], None] = ..., + namespaces: typing.Union[dict[str, str], None] = ..., elems_only: bool = ..., attrs_only: bool = ..., names: Union[Sequence[str], None] = ..., dtype: typing.Union[dict, None] = ..., encoding: typing.Union[str, None] = "utf-8", stylesheet: Union[FilePath, None] = ..., - iterparse: typing.Union[typing.Dict[str, typing.List[str]], None] = ..., + iterparse: typing.Union[dict[str, list[str]], None] = ..., compression: CompressionOptions = "infer", storage_options: StorageOptions = ..., ) -> XMLAsset: ... diff --git a/great_expectations/datasource/fluent/pandas_file_path_datasource.py b/great_expectations/datasource/fluent/pandas_file_path_datasource.py index 206738aab6c6..6a7cc6414388 100644 --- a/great_expectations/datasource/fluent/pandas_file_path_datasource.py +++ b/great_expectations/datasource/fluent/pandas_file_path_datasource.py @@ -3,8 +3,6 @@ from typing import ( TYPE_CHECKING, ClassVar, - List, - Type, ) from great_expectations.datasource.fluent.data_asset.path.file_asset import ( @@ -23,7 +21,7 @@ class _PandasFilePathDatasource(_PandasDatasource): # class attributes - asset_types: ClassVar[List[Type[DataAsset]]] = list(_FILE_PATH_ASSET_MODELS.values()) + asset_types: ClassVar[list[type[DataAsset]]] = list(_FILE_PATH_ASSET_MODELS.values()) # instance attributes - assets: List[FileDataAsset] = [] + assets: list[FileDataAsset] = [] diff --git a/great_expectations/datasource/fluent/pandas_file_path_datasource.pyi b/great_expectations/datasource/fluent/pandas_file_path_datasource.pyi index 36e26763c6ed..f9595bdcd3c3 100644 --- a/great_expectations/datasource/fluent/pandas_file_path_datasource.pyi +++ b/great_expectations/datasource/fluent/pandas_file_path_datasource.pyi @@ -1,5 +1,5 @@ from logging import Logger -from typing import ClassVar, List, Type +from typing import ClassVar from great_expectations.datasource.fluent.data_asset.path.file_asset import FileDataAsset from great_expectations.datasource.fluent.interfaces import DataAsset as DataAsset @@ -8,5 +8,5 @@ from great_expectations.datasource.fluent.pandas_datasource import _PandasDataso logger: Logger class _PandasFilePathDatasource(_PandasDatasource): - asset_types: ClassVar[List[Type[DataAsset]]] - assets: List[FileDataAsset] + asset_types: ClassVar[list[type[DataAsset]]] + assets: list[FileDataAsset] diff --git a/great_expectations/datasource/fluent/pandas_filesystem_datasource.py b/great_expectations/datasource/fluent/pandas_filesystem_datasource.py index 367528474232..18972802e79f 100644 --- a/great_expectations/datasource/fluent/pandas_filesystem_datasource.py +++ b/great_expectations/datasource/fluent/pandas_filesystem_datasource.py @@ -2,7 +2,7 @@ import logging import pathlib -from typing import TYPE_CHECKING, ClassVar, Literal, Optional, Type +from typing import TYPE_CHECKING, ClassVar, Literal, Optional from great_expectations._docs_decorators import public_api from great_expectations.compatibility.typing_extensions import override @@ -25,7 +25,7 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): """Pandas based Datasource for filesystem based data assets.""" # class attributes - data_connector_type: ClassVar[Type[FilesystemDataConnector]] = FilesystemDataConnector + data_connector_type: ClassVar[type[FilesystemDataConnector]] = FilesystemDataConnector # these fields should not be passed to the execution engine _EXTRA_EXCLUDED_EXEC_ENG_ARGS: ClassVar[set] = { "base_directory", diff --git a/great_expectations/datasource/fluent/pandas_filesystem_datasource.pyi b/great_expectations/datasource/fluent/pandas_filesystem_datasource.pyi index 9aaacf7dd024..b16aa99f1d29 100644 --- a/great_expectations/datasource/fluent/pandas_filesystem_datasource.pyi +++ b/great_expectations/datasource/fluent/pandas_filesystem_datasource.pyi @@ -7,7 +7,6 @@ from typing import ( Literal, Optional, Sequence, - Tuple, Union, ) @@ -121,7 +120,7 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): batch_metadata: Optional[BatchMetadata] = ..., sheet_name: typing.Union[str, int, None] = 0, header: Union[int, Sequence[int], None] = 0, - names: typing.Union[typing.List[str], None] = ..., + names: typing.Union[list[str], None] = ..., index_col: Union[int, Sequence[int], None] = ..., usecols: typing.Union[int, str, typing.Sequence[int], None] = ..., squeeze: typing.Union[bool, None] = ..., @@ -134,7 +133,7 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): keep_default_na: bool = ..., na_filter: bool = ..., verbose: bool = ..., - parse_dates: typing.Union[typing.List, typing.Dict, bool] = ..., + parse_dates: typing.Union[list, dict, bool] = ..., thousands: typing.Union[str, None] = ..., decimal: str = ".", comment: typing.Union[str, None] = ..., @@ -160,7 +159,7 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., connect_options: typing.Mapping = ..., - colspecs: Union[Sequence[Tuple[int, int]], str, None] = ..., + colspecs: Union[Sequence[tuple[int, int]], str, None] = ..., widths: Union[Sequence[int], None] = ..., infer_nrows: int = ..., kwargs: Optional[dict] = ..., @@ -175,10 +174,10 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): key: typing.Any = ..., mode: str = "r", errors: str = "strict", - where: typing.Union[str, typing.List, None] = ..., + where: typing.Union[str, list, None] = ..., start: typing.Union[int, None] = ..., stop: typing.Union[int, None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., iterator: bool = ..., chunksize: typing.Union[int, None] = ..., kwargs: typing.Union[dict, None] = ..., @@ -194,12 +193,12 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): header: Union[int, Sequence[int], None] = ..., index_col: Union[int, Sequence[int], None] = ..., skiprows: typing.Union[typing.Sequence[int], int, None] = ..., - attrs: typing.Union[typing.Dict[str, str], None] = ..., + attrs: typing.Union[dict[str, str], None] = ..., parse_dates: bool = ..., thousands: typing.Union[str, None] = ",", encoding: typing.Union[str, None] = ..., decimal: str = ".", - converters: typing.Union[typing.Dict, None] = ..., + converters: typing.Union[dict, None] = ..., na_values: Union[Iterable[object], None] = ..., keep_default_na: bool = ..., displayed_only: bool = ..., @@ -213,7 +212,7 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): orient: typing.Union[str, None] = ..., dtype: typing.Union[dict, None] = ..., convert_axes: typing.Any = ..., - convert_dates: typing.Union[bool, typing.List[str]] = ..., + convert_dates: typing.Union[bool, list[str]] = ..., keep_default_dates: bool = ..., numpy: bool = ..., precise_float: bool = ..., @@ -232,7 +231,7 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): *, glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., kwargs: typing.Union[dict, None] = ..., ) -> ORCAsset: ... def add_parquet_asset( # noqa: PLR0913 @@ -242,7 +241,7 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., engine: str = "auto", - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., storage_options: StorageOptions = ..., use_nullable_dtypes: bool = ..., kwargs: typing.Union[dict, None] = ..., @@ -303,14 +302,14 @@ class PandasFilesystemDatasource(_PandasFilePathDatasource): glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., xpath: str = "./*", - namespaces: typing.Union[typing.Dict[str, str], None] = ..., + namespaces: typing.Union[dict[str, str], None] = ..., elems_only: bool = ..., attrs_only: bool = ..., names: Union[Sequence[str], None] = ..., dtype: typing.Union[dict, None] = ..., encoding: typing.Union[str, None] = "utf-8", stylesheet: Union[FilePath, None] = ..., - iterparse: typing.Union[typing.Dict[str, typing.List[str]], None] = ..., + iterparse: typing.Union[dict[str, list[str]], None] = ..., compression: CompressionOptions = "infer", storage_options: StorageOptions = ..., ) -> XMLAsset: ... diff --git a/great_expectations/datasource/fluent/pandas_google_cloud_storage_datasource.py b/great_expectations/datasource/fluent/pandas_google_cloud_storage_datasource.py index 65f9cb40d5f6..e5c6c9827d82 100644 --- a/great_expectations/datasource/fluent/pandas_google_cloud_storage_datasource.py +++ b/great_expectations/datasource/fluent/pandas_google_cloud_storage_datasource.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Literal, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Union from great_expectations._docs_decorators import public_api from great_expectations.compatibility import google, pydantic @@ -32,7 +32,7 @@ class PandasGoogleCloudStorageDatasourceError(PandasDatasourceError): @public_api class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[GoogleCloudStorageDataConnector]] = ( + data_connector_type: ClassVar[type[GoogleCloudStorageDataConnector]] = ( GoogleCloudStorageDataConnector ) # these fields should not be passed to the execution engine @@ -47,7 +47,7 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): # Google Cloud Storage specific attributes bucket_or_name: str - gcs_options: Dict[str, Union[ConfigStr, Any]] = {} + gcs_options: dict[str, Union[ConfigStr, Any]] = {} # on 3.11 the annotation must be type-checking import otherwise it will fail at import time _gcs_client: Union[Client, None] = pydantic.PrivateAttr(default=None) diff --git a/great_expectations/datasource/fluent/pandas_google_cloud_storage_datasource.pyi b/great_expectations/datasource/fluent/pandas_google_cloud_storage_datasource.pyi index 5efd854f9f16..b0f157ba1a56 100644 --- a/great_expectations/datasource/fluent/pandas_google_cloud_storage_datasource.pyi +++ b/great_expectations/datasource/fluent/pandas_google_cloud_storage_datasource.pyi @@ -2,13 +2,11 @@ import typing from logging import Logger from typing import ( Any, - Dict, Hashable, Iterable, Literal, Optional, Sequence, - Tuple, Union, ) @@ -65,7 +63,7 @@ class PandasGoogleCloudStorageDatasourceError(PandasDatasourceError): ... class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): type: Literal["pandas_gcs"] bucket_or_name: str - gcs_options: Dict[str, Any] + gcs_options: dict[str, Any] _gcs_client: Union[google.Client, None] @@ -141,7 +139,7 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): gcs_max_results: int = 1000, sheet_name: typing.Union[str, int, None] = 0, header: Union[int, Sequence[int], None] = 0, - names: typing.Union[typing.List[str], None] = ..., + names: typing.Union[list[str], None] = ..., index_col: Union[int, Sequence[int], None] = ..., usecols: typing.Union[int, str, typing.Sequence[int], None] = ..., squeeze: typing.Union[bool, None] = ..., @@ -154,7 +152,7 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): keep_default_na: bool = ..., na_filter: bool = ..., verbose: bool = ..., - parse_dates: typing.Union[typing.List, typing.Dict, bool] = ..., + parse_dates: typing.Union[list, dict, bool] = ..., thousands: typing.Union[str, None] = ..., decimal: str = ".", comment: typing.Union[str, None] = ..., @@ -182,7 +180,7 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., connect_options: typing.Mapping = ..., - colspecs: Union[Sequence[Tuple[int, int]], str, None] = ..., + colspecs: Union[Sequence[tuple[int, int]], str, None] = ..., widths: Union[Sequence[int], None] = ..., infer_nrows: int = ..., kwargs: Optional[dict] = ..., @@ -198,10 +196,10 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): key: typing.Any = ..., mode: str = "r", errors: str = "strict", - where: typing.Union[str, typing.List, None] = ..., + where: typing.Union[str, list, None] = ..., start: typing.Union[int, None] = ..., stop: typing.Union[int, None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., iterator: bool = ..., chunksize: typing.Union[int, None] = ..., kwargs: typing.Union[dict, None] = ..., @@ -219,12 +217,12 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): header: Union[int, Sequence[int], None] = ..., index_col: Union[int, Sequence[int], None] = ..., skiprows: typing.Union[typing.Sequence[int], int, None] = ..., - attrs: typing.Union[typing.Dict[str, str], None] = ..., + attrs: typing.Union[dict[str, str], None] = ..., parse_dates: bool = ..., thousands: typing.Union[str, None] = ",", encoding: typing.Union[str, None] = ..., decimal: str = ".", - converters: typing.Union[typing.Dict, None] = ..., + converters: typing.Union[dict, None] = ..., na_values: Union[Iterable[object], None] = ..., keep_default_na: bool = ..., displayed_only: bool = ..., @@ -240,7 +238,7 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): orient: typing.Union[str, None] = ..., dtype: typing.Union[dict, None] = ..., convert_axes: typing.Any = ..., - convert_dates: typing.Union[bool, typing.List[str]] = ..., + convert_dates: typing.Union[bool, list[str]] = ..., keep_default_dates: bool = ..., numpy: bool = ..., precise_float: bool = ..., @@ -261,7 +259,7 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): gcs_prefix: str = "", gcs_delimiter: str = "/", gcs_max_results: int = 1000, - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., kwargs: typing.Union[dict, None] = ..., ) -> ORCAsset: ... def add_parquet_asset( # noqa: PLR0913 @@ -273,7 +271,7 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): gcs_delimiter: str = "/", gcs_max_results: int = 1000, engine: str = "auto", - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., storage_options: StorageOptions = ..., use_nullable_dtypes: bool = ..., kwargs: typing.Union[dict, None] = ..., @@ -344,14 +342,14 @@ class PandasGoogleCloudStorageDatasource(_PandasFilePathDatasource): gcs_delimiter: str = "/", gcs_max_results: int = 1000, xpath: str = "./*", - namespaces: typing.Union[typing.Dict[str, str], None] = ..., + namespaces: typing.Union[dict[str, str], None] = ..., elems_only: bool = ..., attrs_only: bool = ..., names: Union[Sequence[str], None] = ..., dtype: typing.Union[dict, None] = ..., encoding: typing.Union[str, None] = "utf-8", stylesheet: Union[FilePath, None] = ..., - iterparse: typing.Union[typing.Dict[str, typing.List[str]], None] = ..., + iterparse: typing.Union[dict[str, list[str]], None] = ..., compression: CompressionOptions = "infer", storage_options: StorageOptions = ..., ) -> XMLAsset: ... diff --git a/great_expectations/datasource/fluent/pandas_s3_datasource.py b/great_expectations/datasource/fluent/pandas_s3_datasource.py index 443794a1531b..5f7db0269404 100644 --- a/great_expectations/datasource/fluent/pandas_s3_datasource.py +++ b/great_expectations/datasource/fluent/pandas_s3_datasource.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Literal, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Union from great_expectations._docs_decorators import public_api from great_expectations.compatibility import aws, pydantic @@ -33,7 +33,7 @@ class PandasS3DatasourceError(PandasDatasourceError): @public_api class PandasS3Datasource(_PandasFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[S3DataConnector]] = S3DataConnector + data_connector_type: ClassVar[type[S3DataConnector]] = S3DataConnector # these fields should not be passed to the execution engine _EXTRA_EXCLUDED_EXEC_ENG_ARGS: ClassVar[set] = { "bucket", @@ -45,7 +45,7 @@ class PandasS3Datasource(_PandasFilePathDatasource): # S3 specific attributes bucket: str - boto3_options: Dict[str, Union[ConfigStr, Any]] = {} + boto3_options: dict[str, Union[ConfigStr, Any]] = {} _s3_client: Union[BaseClient, None] = pydantic.PrivateAttr(default=None) diff --git a/great_expectations/datasource/fluent/pandas_s3_datasource.pyi b/great_expectations/datasource/fluent/pandas_s3_datasource.pyi index 3cfa98698f37..ff4d094b7901 100644 --- a/great_expectations/datasource/fluent/pandas_s3_datasource.pyi +++ b/great_expectations/datasource/fluent/pandas_s3_datasource.pyi @@ -2,13 +2,11 @@ import typing from logging import Logger from typing import ( Any, - Dict, Hashable, Iterable, Literal, Optional, Sequence, - Tuple, Union, ) @@ -66,7 +64,7 @@ class PandasS3DatasourceError(PandasDatasourceError): ... class PandasS3Datasource(_PandasFilePathDatasource): type: Literal["pandas_s3"] bucket: str - boto3_options: Dict[str, ConfigStr | Any] + boto3_options: dict[str, ConfigStr | Any] @override def test_connection(self, test_assets: bool = ...) -> None: ... def add_csv_asset( # noqa: PLR0913 @@ -139,7 +137,7 @@ class PandasS3Datasource(_PandasFilePathDatasource): s3_max_keys: int = 1000, sheet_name: typing.Union[str, int, None] = 0, header: Union[int, Sequence[int], None] = 0, - names: typing.Union[typing.List[str], None] = ..., + names: typing.Union[list[str], None] = ..., index_col: Union[int, Sequence[int], None] = ..., usecols: typing.Union[int, str, typing.Sequence[int], None] = ..., squeeze: typing.Union[bool, None] = ..., @@ -152,7 +150,7 @@ class PandasS3Datasource(_PandasFilePathDatasource): keep_default_na: bool = ..., na_filter: bool = ..., verbose: bool = ..., - parse_dates: typing.Union[typing.List, typing.Dict, bool] = ..., + parse_dates: typing.Union[list, dict, bool] = ..., thousands: typing.Union[str, None] = ..., decimal: str = ".", comment: typing.Union[str, None] = ..., @@ -180,7 +178,7 @@ class PandasS3Datasource(_PandasFilePathDatasource): glob_directive: str = ..., batch_metadata: Optional[BatchMetadata] = ..., connect_options: typing.Mapping = ..., - colspecs: Union[Sequence[Tuple[int, int]], str, None] = ..., + colspecs: Union[Sequence[tuple[int, int]], str, None] = ..., widths: Union[Sequence[int], None] = ..., infer_nrows: int = ..., kwargs: Optional[dict] = ..., @@ -196,10 +194,10 @@ class PandasS3Datasource(_PandasFilePathDatasource): key: typing.Any = ..., mode: str = "r", errors: str = "strict", - where: typing.Union[str, typing.List, None] = ..., + where: typing.Union[str, list, None] = ..., start: typing.Union[int, None] = ..., stop: typing.Union[int, None] = ..., - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., iterator: bool = ..., chunksize: typing.Union[int, None] = ..., kwargs: typing.Union[dict, None] = ..., @@ -217,12 +215,12 @@ class PandasS3Datasource(_PandasFilePathDatasource): header: Union[int, Sequence[int], None] = ..., index_col: Union[int, Sequence[int], None] = ..., skiprows: typing.Union[typing.Sequence[int], int, None] = ..., - attrs: typing.Union[typing.Dict[str, str], None] = ..., + attrs: typing.Union[dict[str, str], None] = ..., parse_dates: bool = ..., thousands: typing.Union[str, None] = ",", encoding: typing.Union[str, None] = ..., decimal: str = ".", - converters: typing.Union[typing.Dict, None] = ..., + converters: typing.Union[dict, None] = ..., na_values: Union[Iterable[object], None] = ..., keep_default_na: bool = ..., displayed_only: bool = ..., @@ -238,7 +236,7 @@ class PandasS3Datasource(_PandasFilePathDatasource): orient: typing.Union[str, None] = ..., dtype: typing.Union[dict, None] = ..., convert_axes: typing.Any = ..., - convert_dates: typing.Union[bool, typing.List[str]] = ..., + convert_dates: typing.Union[bool, list[str]] = ..., keep_default_dates: bool = ..., numpy: bool = ..., precise_float: bool = ..., @@ -259,7 +257,7 @@ class PandasS3Datasource(_PandasFilePathDatasource): s3_prefix: str = "", s3_delimiter: str = "/", s3_max_keys: int = 1000, - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., kwargs: typing.Union[dict, None] = ..., ) -> ORCAsset: ... def add_parquet_asset( # noqa: PLR0913 @@ -271,7 +269,7 @@ class PandasS3Datasource(_PandasFilePathDatasource): s3_delimiter: str = "/", s3_max_keys: int = 1000, engine: str = "auto", - columns: typing.Union[typing.List[str], None] = ..., + columns: typing.Union[list[str], None] = ..., storage_options: StorageOptions = ..., use_nullable_dtypes: bool = ..., kwargs: typing.Union[dict, None] = ..., @@ -342,14 +340,14 @@ class PandasS3Datasource(_PandasFilePathDatasource): s3_delimiter: str = "/", s3_max_keys: int = 1000, xpath: str = "./*", - namespaces: typing.Union[typing.Dict[str, str], None] = ..., + namespaces: typing.Union[dict[str, str], None] = ..., elems_only: bool = ..., attrs_only: bool = ..., names: Union[Sequence[str], None] = ..., dtype: typing.Union[dict, None] = ..., encoding: typing.Union[str, None] = "utf-8", stylesheet: Union[FilePath, None] = ..., - iterparse: typing.Union[typing.Dict[str, typing.List[str]], None] = ..., + iterparse: typing.Union[dict[str, list[str]], None] = ..., compression: CompressionOptions = "infer", storage_options: StorageOptions = ..., ) -> XMLAsset: ... diff --git a/great_expectations/datasource/fluent/signatures.py b/great_expectations/datasource/fluent/signatures.py index 82946f721d86..a9b0093ceaa8 100644 --- a/great_expectations/datasource/fluent/signatures.py +++ b/great_expectations/datasource/fluent/signatures.py @@ -2,14 +2,14 @@ import inspect from inspect import Parameter, Signature -from typing import Callable, Type +from typing import Callable def _merge_signatures( target: Callable, source: Callable, exclude: set[str] | None = None, - return_type: Type | None = None, + return_type: type | None = None, ) -> Signature: """ Merge 2 method or function signatures. diff --git a/great_expectations/datasource/fluent/snowflake_datasource.py b/great_expectations/datasource/fluent/snowflake_datasource.py index 8bd3fddc9e12..521aec929189 100644 --- a/great_expectations/datasource/fluent/snowflake_datasource.py +++ b/great_expectations/datasource/fluent/snowflake_datasource.py @@ -14,7 +14,6 @@ Iterable, Literal, Optional, - Type, Union, ) @@ -722,7 +721,7 @@ def get_execution_engine(self) -> SqlAlchemyExecutionEngine: For Snowflake specifically we may represent the connection_string as a dict, which is not supported by SQLAlchemy. """ # noqa: E501 - gx_execution_engine_type: Type[SqlAlchemyExecutionEngine] = self.execution_engine_type + gx_execution_engine_type: type[SqlAlchemyExecutionEngine] = self.execution_engine_type connection_string: str | None = ( self.connection_string if isinstance(self.connection_string, str) else None diff --git a/great_expectations/datasource/fluent/sources.py b/great_expectations/datasource/fluent/sources.py index eea2dc4fed5a..e796647dc491 100644 --- a/great_expectations/datasource/fluent/sources.py +++ b/great_expectations/datasource/fluent/sources.py @@ -10,14 +10,10 @@ Any, Callable, ClassVar, - Dict, Generator, - List, NamedTuple, Optional, Sequence, - Tuple, - Type, Union, ) @@ -55,10 +51,10 @@ class TypeRegistrationError(TypeError): class _FieldDetails(NamedTuple): default_value: Any - type_annotation: Type + type_annotation: type -def _get_field_details(model: Type[pydantic.BaseModel], field_name: str) -> _FieldDetails: +def _get_field_details(model: type[pydantic.BaseModel], field_name: str) -> _FieldDetails: """Get the default value of the requested field and its type annotation.""" return _FieldDetails( default_value=model.__fields__[field_name].default, @@ -73,7 +69,7 @@ class CrudMethodType(str, Enum): ADD_OR_UPDATE = "ADD_OR_UPDATE" -CrudMethodInfoFn: TypeAlias = Callable[..., Tuple[CrudMethodType, Type["Datasource"]]] +CrudMethodInfoFn: TypeAlias = Callable[..., tuple[CrudMethodType, type["Datasource"]]] @public_api @@ -91,7 +87,7 @@ class DataSourceManager: # A dict-like two way mapping between previously registered `Datasource` or `DataAsset` types # and a simplified name for those types. type_lookup: ClassVar = TypeLookup() - __crud_registry: ClassVar[Dict[str, CrudMethodInfoFn]] = {} + __crud_registry: ClassVar[dict[str, CrudMethodInfoFn]] = {} _data_context: GXDataContext @@ -99,7 +95,7 @@ def __init__(self, data_context: GXDataContext): self._data_context = data_context @classmethod - def register_datasource(cls, ds_type: Type[Datasource]) -> None: + def register_datasource(cls, ds_type: type[Datasource]) -> None: """ Add/Register a datasource. This registers all the crud datasource methods: add_, delete_, update_, add_or_update_ @@ -143,7 +139,7 @@ def register_datasource(cls, ds_type: Type[Datasource]) -> None: @classmethod def _register_datasource( cls, - ds_type: Type[Datasource], + ds_type: type[Datasource], ds_type_name: str, datasource_type_lookup: TypeLookup, ) -> str: @@ -167,19 +163,19 @@ def _register_datasource( return ds_type_name @classmethod - def _register_add_datasource(cls, ds_type: Type[Datasource], ds_type_name: str): + def _register_add_datasource(cls, ds_type: type[Datasource], ds_type_name: str): method_name = f"add_{ds_type_name}" - def crud_method_info() -> tuple[CrudMethodType, Type[Datasource]]: + def crud_method_info() -> tuple[CrudMethodType, type[Datasource]]: return CrudMethodType.ADD, ds_type cls._register_crud_method(method_name, cls.__doc__, crud_method_info) @classmethod - def _register_update_datasource(cls, ds_type: Type[Datasource], ds_type_name: str): + def _register_update_datasource(cls, ds_type: type[Datasource], ds_type_name: str): method_name = f"update_{ds_type_name}" - def crud_method_info() -> tuple[CrudMethodType, Type[Datasource]]: + def crud_method_info() -> tuple[CrudMethodType, type[Datasource]]: return CrudMethodType.UPDATE, ds_type cls._register_crud_method( @@ -187,10 +183,10 @@ def crud_method_info() -> tuple[CrudMethodType, Type[Datasource]]: ) @classmethod - def _register_add_or_update_datasource(cls, ds_type: Type[Datasource], ds_type_name: str): + def _register_add_or_update_datasource(cls, ds_type: type[Datasource], ds_type_name: str): method_name = f"add_or_update_{ds_type_name}" - def crud_method_info() -> tuple[CrudMethodType, Type[Datasource]]: + def crud_method_info() -> tuple[CrudMethodType, type[Datasource]]: return CrudMethodType.ADD_OR_UPDATE, ds_type cls._register_crud_method( @@ -200,10 +196,10 @@ def crud_method_info() -> tuple[CrudMethodType, Type[Datasource]]: ) @classmethod - def _register_delete_datasource(cls, ds_type: Type[Datasource], ds_type_name: str): + def _register_delete_datasource(cls, ds_type: type[Datasource], ds_type_name: str): method_name = f"delete_{ds_type_name}" - def crud_method_info() -> tuple[CrudMethodType, Type[Datasource]]: + def crud_method_info() -> tuple[CrudMethodType, type[Datasource]]: return CrudMethodType.DELETE, ds_type cls._register_crud_method( @@ -231,8 +227,8 @@ def _register_crud_method( cls.__crud_registry[crud_fn_name] = crud_method_info @classmethod - def _register_assets(cls, ds_type: Type[Datasource], asset_type_lookup: TypeLookup): - asset_types: Sequence[Type[DataAsset]] = ds_type.asset_types + def _register_assets(cls, ds_type: type[Datasource], asset_type_lookup: TypeLookup): + asset_types: Sequence[type[DataAsset]] = ds_type.asset_types if not asset_types: logger.warning( @@ -265,8 +261,8 @@ def _register_assets(cls, ds_type: Type[Datasource], asset_type_lookup: TypeLook @classmethod def _bind_asset_factory_method_if_not_present( cls, - ds_type: Type[Datasource], - asset_type: Type[DataAsset], + ds_type: type[Datasource], + asset_type: type[DataAsset], asset_type_name: str, ): add_asset_factory_method_name = f"add_{asset_type_name}_asset" @@ -362,11 +358,11 @@ def pandas_default(self) -> PandasDatasource: ) @property - def factories(self) -> List[str]: + def factories(self) -> list[str]: return list(self.__crud_registry.keys()) def _validate_current_datasource_type( - self, name: str, datasource_type: Type[Datasource], raise_if_none: bool = True + self, name: str, datasource_type: type[Datasource], raise_if_none: bool = True ) -> None: try: current_datasource = self._data_context.data_sources.get(name) @@ -382,7 +378,7 @@ def _validate_current_datasource_type( def _datasource_passed_in_as_only_argument( self, - datasource_type: Type[Datasource], + datasource_type: type[Datasource], name_or_datasource: Optional[Union[str, Datasource]], **kwargs, ) -> Optional[Datasource]: @@ -409,7 +405,7 @@ def _datasource_passed_in_as_only_argument( def _datasource_passed_in( self, - datasource_type: Type[Datasource], + datasource_type: type[Datasource], name_or_datasource: Optional[Union[str, Datasource]], **kwargs, ) -> Optional[Datasource]: @@ -447,7 +443,7 @@ def _datasource_passed_in( def create_add_crud_method( self, - datasource_type: Type[Datasource], + datasource_type: type[Datasource], doc_string: str = "", ) -> SourceFactoryFn: def add_datasource( @@ -480,7 +476,7 @@ def add_datasource( def create_update_crud_method( self, - datasource_type: Type[Datasource], + datasource_type: type[Datasource], doc_string: str = "", ) -> SourceFactoryFn: def update_datasource( @@ -527,7 +523,7 @@ def update_datasource( def create_add_or_update_crud_method( self, - datasource_type: Type[Datasource], + datasource_type: type[Datasource], doc_string: str = "", ) -> SourceFactoryFn: def add_or_update_datasource( @@ -577,7 +573,7 @@ def add_or_update_datasource( def create_delete_crud_method( self, - datasource_type: Type[Datasource], + datasource_type: type[Datasource], doc_string: str = "", ) -> Callable[[str], None]: def delete_datasource(name: str) -> None: @@ -635,24 +631,24 @@ def __getattr__(self, attr_name: str): raise AttributeError(f"No crud method '{attr_name}' in {self.factories}") from e # noqa: TRY003 @override - def __dir__(self) -> List[str]: + def __dir__(self) -> list[str]: """Preserves autocompletion for dynamic attributes.""" return [*self.factories, *super().__dir__()] def _iter_all_registered_types( include_datasource: bool = True, include_data_asset: bool = True -) -> Generator[tuple[str, Type[Datasource] | Type[DataAsset]], None, None]: +) -> Generator[tuple[str, type[Datasource | DataAsset]], None, None]: """ Iterate through all registered Datasource and DataAsset types. Returns tuples of the registered type name and the actual type/class. """ for ds_name in DataSourceManager.type_lookup.type_names(): - ds_type: Type[Datasource] = DataSourceManager.type_lookup[ds_name] + ds_type: type[Datasource] = DataSourceManager.type_lookup[ds_name] if include_datasource: yield ds_name, ds_type if include_data_asset: for asset_name in ds_type._type_lookup.type_names(): - asset_type: Type[DataAsset] = ds_type._type_lookup[asset_name] + asset_type: type[DataAsset] = ds_type._type_lookup[asset_name] yield asset_name, asset_type diff --git a/great_expectations/datasource/fluent/sources.pyi b/great_expectations/datasource/fluent/sources.pyi index dc79b3954fa0..1a7f76469559 100644 --- a/great_expectations/datasource/fluent/sources.pyi +++ b/great_expectations/datasource/fluent/sources.pyi @@ -7,10 +7,8 @@ from typing import ( ClassVar, Final, Generator, - List, NamedTuple, Optional, - Type, Union, overload, ) @@ -68,9 +66,9 @@ class TypeRegistrationError(TypeError): ... class _FieldDetails(NamedTuple): default_value: Any - type_annotation: Type + type_annotation: type -def _get_field_details(model: Type[pydantic.BaseModel], field_name: str) -> _FieldDetails: ... +def _get_field_details(model: type[pydantic.BaseModel], field_name: str) -> _FieldDetails: ... class DataSourceManager: type_lookup: ClassVar[TypeLookup] @@ -78,15 +76,15 @@ class DataSourceManager: @classmethod def register_datasource( cls, - ds_type: Type[Datasource], + ds_type: type[Datasource], ) -> None: ... @property def pandas_default(self) -> PandasDatasource: ... @property - def factories(self) -> List[str]: ... + def factories(self) -> list[str]: ... def __getattr__(self, attr_name: str): ... @override - def __dir__(self) -> List[str]: ... + def __dir__(self) -> list[str]: ... def add_pandas( self, name_or_datasource: Optional[Union[str, Datasource]] = None, @@ -741,4 +739,4 @@ class DataSourceManager: def _iter_all_registered_types( include_datasource: bool = True, include_data_asset: bool = True -) -> Generator[tuple[str, Type[Datasource] | Type[DataAsset]], None, None]: ... +) -> Generator[tuple[str, type[Datasource | DataAsset]], None, None]: ... diff --git a/great_expectations/datasource/fluent/spark_azure_blob_storage_datasource.py b/great_expectations/datasource/fluent/spark_azure_blob_storage_datasource.py index a899ca3ee39b..68b47420a1c6 100644 --- a/great_expectations/datasource/fluent/spark_azure_blob_storage_datasource.py +++ b/great_expectations/datasource/fluent/spark_azure_blob_storage_datasource.py @@ -2,7 +2,7 @@ import logging import re -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Final, Literal, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Union from great_expectations._docs_decorators import public_api from great_expectations.compatibility import azure, pydantic @@ -38,7 +38,7 @@ class SparkAzureBlobStorageDatasourceError(SparkDatasourceError): @public_api class SparkAzureBlobStorageDatasource(_SparkFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[AzureBlobStorageDataConnector]] = ( + data_connector_type: ClassVar[type[AzureBlobStorageDataConnector]] = ( AzureBlobStorageDataConnector ) @@ -46,7 +46,7 @@ class SparkAzureBlobStorageDatasource(_SparkFilePathDatasource): type: Literal["spark_abs"] = "spark_abs" # Azure Blob Storage specific attributes - azure_options: Dict[str, Union[ConfigStr, Any]] = {} + azure_options: dict[str, Union[ConfigStr, Any]] = {} _account_name: str = pydantic.PrivateAttr(default="") # on 3.11 the annotation must be type-checking import otherwise it will fail at import time diff --git a/great_expectations/datasource/fluent/spark_azure_blob_storage_datasource.pyi b/great_expectations/datasource/fluent/spark_azure_blob_storage_datasource.pyi index 2a69e1567ada..b6cb4a570554 100644 --- a/great_expectations/datasource/fluent/spark_azure_blob_storage_datasource.pyi +++ b/great_expectations/datasource/fluent/spark_azure_blob_storage_datasource.pyi @@ -1,5 +1,5 @@ from logging import Logger -from typing import Any, ClassVar, Literal, Optional, Type +from typing import Any, ClassVar, Literal, Optional from great_expectations.compatibility import azure from great_expectations.datasource.fluent import _SparkFilePathDatasource @@ -23,7 +23,7 @@ class SparkAzureBlobStorageDatasourceError(SparkDatasourceError): ... class SparkAzureBlobStorageDatasource(_SparkFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[S3DataConnector]] = ... + data_connector_type: ClassVar[type[S3DataConnector]] = ... # instance attributes type: Literal["spark_abs"] = "spark_abs" diff --git a/great_expectations/datasource/fluent/spark_datasource.py b/great_expectations/datasource/fluent/spark_datasource.py index b6d409c4e480..a99f89047573 100644 --- a/great_expectations/datasource/fluent/spark_datasource.py +++ b/great_expectations/datasource/fluent/spark_datasource.py @@ -1,5 +1,6 @@ from __future__ import annotations +import builtins import logging import warnings from pprint import pformat as pf @@ -7,12 +8,9 @@ TYPE_CHECKING, Any, ClassVar, - Dict, Generic, - List, Literal, Optional, - Type, TypeVar, Union, ) @@ -63,7 +61,7 @@ # this enables us to include dataframe in the json schema _SparkDataFrameT = TypeVar("_SparkDataFrameT") -SparkConfig: TypeAlias = Dict[StrictStr, Union[StrictStr, StrictInt, StrictFloat, StrictBool]] +SparkConfig: TypeAlias = dict[StrictStr, Union[StrictStr, StrictInt, StrictFloat, StrictBool]] class SparkDatasourceError(Exception): @@ -102,7 +100,7 @@ def update_forward_refs(cls) -> None: # type: ignore[override] @staticmethod @override - def _update_asset_forward_refs(asset_type: Type[_DataAssetT]) -> None: + def _update_asset_forward_refs(asset_type: type[_DataAssetT]) -> None: # Only update forward refs if pyspark types are available if pyspark: asset_type.update_forward_refs() @@ -110,7 +108,7 @@ def _update_asset_forward_refs(asset_type: Type[_DataAssetT]) -> None: # Abstract Methods @property @override - def execution_engine_type(self) -> Type[SparkDFExecutionEngine]: + def execution_engine_type(self) -> type[SparkDFExecutionEngine]: """Return the SparkDFExecutionEngine unless the override is set""" from great_expectations.execution_engine.sparkdf_execution_engine import ( SparkDFExecutionEngine, @@ -272,7 +270,7 @@ def _validate_batch_request(self, batch_request: BatchRequest) -> None: ) @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: return [IDDict(batch_request.options)] @override @@ -328,12 +326,12 @@ def is_spark_data_frame(df: Any) -> TypeGuard[Union[DataFrame, ConnectDataFrame] @public_api class SparkDatasource(_SparkDatasource): # class attributes - asset_types: ClassVar[List[Type[DataAsset]]] = [DataFrameAsset] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [DataFrameAsset] # instance attributes type: Literal["spark"] = "spark" - assets: List[DataFrameAsset] = [] + assets: list[DataFrameAsset] = [] @public_api def add_dataframe_asset( diff --git a/great_expectations/datasource/fluent/spark_dbfs_datasource.py b/great_expectations/datasource/fluent/spark_dbfs_datasource.py index 43340431f6ee..d57d81a5cac2 100644 --- a/great_expectations/datasource/fluent/spark_dbfs_datasource.py +++ b/great_expectations/datasource/fluent/spark_dbfs_datasource.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, ClassVar, Literal, Type +from typing import TYPE_CHECKING, ClassVar, Literal from great_expectations._docs_decorators import public_api from great_expectations.compatibility.typing_extensions import override @@ -24,7 +24,7 @@ class SparkDBFSDatasource(SparkFilesystemDatasource): """Spark based Datasource for DataBricks File System (DBFS) based data assets.""" # class attributes - data_connector_type: ClassVar[Type[DBFSDataConnector]] = DBFSDataConnector + data_connector_type: ClassVar[type[DBFSDataConnector]] = DBFSDataConnector # instance attributes # overridden from base `Literal['spark_filesystem']` diff --git a/great_expectations/datasource/fluent/spark_file_path_datasource.py b/great_expectations/datasource/fluent/spark_file_path_datasource.py index 54756da5c00a..b7b671912226 100644 --- a/great_expectations/datasource/fluent/spark_file_path_datasource.py +++ b/great_expectations/datasource/fluent/spark_file_path_datasource.py @@ -2,9 +2,7 @@ from typing import ( ClassVar, - List, Sequence, - Type, ) from great_expectations.datasource.fluent import _SparkDatasource @@ -19,7 +17,7 @@ class _SparkFilePathDatasource(_SparkDatasource): # class attributes - asset_types: ClassVar[Sequence[Type[DataAsset]]] = SPARK_PATH_ASSET_TYPES + asset_types: ClassVar[Sequence[type[DataAsset]]] = SPARK_PATH_ASSET_TYPES # instance attributes - assets: List[SPARK_PATH_ASSET_UNION] = [] + assets: list[SPARK_PATH_ASSET_UNION] = [] diff --git a/great_expectations/datasource/fluent/spark_filesystem_datasource.py b/great_expectations/datasource/fluent/spark_filesystem_datasource.py index 025e48d21602..a1c3c5ec1fe3 100644 --- a/great_expectations/datasource/fluent/spark_filesystem_datasource.py +++ b/great_expectations/datasource/fluent/spark_filesystem_datasource.py @@ -2,7 +2,7 @@ import logging import pathlib -from typing import TYPE_CHECKING, ClassVar, Literal, Optional, Type +from typing import TYPE_CHECKING, ClassVar, Literal, Optional from great_expectations._docs_decorators import public_api from great_expectations.compatibility.typing_extensions import override @@ -23,7 +23,7 @@ @public_api class SparkFilesystemDatasource(_SparkFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[FilesystemDataConnector]] = FilesystemDataConnector + data_connector_type: ClassVar[type[FilesystemDataConnector]] = FilesystemDataConnector # these fields should not be passed to the execution engine _EXTRA_EXCLUDED_EXEC_ENG_ARGS: ClassVar[set] = { "base_directory", diff --git a/great_expectations/datasource/fluent/spark_filesystem_datasource.pyi b/great_expectations/datasource/fluent/spark_filesystem_datasource.pyi index 3428fb96807c..b6fac134b4db 100644 --- a/great_expectations/datasource/fluent/spark_filesystem_datasource.pyi +++ b/great_expectations/datasource/fluent/spark_filesystem_datasource.pyi @@ -1,6 +1,6 @@ import pathlib from logging import Logger -from typing import ClassVar, Literal, Optional, Type, Union +from typing import ClassVar, Literal, Optional, Union from great_expectations.compatibility.pyspark import ( types as pyspark_types, @@ -38,7 +38,7 @@ logger: Logger class SparkFilesystemDatasource(_SparkFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[FilesystemDataConnector]] = ... + data_connector_type: ClassVar[type[FilesystemDataConnector]] = ... # instance attributes type: Literal["spark_filesystem"] = "spark_filesystem" diff --git a/great_expectations/datasource/fluent/spark_google_cloud_storage_datasource.py b/great_expectations/datasource/fluent/spark_google_cloud_storage_datasource.py index 3f0230f56a97..3a8adcece3a7 100644 --- a/great_expectations/datasource/fluent/spark_google_cloud_storage_datasource.py +++ b/great_expectations/datasource/fluent/spark_google_cloud_storage_datasource.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Literal, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Union from great_expectations._docs_decorators import public_api from great_expectations.compatibility import google, pydantic @@ -34,7 +34,7 @@ class SparkGoogleCloudStorageDatasourceError(SparkDatasourceError): @public_api class SparkGoogleCloudStorageDatasource(_SparkFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[GoogleCloudStorageDataConnector]] = ( + data_connector_type: ClassVar[type[GoogleCloudStorageDataConnector]] = ( GoogleCloudStorageDataConnector ) # these fields should not be passed to the execution engine @@ -49,7 +49,7 @@ class SparkGoogleCloudStorageDatasource(_SparkFilePathDatasource): # Google Cloud Storage specific attributes bucket_or_name: str - gcs_options: Dict[str, Union[ConfigStr, Any]] = {} + gcs_options: dict[str, Union[ConfigStr, Any]] = {} # on 3.11 the annotation must be type-checking import otherwise it will fail at import time _gcs_client: Union[Client, None] = pydantic.PrivateAttr(default=None) diff --git a/great_expectations/datasource/fluent/spark_google_cloud_storage_datasource.pyi b/great_expectations/datasource/fluent/spark_google_cloud_storage_datasource.pyi index f3cb0f084c9d..c88c14d611d8 100644 --- a/great_expectations/datasource/fluent/spark_google_cloud_storage_datasource.pyi +++ b/great_expectations/datasource/fluent/spark_google_cloud_storage_datasource.pyi @@ -1,5 +1,5 @@ from logging import Logger -from typing import Any, ClassVar, Literal, Optional, Type +from typing import Any, ClassVar, Literal, Optional from great_expectations.compatibility import google from great_expectations.datasource.fluent import BatchMetadata, _SparkFilePathDatasource @@ -15,7 +15,7 @@ logger: Logger class SparkGoogleCloudStorageDatasource(_SparkFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[GoogleCloudStorageDataConnector]] = ... + data_connector_type: ClassVar[type[GoogleCloudStorageDataConnector]] = ... # instance attributes type: Literal["spark_gcs"] = "spark_gcs" diff --git a/great_expectations/datasource/fluent/spark_s3_datasource.py b/great_expectations/datasource/fluent/spark_s3_datasource.py index 491c93e65772..ac13c7c7a925 100644 --- a/great_expectations/datasource/fluent/spark_s3_datasource.py +++ b/great_expectations/datasource/fluent/spark_s3_datasource.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Literal, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Union from great_expectations._docs_decorators import public_api from great_expectations.compatibility import aws, pydantic @@ -35,7 +35,7 @@ class SparkS3DatasourceError(SparkDatasourceError): @public_api class SparkS3Datasource(_SparkFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[S3DataConnector]] = S3DataConnector + data_connector_type: ClassVar[type[S3DataConnector]] = S3DataConnector # these fields should not be passed to the execution engine _EXTRA_EXCLUDED_EXEC_ENG_ARGS: ClassVar[set] = { "bucket", @@ -47,7 +47,7 @@ class SparkS3Datasource(_SparkFilePathDatasource): # S3 specific attributes bucket: str - boto3_options: Dict[str, Union[ConfigStr, Any]] = {} + boto3_options: dict[str, Union[ConfigStr, Any]] = {} _s3_client: Union[BaseClient, None] = pydantic.PrivateAttr(default=None) diff --git a/great_expectations/datasource/fluent/spark_s3_datasource.pyi b/great_expectations/datasource/fluent/spark_s3_datasource.pyi index 5eec2adc1c05..e0e2949c1326 100644 --- a/great_expectations/datasource/fluent/spark_s3_datasource.pyi +++ b/great_expectations/datasource/fluent/spark_s3_datasource.pyi @@ -1,5 +1,5 @@ from logging import Logger -from typing import Any, ClassVar, Literal, Optional, Type +from typing import Any, ClassVar, Literal, Optional from great_expectations.datasource.fluent import BatchMetadata, _SparkFilePathDatasource from great_expectations.datasource.fluent.config_str import ( @@ -14,7 +14,7 @@ logger: Logger class SparkS3Datasource(_SparkFilePathDatasource): # class attributes - data_connector_type: ClassVar[Type[S3DataConnector]] = ... + data_connector_type: ClassVar[type[S3DataConnector]] = ... # instance attributes type: Literal["spark_s3"] = "spark_s3" diff --git a/great_expectations/datasource/fluent/sql_datasource.py b/great_expectations/datasource/fluent/sql_datasource.py index ff9f1843bc0c..fd4e18cc1fe1 100644 --- a/great_expectations/datasource/fluent/sql_datasource.py +++ b/great_expectations/datasource/fluent/sql_datasource.py @@ -1,5 +1,6 @@ from __future__ import annotations +import builtins import copy import logging import warnings @@ -9,16 +10,12 @@ TYPE_CHECKING, Any, ClassVar, - Dict, Final, Generic, - List, Literal, Optional, Protocol, Sequence, - Tuple, - Type, Union, cast, overload, @@ -91,7 +88,7 @@ LOGGER: Final[logging.Logger] = logging.getLogger(__name__) -DEFAULT_QUOTE_CHARACTERS: Final[Tuple[str, str]] = ('"', "'") +DEFAULT_QUOTE_CHARACTERS: Final[tuple[str, str]] = ('"', "'") @overload @@ -155,7 +152,7 @@ def __init__( class _Partitioner(PartitionerProtocol, Protocol): - def param_defaults(self, sql_asset: _SQLAsset) -> List[Dict]: + def param_defaults(self, sql_asset: _SQLAsset) -> list[dict]: """Creates all valid batch requests options for sql_asset This can be implemented by querying the data defined in the sql_asset to generate @@ -201,9 +198,9 @@ def param_defaults(self, sql_asset: _SQLAsset) -> list[dict]: def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """Validates all the datetime parameters for this partitioner exist in `options`.""" - identifiers: Dict = {} + identifiers: dict = {} for part in self.param_names: if part not in options: raise ValueError(f"'{part}' must be specified in the batch parameters") # noqa: TRY003 @@ -214,7 +211,7 @@ def batch_parameters_to_batch_spec_kwarg_identifiers( def param_names(self) -> list[str]: raise NotImplementedError - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: raise NotImplementedError @@ -225,11 +222,11 @@ class SqlPartitionerYear(_PartitionerDatetime): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["year"] @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return {"column_name": self.column_name} @@ -240,11 +237,11 @@ class SqlPartitionerYearAndMonth(_PartitionerDatetime): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["year", "month"] @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return {"column_name": self.column_name} @@ -257,27 +254,27 @@ class SqlPartitionerYearAndMonthAndDay(_PartitionerDatetime): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["year", "month", "day"] @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return {"column_name": self.column_name} class SqlPartitionerDatetimePart(_PartitionerDatetime): - datetime_parts: List[str] + datetime_parts: list[str] column_name: str sort_ascending: bool = True method_name: Literal["partition_on_date_parts"] = "partition_on_date_parts" @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return self.datetime_parts @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return {"column_name": self.column_name, "date_parts": self.param_names} @pydantic.validator("datetime_parts", each_item=True) @@ -311,12 +308,12 @@ def param_defaults(self, sql_asset: _SQLAsset) -> list[dict]: def param_names(self) -> list[str]: raise NotImplementedError - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: raise NotImplementedError def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: raise NotImplementedError @@ -327,17 +324,17 @@ class SqlPartitionerDividedInteger(_PartitionerOneColumnOneParam): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["quotient"] @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return {"column_name": self.column_name, "divisor": self.divisor} @override def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: if "quotient" not in options: raise ValueError("'quotient' must be specified in the batch parameters") # noqa: TRY003 return {self.column_name: options["quotient"]} @@ -350,17 +347,17 @@ class SqlPartitionerModInteger(_PartitionerOneColumnOneParam): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["remainder"] @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return {"column_name": self.column_name, "mod": self.mod} @override def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: if "remainder" not in options: raise ValueError("'remainder' must be specified in the batch parameters") # noqa: TRY003 return {self.column_name: options["remainder"]} @@ -372,17 +369,17 @@ class SqlPartitionerColumnValue(_PartitionerOneColumnOneParam): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return [self.column_name] @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return {"column_name": self.column_name} @override def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: if self.column_name not in options: raise ValueError(f"'{self.column_name}' must be specified in the batch parameters") # noqa: TRY003 return {self.column_name: options[self.column_name]} @@ -397,7 +394,7 @@ def param_defaults(self, sql_asset: _SQLAsset) -> list[dict]: class SqlPartitionerMultiColumnValue(FluentBaseModel): - column_names: List[str] + column_names: list[str] sort_ascending: bool = True method_name: Literal["partition_on_multi_column_values"] = "partition_on_multi_column_values" @@ -406,15 +403,15 @@ def columns(self): return self.column_names @property - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return self.column_names - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return {"column_names": self.column_names} def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: if not (set(self.column_names) <= set(options.keys())): raise ValueError( # noqa: TRY003 f"All column names, {self.column_names}, must be specified in the batch parameters. " # noqa: E501 @@ -446,13 +443,13 @@ class SqlitePartitionerConvertedDateTime(_PartitionerOneColumnOneParam): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: # The datetime parameter will be a string representing a datetime in the format # given by self.date_format_string. return ["datetime"] @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return { "column_name": self.column_name, "date_format_string": self.date_format_string, @@ -461,7 +458,7 @@ def partitioner_method_kwargs(self) -> Dict[str, Any]: @override def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: if "datetime" not in options: raise ValueError( # noqa: TRY003 "'datetime' must be specified in the batch parameters to create a batch identifier" @@ -498,8 +495,8 @@ class _SQLAsset(DataAsset[DatasourceT, ColumnPartitioner], Generic[DatasourceT]) # Instance fields type: str = pydantic.Field("_sql_asset") name: str - _partitioner_implementation_map: Dict[ - Type[ColumnPartitioner], Optional[Type[SqlPartitioner]] + _partitioner_implementation_map: dict[ + type[ColumnPartitioner], Optional[type[SqlPartitioner]] ] = pydantic.PrivateAttr( default={ ColumnPartitionerYearly: SqlPartitionerYear, @@ -530,7 +527,7 @@ def get_batch_parameters_keys( self, partitioner: Optional[ColumnPartitioner] = None, ) -> tuple[str, ...]: - option_keys: Tuple[str, ...] = tuple() + option_keys: tuple[str, ...] = tuple() if partitioner: sql_partitioner = self.get_partitioner_implementation(partitioner) option_keys += tuple(sql_partitioner.param_names) @@ -541,7 +538,7 @@ def test_connection(self) -> None: pass @staticmethod - def _matches_request_options(candidate: Dict, requested_options: BatchParameters) -> bool: + def _matches_request_options(candidate: dict, requested_options: BatchParameters) -> bool: for k, v in requested_options.items(): if isinstance(candidate[k], (datetime, date)): candidate[k] = str(candidate[k]) @@ -550,7 +547,7 @@ def _matches_request_options(candidate: Dict, requested_options: BatchParameters return False return True - def _fully_specified_batch_requests(self, batch_request: BatchRequest) -> List[BatchRequest]: + def _fully_specified_batch_requests(self, batch_request: BatchRequest) -> list[BatchRequest]: """Populates a batch requests unspecified params producing a list of batch requests.""" if batch_request.partitioner is None: @@ -564,7 +561,7 @@ def _fully_specified_batch_requests(self, batch_request: BatchRequest) -> List[B sql_partitioner = self.get_partitioner_implementation(batch_request.partitioner) - batch_requests: List[BatchRequest] = [] + batch_requests: list[BatchRequest] = [] # We iterate through all possible batches as determined by the partitioner for params in sql_partitioner.param_defaults(self): # If the params from the partitioner don't match the batch parameters @@ -584,7 +581,7 @@ def _fully_specified_batch_requests(self, batch_request: BatchRequest) -> List[B return batch_requests @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: self._validate_batch_request(batch_request) if batch_request.partitioner: sql_partitioner = self.get_partitioner_implementation(batch_request.partitioner) @@ -645,7 +642,7 @@ def get_batch(self, batch_request: BatchRequest) -> Batch: batch_spec_kwargs["partitioner_kwargs"] = sql_partitioner.partitioner_method_kwargs() # mypy infers that batch_spec_kwargs["batch_identifiers"] is a collection, but # it is hardcoded to a dict above, so we cast it here. - cast(Dict, batch_spec_kwargs["batch_identifiers"]).update( + cast(dict, batch_spec_kwargs["batch_identifiers"]).update( sql_partitioner.batch_parameters_to_batch_spec_kwarg_identifiers(request.options) ) # Creating the batch_spec is our hook into the execution engine. @@ -1031,21 +1028,21 @@ class SQLDatasource(Datasource): """ # class var definitions - asset_types: ClassVar[List[Type[DataAsset]]] = [TableAsset, QueryAsset] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [TableAsset, QueryAsset] # right side of the operator determines the type name # left side enforces the names on instance creation type: Literal["sql"] = "sql" connection_string: Union[ConfigStr, str] create_temp_table: bool = False - kwargs: Dict[str, Union[ConfigStr, Any]] = pydantic.Field( + kwargs: dict[str, Union[ConfigStr, Any]] = pydantic.Field( default={}, description="Optional dictionary of `kwargs` will be passed to the SQLAlchemy Engine" " as part of `create_engine(connection_string, **kwargs)`", ) # We need to explicitly add each asset type to the Union due to how # deserialization is implemented in our pydantic base model. - assets: List[AssetTypes] = [] + assets: list[AssetTypes] = [] # private attrs _cached_connection_string: Union[str, ConfigStr] = pydantic.PrivateAttr("") @@ -1053,12 +1050,12 @@ class SQLDatasource(Datasource): # These are instance var because ClassVars can't contain Type variables. See # https://peps.python.org/pep-0526/#class-and-instance-variable-annotations - _TableAsset: Type[TableAsset] = pydantic.PrivateAttr(TableAsset) - _QueryAsset: Type[QueryAsset] = pydantic.PrivateAttr(QueryAsset) + _TableAsset: builtins.type[TableAsset] = pydantic.PrivateAttr(TableAsset) + _QueryAsset: builtins.type[QueryAsset] = pydantic.PrivateAttr(QueryAsset) @property @override - def execution_engine_type(self) -> Type[SqlAlchemyExecutionEngine]: + def execution_engine_type(self) -> type[SqlAlchemyExecutionEngine]: """Returns the default execution engine type.""" return SqlAlchemyExecutionEngine diff --git a/great_expectations/datasource/fluent/sqlite_datasource.py b/great_expectations/datasource/fluent/sqlite_datasource.py index 9f98119ab466..dc31a282c818 100644 --- a/great_expectations/datasource/fluent/sqlite_datasource.py +++ b/great_expectations/datasource/fluent/sqlite_datasource.py @@ -1,14 +1,12 @@ from __future__ import annotations +import builtins from typing import ( TYPE_CHECKING, Any, ClassVar, - Dict, - List, Literal, Optional, - Type, Union, cast, ) @@ -66,13 +64,13 @@ class PartitionerConvertedDateTime(_PartitionerOneColumnOneParam): @property @override - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: # The datetime parameter will be a string representing a datetime in the format # given by self.date_format_string. return ["datetime"] @override - def partitioner_method_kwargs(self) -> Dict[str, Any]: + def partitioner_method_kwargs(self) -> dict[str, Any]: return { "column_name": self.column_name, "date_format_string": self.date_format_string, @@ -81,7 +79,7 @@ def partitioner_method_kwargs(self) -> Dict[str, Any]: @override def batch_parameters_to_batch_spec_kwarg_identifiers( self, options: BatchParameters - ) -> Dict[str, Any]: + ) -> dict[str, Any]: if "datetime" not in options: raise ValueError( # noqa: TRY003 "'datetime' must be specified in the batch parameters to create a batch identifier" @@ -135,7 +133,7 @@ class SqliteDatasource(SQLDatasource): """ # class var definitions - asset_types: ClassVar[List[Type[DataAsset]]] = [SqliteTableAsset, SqliteQueryAsset] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [SqliteTableAsset, SqliteQueryAsset] # Subclass instance var overrides # right side of the operator determines the type name @@ -143,8 +141,8 @@ class SqliteDatasource(SQLDatasource): type: Literal["sqlite"] = "sqlite" # type: ignore[assignment] connection_string: Union[ConfigStr, SqliteDsn] - _TableAsset: Type[SqlTableAsset] = pydantic.PrivateAttr(SqliteTableAsset) - _QueryAsset: Type[SqlQueryAsset] = pydantic.PrivateAttr(SqliteQueryAsset) + _TableAsset: builtins.type[SqlTableAsset] = pydantic.PrivateAttr(SqliteTableAsset) + _QueryAsset: builtins.type[SqlQueryAsset] = pydantic.PrivateAttr(SqliteQueryAsset) @public_api @override diff --git a/great_expectations/datasource/fluent/type_lookup.py b/great_expectations/datasource/fluent/type_lookup.py index 288eec1949c4..9f3fa4122264 100644 --- a/great_expectations/datasource/fluent/type_lookup.py +++ b/great_expectations/datasource/fluent/type_lookup.py @@ -11,8 +11,6 @@ Iterable, Mapping, Optional, - Set, - Type, Union, overload, ) @@ -29,7 +27,7 @@ class TypeLookupError(ValueError): pass -ValidTypes: TypeAlias = Union[str, Type] +ValidTypes: TypeAlias = Union[str, type] class TypeLookup( @@ -63,10 +61,10 @@ def type_names(self) -> Generator[str, None, None]: continue @overload - def __getitem__(self, key: str) -> Type: ... + def __getitem__(self, key: str) -> type: ... @overload - def __getitem__(self, key: Type) -> str: ... + def __getitem__(self, key: type) -> str: ... @override def __getitem__(self, key: ValidTypes) -> ValidTypes: @@ -107,7 +105,7 @@ def __repr__(self) -> str: def __str__(self) -> str: return str(self.data) - def intersection(self, collection_: Iterable[ValidTypes]) -> Set[ValidTypes]: + def intersection(self, collection_: Iterable[ValidTypes]) -> set[ValidTypes]: """ Returns the intersection of a list (or other iterable) and the keys/values of the `TypeLookup` instance. diff --git a/great_expectations/exceptions/exceptions.py b/great_expectations/exceptions/exceptions.py index 589caefa28e4..208f9c4dc8d5 100644 --- a/great_expectations/exceptions/exceptions.py +++ b/great_expectations/exceptions/exceptions.py @@ -4,7 +4,7 @@ import itertools import json from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from marshmallow import ValidationError @@ -33,7 +33,7 @@ def errors(self) -> list[GreatExpectationsError]: class GreatExpectationsValidationError(ValidationError, GreatExpectationsError): def __init__(self, message, validation_error=None) -> None: self.message = message - self.messages: Union[List[str], List[Any], Dict] = [] + self.messages: Union[list[str], list[Any], dict] = [] if validation_error is not None: self.messages = validation_error.messages diff --git a/great_expectations/execution_engine/execution_engine.py b/great_expectations/execution_engine/execution_engine.py index 87f0151a3da3..9c4e2a15634b 100644 --- a/great_expectations/execution_engine/execution_engine.py +++ b/great_expectations/execution_engine/execution_engine.py @@ -8,12 +8,8 @@ TYPE_CHECKING, Any, Callable, - Dict, Iterable, - List, Optional, - Set, - Tuple, Union, ) @@ -136,7 +132,7 @@ class ExecutionEngine(ABC): validator: Validator object (optional) -- not utilized in V3 and later versions """ # noqa: E501 - recognized_batch_spec_defaults: Set[str] = set() + recognized_batch_spec_defaults: set[str] = set() def __init__( self, @@ -154,7 +150,7 @@ def __init__( self._caching = caching # NOTE: 20200918 - this is a naive cache; update. if self._caching: - self._metric_cache: Union[Dict, NoOpDict] = {} + self._metric_cache: Union[dict, NoOpDict] = {} else: self._metric_cache = NoOpDict() @@ -214,7 +210,7 @@ def batch_manager(self) -> BatchManager: """Getter for batch_manager""" return self._batch_manager - def _load_batch_data_from_dict(self, batch_data_dict: Dict[str, BatchDataType]) -> None: + def _load_batch_data_from_dict(self, batch_data_dict: dict[str, BatchDataType]) -> None: """ Loads all data in batch_data_dict using cache_batch_data """ @@ -243,15 +239,15 @@ def get_batch_data( return batch_data @abstractmethod - def get_batch_data_and_markers(self, batch_spec) -> Tuple[BatchData, BatchMarkers]: + def get_batch_data_and_markers(self, batch_spec) -> tuple[BatchData, BatchMarkers]: raise NotImplementedError def resolve_metrics( self, metrics_to_resolve: Iterable[MetricConfiguration], - metrics: Optional[Dict[Tuple[str, str, str], MetricValue]] = None, + metrics: Optional[dict[tuple[str, str, str], MetricValue]] = None, runtime_configuration: Optional[dict] = None, - ) -> Dict[Tuple[str, str, str], MetricValue]: + ) -> dict[tuple[str, str, str], MetricValue]: """resolve_metrics is the main entrypoint for an execution engine. The execution engine will compute the value of the provided metrics. @@ -266,8 +262,8 @@ def resolve_metrics( if not metrics_to_resolve: return metrics or {} - metric_fn_direct_configurations: List[MetricComputationConfiguration] - metric_fn_bundle_configurations: List[MetricComputationConfiguration] + metric_fn_direct_configurations: list[MetricComputationConfiguration] + metric_fn_bundle_configurations: list[MetricComputationConfiguration] ( metric_fn_direct_configurations, metric_fn_bundle_configurations, @@ -281,7 +277,7 @@ def resolve_metrics( metric_fn_bundle_configurations=metric_fn_bundle_configurations, ) - def resolve_metric_bundle(self, metric_fn_bundle) -> Dict[Tuple[str, str, str], MetricValue]: + def resolve_metric_bundle(self, metric_fn_bundle) -> dict[tuple[str, str, str], MetricValue]: """Resolve a bundle of metrics with the same compute Domain as part of a single trip to the compute engine.""" # noqa: E501 raise NotImplementedError @@ -305,7 +301,7 @@ def get_compute_domain( domain_kwargs: dict, domain_type: Union[str, MetricDomainTypes], accessor_keys: Optional[Iterable[str]] = None, - ) -> Tuple[Any, dict, dict]: + ) -> tuple[Any, dict, dict]: """get_compute_domain() is an interface method, which computes the optimal domain_kwargs for computing metrics based on the given domain_kwargs and specific engine semantics. Args: @@ -372,11 +368,11 @@ def add_column_row_condition( def _build_direct_and_bundled_metric_computation_configurations( self, metrics_to_resolve: Iterable[MetricConfiguration], - metrics: Optional[Dict[Tuple[str, str, str], MetricValue]] = None, + metrics: Optional[dict[tuple[str, str, str], MetricValue]] = None, runtime_configuration: Optional[dict] = None, - ) -> Tuple[ - List[MetricComputationConfiguration], - List[MetricComputationConfiguration], + ) -> tuple[ + list[MetricComputationConfiguration], + list[MetricComputationConfiguration], ]: """ This method organizes "metrics_to_resolve" ("MetricConfiguration" objects) into two lists: direct and bundled. @@ -393,8 +389,8 @@ def _build_direct_and_bundled_metric_computation_configurations( Returns: Tuple with two elements: directly-computable and bundled "MetricComputationConfiguration" objects """ # noqa: E501 - metric_fn_direct_configurations: List[MetricComputationConfiguration] = [] - metric_fn_bundle_configurations: List[MetricComputationConfiguration] = [] + metric_fn_direct_configurations: list[MetricComputationConfiguration] = [] + metric_fn_bundle_configurations: list[MetricComputationConfiguration] = [] if not metrics_to_resolve: return ( @@ -405,8 +401,8 @@ def _build_direct_and_bundled_metric_computation_configurations( if metrics is None: metrics = {} - resolved_metric_dependencies_by_metric_name: Dict[ - str, Union[MetricValue, Tuple[Any, dict, dict]] + resolved_metric_dependencies_by_metric_name: dict[ + str, Union[MetricValue, tuple[Any, dict, dict]] ] metric_class: MetricProvider metric_fn: Union[Callable, None] @@ -471,8 +467,8 @@ def _build_direct_and_bundled_metric_computation_configurations( def _get_computed_metric_evaluation_dependencies_by_metric_name( self, metric_to_resolve: MetricConfiguration, - metrics: Dict[Tuple[str, str, str], MetricValue], - ) -> Dict[str, Union[MetricValue, Tuple[Any, dict, dict]]]: + metrics: dict[tuple[str, str, str], MetricValue], + ) -> dict[str, Union[MetricValue, tuple[Any, dict, dict]]]: """ Gathers resolved (already computed) evaluation dependencies of metric-to-resolve (not yet computed) "MetricConfiguration" object by "metric_name" property of resolved "MetricConfiguration" objects. @@ -484,8 +480,8 @@ def _get_computed_metric_evaluation_dependencies_by_metric_name( Returns: Dictionary keyed by "metric_name" with values as computed metric or partial bundling information tuple """ # noqa: E501 - metric_dependencies_by_metric_name: Dict[ - str, Union[MetricValue, Tuple[Any, dict, dict]] + metric_dependencies_by_metric_name: dict[ + str, Union[MetricValue, tuple[Any, dict, dict]] ] = {} metric_name: str @@ -509,9 +505,9 @@ def _get_computed_metric_evaluation_dependencies_by_metric_name( def _process_direct_and_bundled_metric_computation_configurations( self, - metric_fn_direct_configurations: List[MetricComputationConfiguration], - metric_fn_bundle_configurations: List[MetricComputationConfiguration], - ) -> Dict[Tuple[str, str, str], MetricValue]: + metric_fn_direct_configurations: list[MetricComputationConfiguration], + metric_fn_bundle_configurations: list[MetricComputationConfiguration], + ) -> dict[tuple[str, str, str], MetricValue]: """ This method processes directly-computable and bundled "MetricComputationConfiguration" objects. @@ -522,7 +518,7 @@ def _process_direct_and_bundled_metric_computation_configurations( Returns: resolved_metrics (Dict): a dictionary with the values for the metrics that have just been resolved. """ # noqa: E501 - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] = {} + resolved_metrics: dict[tuple[str, str, str], MetricValue] = {} metric_computation_configuration: MetricComputationConfiguration @@ -541,7 +537,7 @@ def _process_direct_and_bundled_metric_computation_configurations( try: # an engine-specific way of computing metrics together - resolved_metric_bundle: Dict[Tuple[str, str, str], MetricValue] = ( + resolved_metric_bundle: dict[tuple[str, str, str], MetricValue] = ( self.resolve_metric_bundle(metric_fn_bundle=metric_fn_bundle_configurations) ) resolved_metrics.update(resolved_metric_bundle) @@ -561,7 +557,7 @@ def _process_direct_and_bundled_metric_computation_configurations( def _partition_domain_kwargs( self, - domain_kwargs: Dict[str, Any], + domain_kwargs: dict[str, Any], domain_type: Union[str, MetricDomainTypes], accessor_keys: Optional[Iterable[str]] = None, ) -> PartitionDomainKwargs: @@ -616,7 +612,7 @@ class MetricDomainTypes. ) else: compute_domain_kwargs = copy.deepcopy(domain_kwargs) - accessor_domain_kwargs: Dict[str, Any] = {} + accessor_domain_kwargs: dict[str, Any] = {} partition_domain_kwargs = PartitionDomainKwargs( compute_domain_kwargs, accessor_domain_kwargs ) @@ -646,8 +642,8 @@ def _partition_table_metric_domain_kwargs( domain_type == MetricDomainTypes.TABLE ), "This method only supports MetricDomainTypes.TABLE" - compute_domain_kwargs: Dict = copy.deepcopy(domain_kwargs) - accessor_domain_kwargs: Dict = {} + compute_domain_kwargs: dict = copy.deepcopy(domain_kwargs) + accessor_domain_kwargs: dict = {} if accessor_keys is not None and len(list(accessor_keys)) > 0: for key in accessor_keys: @@ -692,8 +688,8 @@ def _partition_column_metric_domain_kwargs( domain_type == MetricDomainTypes.COLUMN ), "This method only supports MetricDomainTypes.COLUMN" - compute_domain_kwargs: Dict = copy.deepcopy(domain_kwargs) - accessor_domain_kwargs: Dict = {} + compute_domain_kwargs: dict = copy.deepcopy(domain_kwargs) + accessor_domain_kwargs: dict = {} if "column" not in compute_domain_kwargs: raise gx_exceptions.GreatExpectationsError( # noqa: TRY003 @@ -724,8 +720,8 @@ def _partition_column_pair_metric_domain_kwargs( domain_type == MetricDomainTypes.COLUMN_PAIR ), "This method only supports MetricDomainTypes.COLUMN_PAIR" - compute_domain_kwargs: Dict = copy.deepcopy(domain_kwargs) - accessor_domain_kwargs: Dict = {} + compute_domain_kwargs: dict = copy.deepcopy(domain_kwargs) + accessor_domain_kwargs: dict = {} if not ("column_A" in domain_kwargs and "column_B" in domain_kwargs): raise gx_exceptions.GreatExpectationsError( # noqa: TRY003 @@ -757,8 +753,8 @@ def _partition_multi_column_metric_domain_kwargs( domain_type == MetricDomainTypes.MULTICOLUMN ), "This method only supports MetricDomainTypes.MULTICOLUMN" - compute_domain_kwargs: Dict = copy.deepcopy(domain_kwargs) - accessor_domain_kwargs: Dict = {} + compute_domain_kwargs: dict = copy.deepcopy(domain_kwargs) + accessor_domain_kwargs: dict = {} if "column_list" not in domain_kwargs: raise gx_exceptions.GreatExpectationsError("column_list not found within domain_kwargs") # noqa: TRY003 diff --git a/great_expectations/execution_engine/pandas_execution_engine.py b/great_expectations/execution_engine/pandas_execution_engine.py index df8a9e5a17dc..11579a6c4bfd 100644 --- a/great_expectations/execution_engine/pandas_execution_engine.py +++ b/great_expectations/execution_engine/pandas_execution_engine.py @@ -10,10 +10,8 @@ TYPE_CHECKING, Any, Callable, - Dict, Iterable, Optional, - Tuple, Union, cast, overload, @@ -111,9 +109,9 @@ def __init__(self, *args, **kwargs) -> None: self.discard_subset_failing_expectations = kwargs.pop( "discard_subset_failing_expectations", False ) - boto3_options: Dict[str, dict] = kwargs.pop("boto3_options", {}) - azure_options: Dict[str, dict] = kwargs.pop("azure_options", {}) - gcs_options: Dict[str, dict] = kwargs.pop("gcs_options", {}) + boto3_options: dict[str, dict] = kwargs.pop("boto3_options", {}) + azure_options: dict[str, dict] = kwargs.pop("azure_options", {}) + gcs_options: dict[str, dict] = kwargs.pop("gcs_options", {}) # Instantiate cloud provider clients as None at first. # They will be instantiated if/when passed cloud-specific in BatchSpec is passed in @@ -204,7 +202,7 @@ def load_batch_data( @override def get_batch_data_and_markers( # noqa: C901, PLR0912, PLR0915 self, batch_spec: BatchSpec | PandasBatchSpecProtocol - ) -> Tuple[PandasBatchData, BatchMarkers]: # batch_data + ) -> tuple[PandasBatchData, BatchMarkers]: # batch_data # We need to build a batch_markers to be used in the dataframe batch_markers = BatchMarkers( { @@ -479,7 +477,7 @@ def _get_reader_fn( ) @override - def resolve_metric_bundle(self, metric_fn_bundle) -> Dict[Tuple[str, str, str], Any]: + def resolve_metric_bundle(self, metric_fn_bundle) -> dict[tuple[str, str, str], Any]: """Resolve a bundle of metrics with the same compute Domain as part of a single trip to the compute engine.""" # noqa: E501 return {} # This is NO-OP for "PandasExecutionEngine" (no bundling for direct execution computational backend). # noqa: E501 @@ -595,7 +593,7 @@ def get_compute_domain( domain_kwargs: dict, domain_type: Union[str, MetricDomainTypes], accessor_keys: Optional[Iterable[str]] = None, - ) -> Tuple[pd.DataFrame, dict, dict]: + ) -> tuple[pd.DataFrame, dict, dict]: """Uses the given Domain kwargs (which include row_condition, condition_parser, and ignore_row_if directives) to obtain and/or query a batch. Returns in the format of a Pandas DataFrame along with Domain arguments required for computing. If the Domain \ diff --git a/great_expectations/execution_engine/partition_and_sample/data_partitioner.py b/great_expectations/execution_engine/partition_and_sample/data_partitioner.py index b54f24a00605..08b31cc5906b 100644 --- a/great_expectations/execution_engine/partition_and_sample/data_partitioner.py +++ b/great_expectations/execution_engine/partition_and_sample/data_partitioner.py @@ -3,7 +3,7 @@ import abc import datetime import enum -from typing import Callable, ClassVar, List, Type +from typing import Callable, ClassVar import ruamel from dateutil.parser import parse @@ -80,7 +80,7 @@ class DataPartitioner(abc.ABC): # noqa: B024 date_part e.g. DataPartitioner.date_part.MONTH """ # noqa: E501 - date_part: ClassVar[Type[DatePart]] = DatePart + date_part: ClassVar[type[DatePart]] = DatePart def get_partitioner_method(self, partitioner_method_name: str) -> Callable: """Get the appropriate partitioner method from the method name. @@ -111,7 +111,7 @@ def _get_partitioner_method_name(partitioner_method_name: str) -> str: return partitioner_method_name @staticmethod - def _convert_date_parts(date_parts: List[DatePart] | List[str]) -> List[DatePart]: + def _convert_date_parts(date_parts: list[DatePart] | list[str]) -> list[DatePart]: """Convert a list of date parts to DatePart objects. Args: @@ -126,7 +126,7 @@ def _convert_date_parts(date_parts: List[DatePart] | List[str]) -> List[DatePart ] @staticmethod - def _validate_date_parts(date_parts: List[DatePart] | List[str]) -> None: + def _validate_date_parts(date_parts: list[DatePart] | list[str]) -> None: """Validate that date parts exist and are of the correct type. Args: @@ -143,7 +143,7 @@ def _validate_date_parts(date_parts: List[DatePart] | List[str]) -> None: raise gx_exceptions.InvalidConfigError("date_parts should be of type DatePart or str.") # noqa: TRY003 @staticmethod - def _verify_all_strings_are_valid_date_parts(date_part_strings: List[str]) -> None: + def _verify_all_strings_are_valid_date_parts(date_part_strings: list[str]) -> None: """Verify date part strings by trying to load as DatePart instances. Args: @@ -162,7 +162,7 @@ def _verify_all_strings_are_valid_date_parts(date_part_strings: List[str]) -> No def _convert_datetime_batch_identifiers_to_date_parts_dict( self, column_batch_identifiers: datetime.datetime | str | dict, - date_parts: List[DatePart], + date_parts: list[DatePart], ) -> dict: """Convert batch identifiers to a dict of {date_part as str: date_part value}. diff --git a/great_expectations/execution_engine/partition_and_sample/pandas_data_partitioner.py b/great_expectations/execution_engine/partition_and_sample/pandas_data_partitioner.py index 08d00e442a42..06566250d44c 100644 --- a/great_expectations/execution_engine/partition_and_sample/pandas_data_partitioner.py +++ b/great_expectations/execution_engine/partition_and_sample/pandas_data_partitioner.py @@ -1,7 +1,7 @@ from __future__ import annotations import hashlib -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, Union import great_expectations.exceptions as gx_exceptions from great_expectations.execution_engine.partition_and_sample.data_partitioner import ( @@ -103,7 +103,7 @@ def partition_on_date_parts( df: pd.DataFrame, column_name: str, batch_identifiers: dict, - date_parts: Union[List[DatePart], List[str]], + date_parts: Union[list[DatePart], list[str]], ) -> pd.DataFrame: """Partition on date_part values in column_name. @@ -203,7 +203,7 @@ def partition_on_mod_integer( @staticmethod def partition_on_multi_column_values( - df, column_names: List[str], batch_identifiers: dict + df, column_names: list[str], batch_identifiers: dict ) -> pd.DataFrame: """Partition on the joint values in the named columns""" diff --git a/great_expectations/execution_engine/partition_and_sample/sparkdf_data_partitioner.py b/great_expectations/execution_engine/partition_and_sample/sparkdf_data_partitioner.py index ffe6252e5f16..f4072fa92685 100644 --- a/great_expectations/execution_engine/partition_and_sample/sparkdf_data_partitioner.py +++ b/great_expectations/execution_engine/partition_and_sample/sparkdf_data_partitioner.py @@ -2,7 +2,7 @@ import hashlib import logging -from typing import List, Union +from typing import Union from great_expectations.compatibility import pyspark from great_expectations.compatibility.pyspark import functions as F @@ -105,7 +105,7 @@ def partition_on_date_parts( df: pyspark.DataFrame, column_name: str, batch_identifiers: dict, - date_parts: Union[List[DatePart], List[str]], + date_parts: Union[list[DatePart], list[str]], ) -> pyspark.DataFrame: """Partition on date_part values in column_name. diff --git a/great_expectations/execution_engine/partition_and_sample/sqlalchemy_data_partitioner.py b/great_expectations/execution_engine/partition_and_sample/sqlalchemy_data_partitioner.py index 572f4e2f3921..077e0d002b50 100644 --- a/great_expectations/execution_engine/partition_and_sample/sqlalchemy_data_partitioner.py +++ b/great_expectations/execution_engine/partition_and_sample/sqlalchemy_data_partitioner.py @@ -16,7 +16,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, Union import great_expectations.exceptions as gx_exceptions from great_expectations.compatibility.sqlalchemy import ( @@ -136,7 +136,7 @@ def partition_on_date_parts( self, column_name: str, batch_identifiers: dict, - date_parts: Union[List[DatePart], List[str]], + date_parts: Union[list[DatePart], list[str]], ) -> Union[sqlalchemy.BinaryExpression, sqlalchemy.BooleanClauseList]: """Partition on date_part values in column_name. @@ -279,7 +279,7 @@ def partition_on_mod_integer( @staticmethod def partition_on_multi_column_values( - column_names: List[str], + column_names: list[str], batch_identifiers: dict, ) -> bool: """Partition on the joint values in the named columns""" @@ -314,7 +314,7 @@ def get_data_for_batch_identifiers( selectable: sqlalchemy.Selectable, partitioner_method_name: str, partitioner_kwargs: dict, - ) -> List[dict]: + ) -> list[dict]: """Build data used to construct batch identifiers for the input table using the provided partitioner config. Sql partitioner configurations yield the unique values that comprise a batch by introspecting your data. @@ -331,7 +331,7 @@ def get_data_for_batch_identifiers( processed_partitioner_method_name: str = self._get_partitioner_method_name( partitioner_method_name ) - batch_identifiers_list: List[dict] + batch_identifiers_list: list[dict] if self._is_datetime_partitioner(processed_partitioner_method_name): partitioner_fn_name: str = ( self.DATETIME_PARTITIONER_METHOD_TO_GET_UNIQUE_BATCH_IDENTIFIERS_METHOD_MAPPING[ @@ -371,7 +371,7 @@ def get_data_for_batch_identifiers_year( execution_engine: SqlAlchemyExecutionEngine, selectable: sqlalchemy.Selectable, column_name: str, - ) -> List[dict]: + ) -> list[dict]: """Build batch_identifiers from a column partition on year. This method builds a query to select the unique date_parts from the @@ -397,7 +397,7 @@ def get_data_for_batch_identifiers_year_and_month( execution_engine: SqlAlchemyExecutionEngine, selectable: sqlalchemy.Selectable, column_name: str, - ) -> List[dict]: + ) -> list[dict]: """Build batch_identifiers from a column partition on year and month. This method builds a query to select the unique date_parts from the @@ -423,7 +423,7 @@ def get_data_for_batch_identifiers_year_and_month_and_day( execution_engine: SqlAlchemyExecutionEngine, selectable: sqlalchemy.Selectable, column_name: str, - ) -> List[dict]: + ) -> list[dict]: """Build batch_identifiers from a column partition on year and month and day. This method builds a query to select the unique date_parts from the @@ -448,7 +448,7 @@ def get_partition_query_for_data_for_batch_identifiers_for_partition_on_date_par self, selectable: sqlalchemy.Selectable, column_name: str, - date_parts: Union[List[DatePart], List[str]], + date_parts: Union[list[DatePart], list[str]], ) -> sqlalchemy.Selectable: """Build a partition query to retrieve batch_identifiers info from a column partition on a list of date parts. @@ -541,8 +541,8 @@ def get_data_for_batch_identifiers_for_partition_on_date_parts( execution_engine: SqlAlchemyExecutionEngine, selectable: sqlalchemy.Selectable, column_name: str, - date_parts: Union[List[DatePart], List[str]], - ) -> List[dict]: + date_parts: Union[list[DatePart], list[str]], + ) -> list[dict]: """Build batch_identifiers from a column partition on a list of date parts. This method builds a query to select the unique date_parts from the @@ -565,7 +565,7 @@ def get_data_for_batch_identifiers_for_partition_on_date_parts( ) ) - result: List[sqlalchemy.Row | sqlalchemy.LegacyRow] = self._execute_partitioned_query( + result: list[sqlalchemy.Row | sqlalchemy.LegacyRow] = self._execute_partitioned_query( execution_engine, partitioned_query ) @@ -577,7 +577,7 @@ def get_data_for_batch_identifiers_for_partition_on_date_parts( def _execute_partitioned_query( execution_engine: SqlAlchemyExecutionEngine, partitioned_query: sqlalchemy.Selectable, - ) -> List[sqlalchemy.Row | sqlalchemy.LegacyRow]: + ) -> list[sqlalchemy.Row | sqlalchemy.LegacyRow]: """Use the provided execution engine to run the partition query and fetch all of the results. Args: @@ -592,9 +592,9 @@ def _execute_partitioned_query( def _get_params_for_batch_identifiers_from_date_part_partitioner( self, column_name: str, - result: List[sqlalchemy.LegacyRow], - date_parts: List[DatePart] | List[str], - ) -> List[dict]: + result: list[sqlalchemy.LegacyRow], + date_parts: list[DatePart] | list[str], + ) -> list[dict]: """Get parameters used to build BatchIdentifiers from the results of a get_data_for_batch_identifiers_for_partition_on_date_parts Args: @@ -608,7 +608,7 @@ def _get_params_for_batch_identifiers_from_date_part_partitioner( """ # noqa: E501 date_parts = self._convert_date_parts(date_parts) - data_for_batch_identifiers: List[dict] = [ + data_for_batch_identifiers: list[dict] = [ { column_name: { date_part.value: getattr(row, date_part.value) for date_part in date_parts @@ -619,8 +619,8 @@ def _get_params_for_batch_identifiers_from_date_part_partitioner( return data_for_batch_identifiers @staticmethod - def _get_column_names_from_partitioner_kwargs(partitioner_kwargs) -> List[str]: - column_names: List[str] = [] + def _get_column_names_from_partitioner_kwargs(partitioner_kwargs) -> list[str]: + column_names: list[str] = [] if "column_names" in partitioner_kwargs: column_names = partitioner_kwargs["column_names"] @@ -635,7 +635,7 @@ def get_data_for_batch_identifiers_for_non_date_part_partitioners( selectable: sqlalchemy.Selectable, partitioner_method_name: str, partitioner_kwargs: dict, - ) -> List[dict]: + ) -> list[dict]: """Build data used to construct batch identifiers for the input table using the provided partitioner config. Sql partitioner configurations yield the unique values that comprise a batch by introspecting your data. @@ -656,10 +656,10 @@ def get_data_for_batch_identifiers_for_non_date_part_partitioners( partitioned_query: sqlalchemy.Selectable = getattr(self, get_partition_query_method_name)( selectable=selectable, **partitioner_kwargs ) - rows: List[sqlalchemy.LegacyRow] = self._execute_partitioned_query( + rows: list[sqlalchemy.LegacyRow] = self._execute_partitioned_query( execution_engine, partitioned_query ) - column_names: List[str] = self._get_column_names_from_partitioner_kwargs(partitioner_kwargs) + column_names: list[str] = self._get_column_names_from_partitioner_kwargs(partitioner_kwargs) return self._get_params_for_batch_identifiers_from_non_date_part_partitioners( column_names, rows ) @@ -689,9 +689,9 @@ def _get_method_name_for_get_data_for_batch_identifiers_method( @staticmethod def _get_params_for_batch_identifiers_from_non_date_part_partitioners( - column_names: List[str], - rows: List[sqlalchemy.LegacyRow], - ) -> List[dict]: + column_names: list[str], + rows: list[sqlalchemy.LegacyRow], + ) -> list[dict]: """Get params used in batch identifiers from output of executing query for non date part partitioners. Args: @@ -832,7 +832,7 @@ def get_partition_query_for_data_for_batch_identifiers_for_partition_on_mod_inte @staticmethod def get_partition_query_for_data_for_batch_identifiers_for_partition_on_multi_column_values( selectable: sqlalchemy.Selectable, - column_names: List[str], + column_names: list[str], ) -> sqlalchemy.Selectable: """Partition on the joint values in the named columns""" return ( diff --git a/great_expectations/execution_engine/sparkdf_execution_engine.py b/great_expectations/execution_engine/sparkdf_execution_engine.py index 4feb31c7c909..7c89cb6b419d 100644 --- a/great_expectations/execution_engine/sparkdf_execution_engine.py +++ b/great_expectations/execution_engine/sparkdf_execution_engine.py @@ -10,9 +10,7 @@ TYPE_CHECKING, Any, Callable, - Dict, Iterable, - List, Optional, Tuple, Union, @@ -414,7 +412,7 @@ def load_batch_data( # type: ignore[override] @override def get_batch_data_and_markers( # noqa: C901, PLR0912, PLR0915 self, batch_spec: BatchSpec - ) -> Tuple[Any, BatchMarkers]: # batch_data + ) -> tuple[Any, BatchMarkers]: # batch_data # We need to build a batch_markers to be used in the dataframe batch_markers = BatchMarkers( { @@ -537,7 +535,7 @@ def get_batch_data_and_markers( # noqa: C901, PLR0912, PLR0915 def _apply_partitioning_and_sampling_methods(self, batch_spec, batch_data): # Note this is to get a batch from tables in AWS Glue Data Catalog by its partitions - partitions: Optional[List[str]] = batch_spec.get("partitions") + partitions: Optional[list[str]] = batch_spec.get("partitions") if partitions: batch_data = self._data_partitioner.partition_on_multi_column_values( df=batch_data, @@ -676,7 +674,7 @@ def get_domain_records( # noqa: C901, PLR0912, PLR0915 ) # Filtering by filter_conditions - filter_conditions: List[RowCondition] = domain_kwargs.get("filter_conditions", []) + filter_conditions: list[RowCondition] = domain_kwargs.get("filter_conditions", []) if len(filter_conditions) > 0: filter_condition = self._combine_row_conditions(filter_conditions) data = data.filter(filter_condition.condition) @@ -728,7 +726,7 @@ def get_domain_records( # noqa: C901, PLR0912, PLR0915 return data @staticmethod - def _combine_row_conditions(row_conditions: List[RowCondition]) -> RowCondition: + def _combine_row_conditions(row_conditions: list[RowCondition]) -> RowCondition: """Combine row conditions using AND if condition_type is SPARK_SQL Note, although this method does not currently use `self` internally we @@ -745,7 +743,7 @@ def _combine_row_conditions(row_conditions: List[RowCondition]) -> RowCondition: condition.condition_type == RowConditionParserType.SPARK_SQL for condition in row_conditions ), "All row conditions must have type SPARK_SQL" - conditions: List[str] = [row_condition.condition for row_condition in row_conditions] + conditions: list[str] = [row_condition.condition for row_condition in row_conditions] joined_condition: str = " AND ".join(conditions) return RowCondition( condition=joined_condition, condition_type=RowConditionParserType.SPARK_SQL @@ -814,7 +812,7 @@ def add_column_row_condition( # type: ignore[explicit-override] # FIXME else: column = domain_kwargs["column"] - filter_conditions: List[RowCondition] = [] + filter_conditions: list[RowCondition] = [] if filter_null: filter_conditions.append( RowCondition( @@ -843,7 +841,7 @@ def add_column_row_condition( # type: ignore[explicit-override] # FIXME def resolve_metric_bundle( self, metric_fn_bundle: Iterable[MetricComputationConfiguration], - ) -> Dict[Tuple[str, str, str], MetricValue]: + ) -> dict[tuple[str, str, str], MetricValue]: """For every metric in a set of Metrics to resolve, obtains necessary metric keyword arguments and builds bundles of the metrics into one large query dictionary so that they are all executed simultaneously. Will fail if bundling the metrics together is not possible. @@ -857,15 +855,15 @@ def resolve_metric_bundle( Returns: A dictionary of "MetricConfiguration" IDs and their corresponding fully resolved values for domains. """ # noqa: E501 - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] = {} + resolved_metrics: dict[tuple[str, str, str], MetricValue] = {} - res: List[pyspark.Row] + res: list[pyspark.Row] - aggregates: Dict[Tuple[str, str, str], dict] = {} + aggregates: dict[tuple[str, str, str], dict] = {} aggregate: dict - domain_id: Tuple[str, str, str] + domain_id: tuple[str, str, str] bundled_metric_configuration: MetricComputationConfiguration for bundled_metric_configuration in metric_fn_bundle: @@ -906,7 +904,7 @@ def resolve_metric_bundle( ), "unexpected number of metrics returned" idx: int - metric_id: Tuple[str, str, str] + metric_id: tuple[str, str, str] for idx, metric_id in enumerate(aggregate["metric_ids"]): # Converting DataFrame.collect() results into JSON-serializable format produces simple data types, # noqa: E501 # amenable for subsequent post-processing by higher-level "Metric" and "Expectation" layers. # noqa: E501 diff --git a/great_expectations/execution_engine/sqlalchemy_batch_data.py b/great_expectations/execution_engine/sqlalchemy_batch_data.py index 25cbc29e02a7..b28d98c63b9a 100644 --- a/great_expectations/execution_engine/sqlalchemy_batch_data.py +++ b/great_expectations/execution_engine/sqlalchemy_batch_data.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Literal, Optional, Tuple, overload +from typing import Literal, Optional, overload from great_expectations.compatibility import sqlalchemy from great_expectations.compatibility.sqlalchemy import Selectable @@ -203,7 +203,7 @@ def _create_temporary_table( # noqa: C901, PLR0912 dialect: GXSqlDialect, query: str, temp_table_schema_name: str | None = None, - ) -> Tuple[str, str]: + ) -> tuple[str, str]: """ Create Temporary table based on sql query. This will be used as a basis for executing expectations. :param query: diff --git a/great_expectations/execution_engine/sqlalchemy_dialect.py b/great_expectations/execution_engine/sqlalchemy_dialect.py index 101c43524d41..10490376671a 100644 --- a/great_expectations/execution_engine/sqlalchemy_dialect.py +++ b/great_expectations/execution_engine/sqlalchemy_dialect.py @@ -1,7 +1,7 @@ from __future__ import annotations from enum import Enum -from typing import Any, Final, List, Literal, Mapping, Union, overload +from typing import Any, Final, Literal, Mapping, Union, overload from great_expectations.compatibility.sqlalchemy import quoted_name from great_expectations.compatibility.typing_extensions import override @@ -53,12 +53,12 @@ def _missing_(cls, value: Any) -> Any: return super()._missing_(value) @classmethod - def get_all_dialect_names(cls) -> List[str]: + def get_all_dialect_names(cls) -> list[str]: """Get dialect names for all SQL dialects.""" return [dialect_name.value for dialect_name in cls if dialect_name != GXSqlDialect.OTHER] @classmethod - def get_all_dialects(cls) -> List[GXSqlDialect]: + def get_all_dialects(cls) -> list[GXSqlDialect]: """Get all dialects.""" return [dialect for dialect in cls if dialect != GXSqlDialect.OTHER] diff --git a/great_expectations/execution_engine/sqlalchemy_execution_engine.py b/great_expectations/execution_engine/sqlalchemy_execution_engine.py index 27a1a8f585f9..9e3512a4aded 100644 --- a/great_expectations/execution_engine/sqlalchemy_execution_engine.py +++ b/great_expectations/execution_engine/sqlalchemy_execution_engine.py @@ -17,12 +17,9 @@ TYPE_CHECKING, Any, Callable, - Dict, Iterable, - List, MutableMapping, Optional, - Tuple, Union, cast, ) @@ -547,7 +544,7 @@ def _build_engine(self, credentials: dict, **kwargs) -> sa.engine.Engine: def _get_sqlalchemy_key_pair_auth_url( drivername: str, credentials: dict, - ) -> Tuple[sa.engine.url.URL, dict]: + ) -> tuple[sa.engine.url.URL, dict]: """ Utilizing a private key path and a passphrase in a given credentials dictionary, attempts to encode the provided values into a private key. If passphrase is incorrect, this will fail and an exception is raised. @@ -663,7 +660,7 @@ def get_domain_records( # noqa: C901, PLR0912, PLR0915 ) # Filtering by filter_conditions - filter_conditions: List[RowCondition] = domain_kwargs.get("filter_conditions", []) + filter_conditions: list[RowCondition] = domain_kwargs.get("filter_conditions", []) # For SqlAlchemyExecutionEngine only one filter condition is allowed if len(filter_conditions) == 1: filter_condition = filter_conditions[0] @@ -795,7 +792,7 @@ def get_compute_domain( domain_kwargs: dict, domain_type: Union[str, MetricDomainTypes], accessor_keys: Optional[Iterable[str]] = None, - ) -> Tuple[sqlalchemy.Selectable, dict, dict]: + ) -> tuple[sqlalchemy.Selectable, dict, dict]: """Uses a given batch dictionary and Domain kwargs to obtain a SqlAlchemy column object. Args: @@ -950,7 +947,7 @@ def _partition_multi_column_metric_domain_kwargs( # type: ignore[override] # Ex def resolve_metric_bundle( # noqa: C901 - too complex self, metric_fn_bundle: Iterable[MetricComputationConfiguration], - ) -> Dict[Tuple[str, str, str], MetricValue]: + ) -> dict[tuple[str, str, str], MetricValue]: """For every metric in a set of Metrics to resolve, obtains necessary metric keyword arguments and builds bundles of the metrics into one large query dictionary so that they are all executed simultaneously. Will fail if bundling the metrics together is not possible. @@ -964,16 +961,16 @@ def resolve_metric_bundle( # noqa: C901 - too complex Returns: A dictionary of "MetricConfiguration" IDs and their corresponding now-queried (fully resolved) values. """ # noqa: E501 - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] = {} + resolved_metrics: dict[tuple[str, str, str], MetricValue] = {} - res: List[sqlalchemy.Row] + res: list[sqlalchemy.Row] # We need a different query for each Domain (where clause). - queries: Dict[Tuple[str, str, str], dict] = {} + queries: dict[tuple[str, str, str], dict] = {} query: dict - domain_id: Tuple[str, str, str] + domain_id: tuple[str, str, str] bundled_metric_configuration: MetricComputationConfiguration for bundled_metric_configuration in metric_fn_bundle: @@ -1049,7 +1046,7 @@ def resolve_metric_bundle( # noqa: C901 - too complex assert len(query["metric_ids"]) == len(res[0]), "unexpected number of metrics returned" idx: int - metric_id: Tuple[str, str, str] + metric_id: tuple[str, str, str] for idx, metric_id in enumerate(query["metric_ids"]): # Converting SQL query execution results into JSON-serializable format produces simple data types, # noqa: E501 # amenable for subsequent post-processing by higher-level "Metric" and "Expectation" layers. # noqa: E501 @@ -1095,7 +1092,7 @@ def _get_partitioner_method(self, partitioner_method_name: str) -> Callable: def execute_partitioned_query( self, partitioned_query: sqlalchemy.Selectable - ) -> List[sqlalchemy.Row]: + ) -> list[sqlalchemy.Row]: """Use the execution engine to run the partitioned query and fetch all of the results. Args: @@ -1122,7 +1119,7 @@ def get_data_for_batch_identifiers( selectable: sqlalchemy.Selectable, partitioner_method_name: str, partitioner_kwargs: dict, - ) -> List[dict]: + ) -> list[dict]: """Build data used to construct batch identifiers for the input table using the provided partitioner config. Sql partitioner configurations yield the unique values that comprise a batch by introspecting your data. @@ -1213,7 +1210,7 @@ def _subselectable(self, batch_spec: BatchSpec) -> sqlalchemy.Selectable: @override def get_batch_data_and_markers( self, batch_spec: BatchSpec - ) -> Tuple[SqlAlchemyBatchData, BatchMarkers]: + ) -> tuple[SqlAlchemyBatchData, BatchMarkers]: if not isinstance(batch_spec, (SqlAlchemyDatasourceBatchSpec, RuntimeQueryBatchSpec)): raise InvalidBatchSpecError( # noqa: TRY003 f"""SqlAlchemyExecutionEngine accepts batch_spec only of type SqlAlchemyDatasourceBatchSpec or diff --git a/great_expectations/execution_engine/util.py b/great_expectations/execution_engine/util.py index 39cc8928321e..04994763a7f5 100644 --- a/great_expectations/execution_engine/util.py +++ b/great_expectations/execution_engine/util.py @@ -2,7 +2,7 @@ # Utility methods for dealing with Dataset objects import logging -from typing import Any, List +from typing import Any import numpy as np @@ -482,7 +482,7 @@ def create_multiple_expectations(df, columns, expectation_type, *args, **kwargs) return results -def get_approximate_percentile_disc_sql(selects: List, sql_engine_dialect: Any) -> str: +def get_approximate_percentile_disc_sql(selects: list, sql_engine_dialect: Any) -> str: return ", ".join( [ "approximate " diff --git a/great_expectations/expectations/core/expect_column_distinct_values_to_be_in_set.py b/great_expectations/expectations/core/expect_column_distinct_values_to_be_in_set.py index b4e50820cad1..b82cc94c040f 100644 --- a/great_expectations/expectations/core/expect_column_distinct_values_to_be_in_set.py +++ b/great_expectations/expectations/core/expect_column_distinct_values_to_be_in_set.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import altair as alt import pandas as pd @@ -195,7 +195,7 @@ class ExpectColumnDistinctValuesToBeInSet(ColumnAggregateExpectation): value_set: ValueSetField - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -219,7 +219,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnDistinctValuesToBeInSet] + schema: dict[str, Any], model: type[ExpectColumnDistinctValuesToBeInSet] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -299,7 +299,7 @@ def _prescriptive_renderer( configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: renderer_configuration: RendererConfiguration = RendererConfiguration( configuration=configuration, result=result, @@ -425,7 +425,7 @@ def _descriptive_value_counts_bar_chart_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_distinct_values_to_contain_set.py b/great_expectations/expectations/core/expect_column_distinct_values_to_contain_set.py index 241315ac9d9f..bc412a59d7b6 100644 --- a/great_expectations/expectations/core/expect_column_distinct_values_to_contain_set.py +++ b/great_expectations/expectations/core/expect_column_distinct_values_to_contain_set.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.expectations.expectation import ( @@ -183,7 +183,7 @@ class ExpectColumnDistinctValuesToContainSet(ColumnAggregateExpectation): value_set: ValueSetField # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -208,7 +208,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnDistinctValuesToContainSet] + schema: dict[str, Any], model: type[ExpectColumnDistinctValuesToContainSet] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -283,7 +283,7 @@ def _prescriptive_renderer( configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: renderer_configuration: RendererConfiguration = RendererConfiguration( configuration=configuration, result=result, @@ -338,7 +338,7 @@ def _prescriptive_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_distinct_values_to_equal_set.py b/great_expectations/expectations/core/expect_column_distinct_values_to_equal_set.py index 4cc09f26afd3..f22ad1940501 100644 --- a/great_expectations/expectations/core/expect_column_distinct_values_to_equal_set.py +++ b/great_expectations/expectations/core/expect_column_distinct_values_to_equal_set.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.expectations.expectation import ( @@ -183,7 +183,7 @@ class ExpectColumnDistinctValuesToEqualSet(ColumnAggregateExpectation): value_set: ValueSetField # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -207,7 +207,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnDistinctValuesToEqualSet] + schema: dict[str, Any], model: type[ExpectColumnDistinctValuesToEqualSet] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -356,7 +356,7 @@ def _prescriptive_renderer( # noqa: C901 - too complex @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_kl_divergence_to_be_less_than.py b/great_expectations/expectations/core/expect_column_kl_divergence_to_be_less_than.py index f447af4e6020..b5ec9d248446 100644 --- a/great_expectations/expectations/core/expect_column_kl_divergence_to_be_less_than.py +++ b/great_expectations/expectations/core/expect_column_kl_divergence_to_be_less_than.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import altair as alt import numpy as np @@ -332,7 +332,7 @@ class ExpectColumnKLDivergenceToBeLessThan(ColumnAggregateExpectation): max_value: Optional[Comparable] = pydantic.Field(None, description=MAX_VALUE_DESCRIPTION) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", @@ -365,7 +365,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnKLDivergenceToBeLessThan] + schema: dict[str, Any], model: type[ExpectColumnKLDivergenceToBeLessThan] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -556,7 +556,7 @@ def get_validation_dependencies( def _validate( # noqa: C901, PLR0912, PLR0915 self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_max_to_be_between.py b/great_expectations/expectations/core/expect_column_max_to_be_between.py index 57d6ab5370e4..00bff90aebe3 100644 --- a/great_expectations/expectations/core/expect_column_max_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_max_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -195,7 +195,7 @@ class ExpectColumnMaxToBeBetween(ColumnAggregateExpectation): strict_min: bool = pydantic.Field(default=False, description=STRICT_MAX_DESCRIPTION) strict_max: bool = pydantic.Field(default=False, description=STRICT_MIN_DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -220,7 +220,7 @@ class Config: title = "Expect column maximum to be between" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnMaxToBeBetween]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnMaxToBeBetween]) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -385,7 +385,7 @@ def _descriptive_stats_table_max_row_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_mean_to_be_between.py b/great_expectations/expectations/core/expect_column_mean_to_be_between.py index ef48c769665e..d6bf6c48a31f 100644 --- a/great_expectations/expectations/core/expect_column_mean_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_mean_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -186,7 +186,7 @@ class ExpectColumnMeanToBeBetween(ColumnAggregateExpectation): strict_min: bool = pydantic.Field(default=False, description=STRICT_MAX_DESCRIPTION) strict_max: bool = pydantic.Field(default=False, description=STRICT_MIN_DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -217,7 +217,7 @@ class Config: title = "Expect column mean to be between" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnMeanToBeBetween]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnMeanToBeBetween]) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -380,7 +380,7 @@ def _descriptive_stats_table_mean_row_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_median_to_be_between.py b/great_expectations/expectations/core/expect_column_median_to_be_between.py index ededd5647b67..6d3ae39e6441 100644 --- a/great_expectations/expectations/core/expect_column_median_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_median_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -182,7 +182,7 @@ class ExpectColumnMedianToBeBetween(ColumnAggregateExpectation): strict_min: bool = pydantic.Field(default=False, description=STRICT_MAX_DESCRIPTION) strict_max: bool = pydantic.Field(default=False, description=STRICT_MIN_DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -214,7 +214,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnMedianToBeBetween] + schema: dict[str, Any], model: type[ExpectColumnMedianToBeBetween] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -357,7 +357,7 @@ def _prescriptive_renderer( # type: ignore[override] # TODO: Fix this type igno @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_min_to_be_between.py b/great_expectations/expectations/core/expect_column_min_to_be_between.py index 68bd0ba11143..997e217748aa 100644 --- a/great_expectations/expectations/core/expect_column_min_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_min_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -188,7 +188,7 @@ class ExpectColumnMinToBeBetween(ColumnAggregateExpectation): strict_min: bool = pydantic.Field(default=False, description=STRICT_MAX_DESCRIPTION) strict_max: bool = pydantic.Field(default=False, description=STRICT_MIN_DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -219,7 +219,7 @@ class Config: title = "Expect column minimum to be between" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnMinToBeBetween]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnMinToBeBetween]) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -384,7 +384,7 @@ def _descriptive_stats_table_min_row_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_most_common_value_to_be_in_set.py b/great_expectations/expectations/core/expect_column_most_common_value_to_be_in_set.py index 145bb4098d1f..48e3255ee95c 100644 --- a/great_expectations/expectations/core/expect_column_most_common_value_to_be_in_set.py +++ b/great_expectations/expectations/core/expect_column_most_common_value_to_be_in_set.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.expectations.expectation import ( @@ -174,7 +174,7 @@ class ExpectColumnMostCommonValueToBeInSet(ColumnAggregateExpectation): ) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -201,7 +201,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnMostCommonValueToBeInSet] + schema: dict[str, Any], model: type[ExpectColumnMostCommonValueToBeInSet] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -331,7 +331,7 @@ def _prescriptive_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_pair_values_a_to_be_greater_than_b.py b/great_expectations/expectations/core/expect_column_pair_values_a_to_be_greater_than_b.py index 1e4bf4017a31..708cb70ea134 100644 --- a/great_expectations/expectations/core/expect_column_pair_values_a_to_be_greater_than_b.py +++ b/great_expectations/expectations/core/expect_column_pair_values_a_to_be_greater_than_b.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Literal, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.expectations.expectation import ( @@ -187,7 +187,7 @@ class ExpectColumnPairValuesAToBeGreaterThanB(ColumnPairMapExpectation): ) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", @@ -219,7 +219,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnPairValuesAToBeGreaterThanB] + schema: dict[str, Any], model: type[ExpectColumnPairValuesAToBeGreaterThanB] ) -> None: ColumnPairMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_pair_values_to_be_equal.py b/great_expectations/expectations/core/expect_column_pair_values_to_be_equal.py index a493a2de1c72..fee262f8f270 100644 --- a/great_expectations/expectations/core/expect_column_pair_values_to_be_equal.py +++ b/great_expectations/expectations/core/expect_column_pair_values_to_be_equal.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Literal, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.expectations.expectation import ( @@ -180,7 +180,7 @@ class ExpectColumnPairValuesToBeEqual(ColumnPairMapExpectation): ) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", @@ -209,7 +209,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnPairValuesToBeEqual] + schema: dict[str, Any], model: type[ExpectColumnPairValuesToBeEqual] ) -> None: ColumnPairMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_pair_values_to_be_in_set.py b/great_expectations/expectations/core/expect_column_pair_values_to_be_in_set.py index 47e6650870aa..6e92c898ed76 100644 --- a/great_expectations/expectations/core/expect_column_pair_values_to_be_in_set.py +++ b/great_expectations/expectations/core/expect_column_pair_values_to_be_in_set.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, ClassVar, Dict, List, Literal, Tuple, Type, Union +from typing import Any, ClassVar, Literal, Union from great_expectations.expectations.expectation import ( ColumnPairMapExpectation, @@ -163,13 +163,13 @@ class ExpectColumnPairValuesToBeInSet(ColumnPairMapExpectation): }} """ # noqa: E501 - value_pairs_set: List[Tuple[Any, Any]] + value_pairs_set: list[tuple[Any, Any]] ignore_row_if: Literal["both_values_are_missing", "either_value_is_missing", "neither"] = ( "both_values_are_missing" ) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", @@ -183,7 +183,7 @@ class ExpectColumnPairValuesToBeInSet(ColumnPairMapExpectation): _library_metadata = library_metadata map_metric = "column_pair_values.in_set" - success_keys: ClassVar[Tuple[str, ...]] = ( + success_keys: ClassVar[tuple[str, ...]] = ( "value_pairs_set", "ignore_row_if", "mostly", @@ -199,7 +199,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnPairValuesToBeInSet] + schema: dict[str, Any], model: type[ExpectColumnPairValuesToBeInSet] ) -> None: ColumnPairMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_proportion_of_unique_values_to_be_between.py b/great_expectations/expectations/core/expect_column_proportion_of_unique_values_to_be_between.py index bd75670c91d9..b16d4b665cb2 100644 --- a/great_expectations/expectations/core/expect_column_proportion_of_unique_values_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_proportion_of_unique_values_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Optional, Type +from typing import TYPE_CHECKING, Any, Optional from great_expectations.compatibility import pydantic from great_expectations.core.types import Comparable # noqa: TCH001 @@ -225,7 +225,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnProportionOfUniqueValuesToBeBetween] + schema: dict[str, Any], model: type[ExpectColumnProportionOfUniqueValuesToBeBetween] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -395,7 +395,7 @@ def _descriptive_column_properties_table_distinct_percent_row_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py b/great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py index aeec1ba4ef6d..da9e5999f793 100644 --- a/great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_quantile_values_to_be_between.py @@ -1,7 +1,7 @@ from __future__ import annotations from numbers import Number -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import numpy as np @@ -51,8 +51,8 @@ class QuantileRange(pydantic.BaseModel): - quantiles: List[float] - value_ranges: List[List[Union[float, int, None]]] + quantiles: list[float] + value_ranges: list[list[Union[float, int, None]]] EXPECTATION_SHORT_DESCRIPTION = ( @@ -241,7 +241,7 @@ class ExpectColumnQuantileValuesToBeBetween(ColumnAggregateExpectation): ) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -269,7 +269,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnQuantileValuesToBeBetween] + schema: dict[str, Any], model: type[ExpectColumnQuantileValuesToBeBetween] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -714,7 +714,7 @@ def get_validation_dependencies( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_stdev_to_be_between.py b/great_expectations/expectations/core/expect_column_stdev_to_be_between.py index 2994eeb1ead6..42f96dbc1fde 100644 --- a/great_expectations/expectations/core/expect_column_stdev_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_stdev_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.types import Comparable # noqa: TCH001 @@ -181,7 +181,7 @@ class ExpectColumnStdevToBeBetween(ColumnAggregateExpectation): strict_max: bool = pydantic.Field(False, description=STRICT_MAX_DESCRIPTION) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -212,7 +212,7 @@ class Config: title = "Expect column standard deviation to be between" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnStdevToBeBetween]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnStdevToBeBetween]) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -350,7 +350,7 @@ def _prescriptive_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_sum_to_be_between.py b/great_expectations/expectations/core/expect_column_sum_to_be_between.py index 8850c7ed1a10..cfc815590ff4 100644 --- a/great_expectations/expectations/core/expect_column_sum_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_sum_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -169,7 +169,7 @@ class ExpectColumnSumToBeBetween(ColumnAggregateExpectation): strict_max: bool = pydantic.Field(False, description=STRICT_MAX_DESCRIPTION) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column aggregate expectation"], "contributors": ["@great_expectations"], @@ -201,7 +201,7 @@ class Config: title = "Expect column sum to be between" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnSumToBeBetween]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnSumToBeBetween]) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -344,7 +344,7 @@ def _prescriptive_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_to_exist.py b/great_expectations/expectations/core/expect_column_to_exist.py index 92ecb7d549b6..41eee66b10c4 100644 --- a/great_expectations/expectations/core/expect_column_to_exist.py +++ b/great_expectations/expectations/core/expect_column_to_exist.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility.pydantic import Field, StrictStr from great_expectations.compatibility.typing_extensions import override @@ -141,7 +141,7 @@ class ExpectColumnToExist(BatchExpectation): ) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "table expectation"], "contributors": ["@great_expectations"], @@ -166,7 +166,7 @@ class Config: title = "Expect column to exist" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnToExist]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnToExist]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -273,7 +273,7 @@ def _prescriptive_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_unique_value_count_to_be_between.py b/great_expectations/expectations/core/expect_column_unique_value_count_to_be_between.py index 662f657398d7..d0ec7d933e76 100644 --- a/great_expectations/expectations/core/expect_column_unique_value_count_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_unique_value_count_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Optional, Type +from typing import TYPE_CHECKING, Any, Optional from great_expectations.compatibility import pydantic from great_expectations.core.types import Comparable # noqa: TCH001 @@ -218,7 +218,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnUniqueValueCountToBeBetween] + schema: dict[str, Any], model: type[ExpectColumnUniqueValueCountToBeBetween] ) -> None: ColumnAggregateExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -382,7 +382,7 @@ def _descriptive_column_properties_table_distinct_count_row_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_value_lengths_to_be_between.py b/great_expectations/expectations/core/expect_column_value_lengths_to_be_between.py index a149b3f2b755..20643c1b2de5 100644 --- a/great_expectations/expectations/core/expect_column_value_lengths_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_value_lengths_to_be_between.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.pydantic import ( @@ -223,7 +223,7 @@ class ExpectColumnValueLengthsToBeBetween(ColumnMapExpectation): strict_max: bool = pydantic.Field(default=False, description=STRICT_MAX_DESCRIPTION) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -253,7 +253,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValueLengthsToBeBetween] + schema: dict[str, Any], model: type[ExpectColumnValueLengthsToBeBetween] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -359,7 +359,7 @@ def _prescriptive_renderer( # noqa: C901 - too complex result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, **kwargs, - ) -> List[ + ) -> list[ Union[ dict, str, diff --git a/great_expectations/expectations/core/expect_column_value_lengths_to_equal.py b/great_expectations/expectations/core/expect_column_value_lengths_to_equal.py index b41f6b98ab25..739974298463 100644 --- a/great_expectations/expectations/core/expect_column_value_lengths_to_equal.py +++ b/great_expectations/expectations/core/expect_column_value_lengths_to_equal.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -176,7 +176,7 @@ class ExpectColumnValueLengthsToEqual(ColumnMapExpectation): value: Union[float, SuiteParameterDict] = pydantic.Field(description=VALUE_DESCRIPTION) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -198,7 +198,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValueLengthsToEqual] + schema: dict[str, Any], model: type[ExpectColumnValueLengthsToEqual] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_value_z_scores_to_be_less_than.py b/great_expectations/expectations/core/expect_column_value_z_scores_to_be_less_than.py index 03a4fcd57aa8..64bbb9ada2e9 100644 --- a/great_expectations/expectations/core/expect_column_value_z_scores_to_be_less_than.py +++ b/great_expectations/expectations/core/expect_column_value_z_scores_to_be_less_than.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -176,14 +176,14 @@ class ExpectColumnValueZScoresToBeLessThan(ColumnMapExpectation): double_sided: Union[bool, SuiteParameterDict] = pydantic.Field( description=DOUBLE_SIDED_DESCRIPTION ) - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "column", "row_condition", "condition_parser", ) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -203,7 +203,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValueZScoresToBeLessThan] + schema: dict[str, Any], model: type[ExpectColumnValueZScoresToBeLessThan] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_values_to_be_between.py b/great_expectations/expectations/core/expect_column_values_to_be_between.py index c1c5486c170c..87931996ab34 100644 --- a/great_expectations/expectations/core/expect_column_values_to_be_between.py +++ b/great_expectations/expectations/core/expect_column_values_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.pydantic import root_validator @@ -211,7 +211,7 @@ def check_min_val_or_max_val(cls, values: dict) -> dict: return values # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -243,7 +243,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToBeBetween] + schema: dict[str, Any], model: type[ExpectColumnValuesToBeBetween] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_values_to_be_in_set.py b/great_expectations/expectations/core/expect_column_values_to_be_in_set.py index c3cb8668d325..bf61557836ef 100644 --- a/great_expectations/expectations/core/expect_column_values_to_be_in_set.py +++ b/great_expectations/expectations/core/expect_column_values_to_be_in_set.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.expectations.expectation import ( ColumnMapExpectation, @@ -186,7 +186,7 @@ class ExpectColumnValuesToBeInSet(ColumnMapExpectation): value_set: ValueSetField - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -203,7 +203,7 @@ class ExpectColumnValuesToBeInSet(ColumnMapExpectation): "value_set", ) - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "column", "row_condition", "condition_parser", @@ -217,7 +217,7 @@ class Config: title = "Expect column values to be in set" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnValuesToBeInSet]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnValuesToBeInSet]) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -298,7 +298,7 @@ def _prescriptive_renderer( configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: renderer_configuration: RendererConfiguration = RendererConfiguration( configuration=configuration, result=result, diff --git a/great_expectations/expectations/core/expect_column_values_to_be_in_type_list.py b/great_expectations/expectations/core/expect_column_values_to_be_in_type_list.py index 445c0fbbf99c..17f333c40a73 100644 --- a/great_expectations/expectations/core/expect_column_values_to_be_in_type_list.py +++ b/great_expectations/expectations/core/expect_column_values_to_be_in_type_list.py @@ -2,7 +2,7 @@ import inspect import logging -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import numpy as np import pandas as pd @@ -205,11 +205,11 @@ class ExpectColumnValuesToBeInTypeList(ColumnMapExpectation): }} """ # noqa: E501 - type_list: Union[List[str], SuiteParameterDict, None] = pydantic.Field( + type_list: Union[list[str], SuiteParameterDict, None] = pydantic.Field( description=TYPE_LIST_DESCRIPTION ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -220,7 +220,7 @@ class ExpectColumnValuesToBeInTypeList(ColumnMapExpectation): _library_metadata = library_metadata map_metric = "column_values.in_type_list" - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "column", "row_condition", "condition_parser", @@ -240,7 +240,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToBeInTypeList] + schema: dict[str, Any], model: type[ExpectColumnValuesToBeInTypeList] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -616,7 +616,7 @@ def get_validation_dependencies( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_values_to_be_null.py b/great_expectations/expectations/core/expect_column_values_to_be_null.py index 3529f04c8ac1..40d2d795e47c 100644 --- a/great_expectations/expectations/core/expect_column_values_to_be_null.py +++ b/great_expectations/expectations/core/expect_column_values_to_be_null.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.core.metric_function_types import ( SummarizationMetricNameSuffixes, @@ -170,13 +170,13 @@ class ExpectColumnValuesToBeNull(ColumnMapExpectation): }} """ # noqa: E501 - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "column", "row_condition", "condition_parser", ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -193,7 +193,7 @@ class Config: title = "Expect column values to be null" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnValuesToBeNull]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnValuesToBeNull]) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -336,7 +336,7 @@ def get_validation_dependencies( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_values_to_be_of_type.py b/great_expectations/expectations/core/expect_column_values_to_be_of_type.py index ab0cdbb8ab85..9cc611d0062b 100644 --- a/great_expectations/expectations/core/expect_column_values_to_be_of_type.py +++ b/great_expectations/expectations/core/expect_column_values_to_be_of_type.py @@ -2,7 +2,7 @@ import inspect import logging -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import numpy as np import pandas as pd @@ -230,7 +230,7 @@ class ExpectColumnValuesToBeOfType(ColumnMapExpectation): type_: str = pydantic.Field(description=TYPE__DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -241,7 +241,7 @@ class ExpectColumnValuesToBeOfType(ColumnMapExpectation): _library_metadata = library_metadata map_metric = "column_values.of_type" - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "column", "row_condition", "condition_parser", @@ -259,7 +259,7 @@ class Config: title = "Expect column values to be of type" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnValuesToBeOfType]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnValuesToBeOfType]) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -578,7 +578,7 @@ def get_validation_dependencies( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_values_to_be_unique.py b/great_expectations/expectations/core/expect_column_values_to_be_unique.py index 1dede60d8957..667fb0ff05ab 100644 --- a/great_expectations/expectations/core/expect_column_values_to_be_unique.py +++ b/great_expectations/expectations/core/expect_column_values_to_be_unique.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.expectations.expectation import ( @@ -164,7 +164,7 @@ class ExpectColumnValuesToBeUnique(ColumnMapExpectation): }} """ # noqa: E501 - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -182,7 +182,7 @@ class Config: title = "Expect column values to be unique" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectColumnValuesToBeUnique]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectColumnValuesToBeUnique]) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { diff --git a/great_expectations/expectations/core/expect_column_values_to_match_like_pattern.py b/great_expectations/expectations/core/expect_column_values_to_match_like_pattern.py index aa76f0383d20..e9e9cbc9c5f0 100644 --- a/great_expectations/expectations/core/expect_column_values_to_match_like_pattern.py +++ b/great_expectations/expectations/core/expect_column_values_to_match_like_pattern.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -168,7 +168,7 @@ class ExpectColumnValuesToMatchLikePattern(ColumnMapExpectation): description=LIKE_PATTERN_DESCRIPTION ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -195,7 +195,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToMatchLikePattern] + schema: dict[str, Any], model: type[ExpectColumnValuesToMatchLikePattern] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -265,7 +265,7 @@ def _prescriptive_renderer( result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, **kwargs, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: runtime_configuration = runtime_configuration or {} _ = runtime_configuration.get("include_column_name") is not False styling = runtime_configuration.get("styling") diff --git a/great_expectations/expectations/core/expect_column_values_to_match_like_pattern_list.py b/great_expectations/expectations/core/expect_column_values_to_match_like_pattern_list.py index 7392c32dcf6a..4d747c210b93 100644 --- a/great_expectations/expectations/core/expect_column_values_to_match_like_pattern_list.py +++ b/great_expectations/expectations/core/expect_column_values_to_match_like_pattern_list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Literal, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -173,7 +173,7 @@ class ExpectColumnValuesToMatchLikePatternList(ColumnMapExpectation): }} """ # noqa: E501 - like_pattern_list: Union[List[str], SuiteParameterDict] = pydantic.Field( + like_pattern_list: Union[list[str], SuiteParameterDict] = pydantic.Field( description=LIKE_PATTERN_DESCRIPTION ) match_on: Literal["any", "all"] = pydantic.Field( @@ -189,7 +189,7 @@ def validate_like_pattern_list( return like_pattern_list - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -213,7 +213,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToMatchLikePatternList] + schema: dict[str, Any], model: type[ExpectColumnValuesToMatchLikePatternList] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -299,7 +299,7 @@ def _prescriptive_renderer( result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, **kwargs, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: runtime_configuration = runtime_configuration or {} _ = runtime_configuration.get("include_column_name") is not False styling = runtime_configuration.get("styling") diff --git a/great_expectations/expectations/core/expect_column_values_to_match_regex.py b/great_expectations/expectations/core/expect_column_values_to_match_regex.py index 7d9c1c1812ef..0f45c4fe713c 100644 --- a/great_expectations/expectations/core/expect_column_values_to_match_regex.py +++ b/great_expectations/expectations/core/expect_column_values_to_match_regex.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -172,7 +172,7 @@ class ExpectColumnValuesToMatchRegex(ColumnMapExpectation): default="(?s).*", description=REGEX_DESCRIPTION ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -199,7 +199,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToMatchRegex] + schema: dict[str, Any], model: type[ExpectColumnValuesToMatchRegex] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_values_to_match_regex_list.py b/great_expectations/expectations/core/expect_column_values_to_match_regex_list.py index 9d8031ddedb4..c54ac4441ae3 100644 --- a/great_expectations/expectations/core/expect_column_values_to_match_regex_list.py +++ b/great_expectations/expectations/core/expect_column_values_to_match_regex_list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Literal, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -178,14 +178,14 @@ class ExpectColumnValuesToMatchRegexList(ColumnMapExpectation): }} """ # noqa: E501 - regex_list: Union[List[str], SuiteParameterDict] = pydantic.Field( + regex_list: Union[list[str], SuiteParameterDict] = pydantic.Field( description=REGEX_LIST_DESCRIPTION ) match_on: Literal["any", "all"] = pydantic.Field( default="any", description=MATCH_ON_DESCRIPTION ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -213,7 +213,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToMatchRegexList] + schema: dict[str, Any], model: type[ExpectColumnValuesToMatchRegexList] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_values_to_not_be_in_set.py b/great_expectations/expectations/core/expect_column_values_to_not_be_in_set.py index 69c9eea0bd34..f9d2b02d4765 100644 --- a/great_expectations/expectations/core/expect_column_values_to_not_be_in_set.py +++ b/great_expectations/expectations/core/expect_column_values_to_not_be_in_set.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import numpy as np import pandas as pd @@ -175,7 +175,7 @@ class ExpectColumnValuesToNotBeInSet(ColumnMapExpectation): value_set: ValueSetField - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -202,7 +202,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToNotBeInSet] + schema: dict[str, Any], model: type[ExpectColumnValuesToNotBeInSet] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -347,7 +347,7 @@ def _prescriptive_renderer( def _pandas_column_values_not_in_set( # noqa: PLR0913 self, series: pd.Series, - metrics: Dict, + metrics: dict, metric_domain_kwargs: dict, metric_value_kwargs: dict, runtime_configuration: Optional[dict] = None, diff --git a/great_expectations/expectations/core/expect_column_values_to_not_be_null.py b/great_expectations/expectations/core/expect_column_values_to_not_be_null.py index b581903a5030..b7c6a70a4b36 100644 --- a/great_expectations/expectations/core/expect_column_values_to_not_be_null.py +++ b/great_expectations/expectations/core/expect_column_values_to_not_be_null.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.metric_function_types import ( @@ -173,7 +173,7 @@ class ExpectColumnValuesToNotBeNull(ColumnMapExpectation): }} """ # noqa: E501 - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": ["@great_expectations"], @@ -184,14 +184,14 @@ class ExpectColumnValuesToNotBeNull(ColumnMapExpectation): _library_metadata = library_metadata map_metric: ClassVar[str] = "column_values.nonnull" - args_keys: ClassVar[Tuple[str, ...]] = ("column",) + args_keys: ClassVar[tuple[str, ...]] = ("column",) class Config: title = "Expect column values to not be null" @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToNotBeNull] + schema: dict[str, Any], model: type[ExpectColumnValuesToNotBeNull] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -375,7 +375,7 @@ def _descriptive_column_properties_table_missing_percent_row_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_column_values_to_not_match_like_pattern.py b/great_expectations/expectations/core/expect_column_values_to_not_match_like_pattern.py index 57e519767077..466ca44a678a 100644 --- a/great_expectations/expectations/core/expect_column_values_to_not_match_like_pattern.py +++ b/great_expectations/expectations/core/expect_column_values_to_not_match_like_pattern.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -168,7 +168,7 @@ class ExpectColumnValuesToNotMatchLikePattern(ColumnMapExpectation): description=LIKE_PATTERN_DESCRIPTION ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -195,7 +195,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToNotMatchLikePattern] + schema: dict[str, Any], model: type[ExpectColumnValuesToNotMatchLikePattern] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -265,7 +265,7 @@ def _prescriptive_renderer( result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, **kwargs, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: runtime_configuration = runtime_configuration or {} _ = runtime_configuration.get("include_column_name") is not False styling = runtime_configuration.get("styling") diff --git a/great_expectations/expectations/core/expect_column_values_to_not_match_like_pattern_list.py b/great_expectations/expectations/core/expect_column_values_to_not_match_like_pattern_list.py index 803d190d753c..8e8d188b33dc 100644 --- a/great_expectations/expectations/core/expect_column_values_to_not_match_like_pattern_list.py +++ b/great_expectations/expectations/core/expect_column_values_to_not_match_like_pattern_list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -169,7 +169,7 @@ class ExpectColumnValuesToNotMatchLikePatternList(ColumnMapExpectation): }} """ # noqa: E501 - like_pattern_list: Union[List[str], SuiteParameterDict] = pydantic.Field( + like_pattern_list: Union[list[str], SuiteParameterDict] = pydantic.Field( description=LIKE_PATTERN_LIST_DESCRIPTION ) @@ -182,7 +182,7 @@ def validate_like_pattern_list( return like_pattern_list - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -209,7 +209,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToNotMatchLikePatternList] + schema: dict[str, Any], model: type[ExpectColumnValuesToNotMatchLikePatternList] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -289,7 +289,7 @@ def _prescriptive_renderer( result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, **kwargs, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: runtime_configuration = runtime_configuration or {} _ = runtime_configuration.get("include_column_name") is not False styling = runtime_configuration.get("styling") diff --git a/great_expectations/expectations/core/expect_column_values_to_not_match_regex.py b/great_expectations/expectations/core/expect_column_values_to_not_match_regex.py index d5473649255b..59ed39e728ec 100644 --- a/great_expectations/expectations/core/expect_column_values_to_not_match_regex.py +++ b/great_expectations/expectations/core/expect_column_values_to_not_match_regex.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -183,7 +183,7 @@ class ExpectColumnValuesToNotMatchRegex(ColumnMapExpectation): regex: Union[str, SuiteParameterDict] = pydantic.Field(description=REGEX_DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -210,7 +210,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToNotMatchRegex] + schema: dict[str, Any], model: type[ExpectColumnValuesToNotMatchRegex] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_column_values_to_not_match_regex_list.py b/great_expectations/expectations/core/expect_column_values_to_not_match_regex_list.py index e0f4e590c5a2..623f1dab9eb0 100644 --- a/great_expectations/expectations/core/expect_column_values_to_not_match_regex_list.py +++ b/great_expectations/expectations/core/expect_column_values_to_not_match_regex_list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -166,11 +166,11 @@ class ExpectColumnValuesToNotMatchRegexList(ColumnMapExpectation): }} """ # noqa: E501 - regex_list: Union[List[str], SuiteParameterDict] = pydantic.Field( + regex_list: Union[list[str], SuiteParameterDict] = pydantic.Field( description=REGEX_LIST_DESCRIPTION ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "column map expectation"], "contributors": [ @@ -197,7 +197,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectColumnValuesToNotMatchRegexList] + schema: dict[str, Any], model: type[ExpectColumnValuesToNotMatchRegexList] ) -> None: ColumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_compound_columns_to_be_unique.py b/great_expectations/expectations/core/expect_compound_columns_to_be_unique.py index 65d737c74b62..7793c27f2058 100644 --- a/great_expectations/expectations/core/expect_compound_columns_to_be_unique.py +++ b/great_expectations/expectations/core/expect_compound_columns_to_be_unique.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Sequence, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Sequence, Union from great_expectations.compatibility import pydantic from great_expectations.expectations.expectation import ( @@ -174,7 +174,7 @@ class ExpectCompoundColumnsToBeUnique(MulticolumnMapExpectation): column_list: Sequence[str] = pydantic.Field(description=COLUMN_LIST_DESCRIPTION) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", @@ -197,7 +197,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectCompoundColumnsToBeUnique] + schema: dict[str, Any], model: type[ExpectCompoundColumnsToBeUnique] ) -> None: MulticolumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_multicolumn_sum_to_equal.py b/great_expectations/expectations/core/expect_multicolumn_sum_to_equal.py index 35a1cabc7249..da1511d60f5b 100644 --- a/great_expectations/expectations/core/expect_multicolumn_sum_to_equal.py +++ b/great_expectations/expectations/core/expect_multicolumn_sum_to_equal.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Literal, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.expectations.expectation import ( @@ -190,7 +190,7 @@ class ExpectMulticolumnSumToEqual(MulticolumnMapExpectation): ) # This dictionary contains metadata for display in the public gallery - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", @@ -214,7 +214,7 @@ class Config: title = "Expect multicolumn sum to equal" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectMulticolumnSumToEqual]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectMulticolumnSumToEqual]) -> None: MulticolumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -292,7 +292,7 @@ def _prescriptive_renderer( result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, **kwargs, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: runtime_configuration = runtime_configuration or {} styling = runtime_configuration.get("styling") params = substitute_none_for_missing( diff --git a/great_expectations/expectations/core/expect_select_column_values_to_be_unique_within_record.py b/great_expectations/expectations/core/expect_select_column_values_to_be_unique_within_record.py index d6679c0cb8fb..bfb489def570 100644 --- a/great_expectations/expectations/core/expect_select_column_values_to_be_unique_within_record.py +++ b/great_expectations/expectations/core/expect_select_column_values_to_be_unique_within_record.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Sequence, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Sequence, Union from great_expectations.compatibility import pydantic from great_expectations.expectations.expectation import ( @@ -182,7 +182,7 @@ class ExpectSelectColumnValuesToBeUniqueWithinRecord(MulticolumnMapExpectation): default="all_values_are_missing", description=IGNORE_ROW_IF_DESCRIPTION ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": [ "core expectation", @@ -205,7 +205,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectSelectColumnValuesToBeUniqueWithinRecord] + schema: dict[str, Any], model: type[ExpectSelectColumnValuesToBeUniqueWithinRecord] ) -> None: MulticolumnMapExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( diff --git a/great_expectations/expectations/core/expect_table_column_count_to_be_between.py b/great_expectations/expectations/core/expect_table_column_count_to_be_between.py index 91026a113570..cfb94bce9b14 100644 --- a/great_expectations/expectations/core/expect_table_column_count_to_be_between.py +++ b/great_expectations/expectations/core/expect_table_column_count_to_be_between.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -152,7 +152,7 @@ class ExpectTableColumnCountToBeBetween(BatchExpectation): min_value: Optional[Comparable] = pydantic.Field(description=MIN_VALUE_DESCRIPTION) max_value: Optional[Comparable] = pydantic.Field(description=MAX_VALUE_DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "table expectation"], "contributors": [ @@ -179,7 +179,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectTableColumnCountToBeBetween] + schema: dict[str, Any], model: type[ExpectTableColumnCountToBeBetween] ) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -298,7 +298,7 @@ def _prescriptive_renderer( # type: ignore[override] # TODO: Fix this type igno @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_table_column_count_to_equal.py b/great_expectations/expectations/core/expect_table_column_count_to_equal.py index 9bea9c2fc8ee..131bd83b5645 100644 --- a/great_expectations/expectations/core/expect_table_column_count_to_equal.py +++ b/great_expectations/expectations/core/expect_table_column_count_to_equal.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -139,7 +139,7 @@ class ExpectTableColumnCountToEqual(BatchExpectation): value: Union[int, SuiteParameterDict] = pydantic.Field(description=VALUE_DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "table expectation"], "contributors": [ @@ -159,7 +159,7 @@ class Config: title = "Expect table column count to equal" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[Expectation]) -> None: + def schema_extra(schema: dict[str, Any], model: type[Expectation]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -225,7 +225,7 @@ def _prescriptive_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_table_columns_to_match_ordered_list.py b/great_expectations/expectations/core/expect_table_columns_to_match_ordered_list.py index 1dee3d987dcc..fb4a306661e5 100644 --- a/great_expectations/expectations/core/expect_table_columns_to_match_ordered_list.py +++ b/great_expectations/expectations/core/expect_table_columns_to_match_ordered_list.py @@ -1,7 +1,7 @@ from __future__ import annotations from itertools import zip_longest -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -166,7 +166,7 @@ class ExpectTableColumnsToMatchOrderedList(BatchExpectation): description=COLUMN_LIST_DESCRIPTION ) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "table expectation"], "contributors": ["@great_expectations"], @@ -191,7 +191,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectTableColumnsToMatchOrderedList] + schema: dict[str, Any], model: type[ExpectTableColumnsToMatchOrderedList] ) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -295,7 +295,7 @@ def _prescriptive_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_table_columns_to_match_set.py b/great_expectations/expectations/core/expect_table_columns_to_match_set.py index 2b8ae9e59fd0..9900ce5dfaf8 100644 --- a/great_expectations/expectations/core/expect_table_columns_to_match_set.py +++ b/great_expectations/expectations/core/expect_table_columns_to_match_set.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.core.suite_parameters import ( @@ -171,7 +171,7 @@ class ExpectTableColumnsToMatchSet(BatchExpectation): ) exact_match: Union[bool, None] = pydantic.Field(description=EXACT_MATCH_DESCRIPTION) - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "table expectation"], "contributors": [ @@ -197,7 +197,7 @@ class Config: title = "Expect table columns to match set" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectTableColumnsToMatchSet]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectTableColumnsToMatchSet]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -314,7 +314,7 @@ def _prescriptive_renderer( def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_table_row_count_to_be_between.py b/great_expectations/expectations/core/expect_table_row_count_to_be_between.py index 437a9d89ba03..c1fb2bd06212 100644 --- a/great_expectations/expectations/core/expect_table_row_count_to_be_between.py +++ b/great_expectations/expectations/core/expect_table_row_count_to_be_between.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -164,7 +164,7 @@ class ExpectTableRowCountToBeBetween(BatchExpectation): row_condition: Union[str, None] = None condition_parser: Union[ConditionParser, None] = None - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "table expectation"], "contributors": ["@great_expectations"], @@ -175,7 +175,7 @@ class ExpectTableRowCountToBeBetween(BatchExpectation): _library_metadata = library_metadata metric_dependencies = ("table.row_count",) - domain_keys: ClassVar[Tuple[str, ...]] = tuple() + domain_keys: ClassVar[tuple[str, ...]] = tuple() success_keys = ( "min_value", "max_value", @@ -190,7 +190,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectTableRowCountToBeBetween] + schema: dict[str, Any], model: type[ExpectTableRowCountToBeBetween] ) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -315,7 +315,7 @@ def _prescriptive_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_table_row_count_to_equal.py b/great_expectations/expectations/core/expect_table_row_count_to_equal.py index bdf65cd8cb19..c04b7294f17a 100644 --- a/great_expectations/expectations/core/expect_table_row_count_to_equal.py +++ b/great_expectations/expectations/core/expect_table_row_count_to_equal.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -143,7 +143,7 @@ class ExpectTableRowCountToEqual(BatchExpectation): row_condition: Union[str, None] = None condition_parser: Union[ConditionParser, None] = None - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "table expectation"], "contributors": ["@great_expectations"], @@ -161,7 +161,7 @@ class Config: title = "Expect table row count to equal" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ExpectTableRowCountToEqual]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ExpectTableRowCountToEqual]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -207,7 +207,7 @@ def _prescriptive_renderer( configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: renderer_configuration: RendererConfiguration = RendererConfiguration( configuration=configuration, result=result, @@ -237,7 +237,7 @@ def _prescriptive_renderer( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/expect_table_row_count_to_equal_other_table.py b/great_expectations/expectations/core/expect_table_row_count_to_equal_other_table.py index 411ae92ae7aa..bf7a282421d9 100644 --- a/great_expectations/expectations/core/expect_table_row_count_to_equal_other_table.py +++ b/great_expectations/expectations/core/expect_table_row_count_to_equal_other_table.py @@ -1,7 +1,7 @@ from __future__ import annotations from copy import deepcopy -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -151,7 +151,7 @@ class ExpectTableRowCountToEqualOtherTable(BatchExpectation): row_condition: Union[str, None] = None condition_parser: Union[ConditionParser, None] = None - library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + library_metadata: ClassVar[dict[str, Union[str, list, bool]]] = { "maturity": "production", "tags": ["core expectation", "table expectation", "multi-table expectation"], "contributors": [ @@ -172,7 +172,7 @@ class Config: @staticmethod def schema_extra( - schema: Dict[str, Any], model: Type[ExpectTableRowCountToEqualOtherTable] + schema: dict[str, Any], model: type[ExpectTableRowCountToEqualOtherTable] ) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( @@ -313,7 +313,7 @@ def get_validation_dependencies( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): diff --git a/great_expectations/expectations/core/unexpected_rows_expectation.py b/great_expectations/expectations/core/unexpected_rows_expectation.py index 49ac26991e11..6b4ee6a396cf 100644 --- a/great_expectations/expectations/core/unexpected_rows_expectation.py +++ b/great_expectations/expectations/core/unexpected_rows_expectation.py @@ -2,7 +2,7 @@ import logging from string import Formatter -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, ClassVar, Union from great_expectations.compatibility import pydantic from great_expectations.compatibility.typing_extensions import override @@ -65,9 +65,9 @@ class UnexpectedRowsExpectation(BatchExpectation): unexpected_rows_query: str = pydantic.Field(description=UNEXPECTED_ROWS_QUERY_DESCRIPTION) - metric_dependencies: ClassVar[Tuple[str, ...]] = ("unexpected_rows_query.table",) - success_keys: ClassVar[Tuple[str, ...]] = ("unexpected_rows_query",) - domain_keys: ClassVar[Tuple[str, ...]] = ( + metric_dependencies: ClassVar[tuple[str, ...]] = ("unexpected_rows_query.table",) + success_keys: ClassVar[tuple[str, ...]] = ("unexpected_rows_query",) + domain_keys: ClassVar[tuple[str, ...]] = ( "batch_id", "row_condition", "condition_parser", @@ -93,7 +93,7 @@ class Config: title = "Custom Expectation with SQL" @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[UnexpectedRowsExpectation]) -> None: + def schema_extra(schema: dict[str, Any], model: type[UnexpectedRowsExpectation]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { diff --git a/great_expectations/expectations/expectation.py b/great_expectations/expectations/expectation.py index 1f9c3019025e..222aea66fa6f 100644 --- a/great_expectations/expectations/expectation.py +++ b/great_expectations/expectations/expectation.py @@ -15,14 +15,9 @@ Any, Callable, ClassVar, - Dict, - List, Literal, Optional, Sequence, - Set, - Tuple, - Type, TypeVar, Union, ) @@ -117,7 +112,7 @@ logger = logging.getLogger(__name__) P = ParamSpec("P") -T = TypeVar("T", List[RenderedStringTemplateContent], RenderedAtomicContent) +T = TypeVar("T", list[RenderedStringTemplateContent], RenderedAtomicContent) def render_suite_parameter_string(render_func: Callable[P, T]) -> Callable[P, T]: # noqa: C901 @@ -203,7 +198,7 @@ def wrapper( renderer_configuration: RendererConfiguration, ) -> Optional[Any]: try: - return_type: Type = param_func.__annotations__["return"] + return_type: type = param_func.__annotations__["return"] except KeyError: method_name: str = getattr(param_func, "__name__", repr(param_func)) raise RendererConfigurationError( # noqa: TRY003 @@ -294,7 +289,7 @@ class Config: json_encoders = {RenderedAtomicContent: lambda data: data.to_json_dict()} @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[Expectation]) -> None: + def schema_extra(schema: dict[str, Any], model: type[Expectation]) -> None: # Add metadata to the schema schema["properties"]["metadata"] = { "type": "object", @@ -322,27 +317,27 @@ def schema_extra(schema: Dict[str, Any], model: Type[Expectation]) -> None: id: Union[str, None] = None meta: Union[dict, None] = None - notes: Union[str, List[str], None] = None + notes: Union[str, list[str], None] = None result_format: Union[ResultFormat, dict] = ResultFormat.BASIC description: Union[str, None] = pydantic.Field( default=None, description="A short description of your Expectation" ) catch_exceptions: bool = False - rendered_content: Optional[List[RenderedAtomicContent]] = None + rendered_content: Optional[list[RenderedAtomicContent]] = None version: ClassVar[str] = ge_version - domain_keys: ClassVar[Tuple[str, ...]] = () - success_keys: ClassVar[Tuple[str, ...]] = () - runtime_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = () + success_keys: ClassVar[tuple[str, ...]] = () + runtime_keys: ClassVar[tuple[str, ...]] = ( "catch_exceptions", "result_format", ) - args_keys: ClassVar[Tuple[str, ...]] = () + args_keys: ClassVar[tuple[str, ...]] = () expectation_type: ClassVar[str] - windows: Optional[List[Window]] = pydantic.Field(default=None, description=WINDOWS_DESCRIPTION) - examples: ClassVar[List[dict]] = [] + windows: Optional[list[Window]] = pydantic.Field(default=None, description=WINDOWS_DESCRIPTION) + examples: ClassVar[list[dict]] = [] _save_callback: Union[Callable[[Expectation], Expectation], None] = pydantic.PrivateAttr( default=None @@ -526,7 +521,7 @@ def _atomic_prescriptive_template( configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> Tuple[Optional[str], dict, MetaNotes, Optional[dict]]: + ) -> tuple[Optional[str], dict, MetaNotes, Optional[dict]]: """ Template function that contains the logic that is shared by AtomicPrescriptiveRendererType.SUMMARY and LegacyRendererType.PRESCRIPTIVE. @@ -593,7 +588,7 @@ def _prescriptive_renderer( configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: renderer_configuration: RendererConfiguration = RendererConfiguration( configuration=configuration, result=result, @@ -627,7 +622,7 @@ def _prescriptive_renderer( def _diagnostic_meta_properties_renderer( # noqa: C901 cls, result: Optional[ExpectationValidationResult] = None, - ) -> Union[list, List[str], List[list]]: + ) -> Union[list, list[str], list[list]]: """ Render function used to add custom meta to Data Docs It gets a column set in the `properties_to_render` dictionary within `meta` and adds columns in Data Docs with the values that were set. @@ -769,7 +764,7 @@ def _diagnostic_unexpected_statement_renderer( configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Union[RenderedStringTemplateContent, CollapseContent]]: + ) -> list[Union[RenderedStringTemplateContent, CollapseContent]]: assert result, "Must provide a result object." success: Optional[bool] = result.success result_dict: dict = result.result @@ -881,7 +876,7 @@ def _diagnostic_unexpected_table_renderer( # noqa: C901, PLR0912 configuration: Optional[ExpectationConfiguration] = None, result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, - ) -> Optional[List[Union[RenderedTableContent, CollapseContent]]]: + ) -> Optional[list[Union[RenderedTableContent, CollapseContent]]]: if result is None: return None @@ -894,21 +889,21 @@ def _diagnostic_unexpected_table_renderer( # noqa: C901, PLR0912 "partial_unexpected_counts" ): return None - table_rows: List[Any] = [] + table_rows: list[Any] = [] - partial_unexpected_counts: Optional[List[dict]] = result_dict.get( + partial_unexpected_counts: Optional[list[dict]] = result_dict.get( "partial_unexpected_counts" ) # this means the result_format is COMPLETE and we have the full set of unexpected indices - unexpected_index_list: Optional[List[dict]] = result_dict.get("unexpected_index_list") + unexpected_index_list: Optional[list[dict]] = result_dict.get("unexpected_index_list") unexpected_count: int = result_dict["unexpected_count"] if partial_unexpected_counts: # We will check to see whether we have *all* of the unexpected values # accounted for in our count, and include counts if we do. If we do not, # we will use this as simply a better (non-repeating) source of # "sampled" unexpected values - unexpected_list: Optional[List[dict]] = result_dict.get("unexpected_list") - unexpected_index_column_names: Optional[List[str]] = result_dict.get( + unexpected_list: Optional[list[dict]] = result_dict.get("unexpected_list") + unexpected_index_column_names: Optional[list[str]] = result_dict.get( "unexpected_index_column_names" ) if unexpected_index_list: @@ -928,7 +923,7 @@ def _diagnostic_unexpected_table_renderer( # noqa: C901, PLR0912 else: header_row = ["Sampled Unexpected Values"] sampled_values_set = set() - partial_unexpected_list: Optional[List[Any]] = result_dict.get( + partial_unexpected_list: Optional[list[Any]] = result_dict.get( "partial_unexpected_list" ) if partial_unexpected_list: @@ -1076,8 +1071,8 @@ def _diagnostic_observed_value_renderer( return cls._get_observed_value_from_evr(result=result) @classmethod - def get_allowed_config_keys(cls) -> Union[Tuple[str, ...], Tuple[str]]: - key_list: Union[list, List[str]] = [] + def get_allowed_config_keys(cls) -> Union[tuple[str, ...], tuple[str]]: + key_list: Union[list, list[str]] = [] if len(cls.domain_keys) > 0: key_list.extend(list(cls.domain_keys)) if len(cls.success_keys) > 0: @@ -1103,7 +1098,7 @@ def metrics_validate( ) runtime_configuration["result_format"] = validation_dependencies.result_format - validation_dependencies_metric_configurations: List[MetricConfiguration] = ( + validation_dependencies_metric_configurations: list[MetricConfiguration] = ( validation_dependencies.get_metric_configurations() ) @@ -1114,7 +1109,7 @@ def metrics_validate( metric_name: str metric_configuration: MetricConfiguration - provided_metrics: Dict[str, MetricValue] = { + provided_metrics: dict[str, MetricValue] = { metric_name: metrics[metric_configuration.id] for metric_name, metric_configuration in validation_dependencies.metric_configurations.items() # noqa: E501 } @@ -1183,19 +1178,19 @@ def _get_default_value(self, key: str) -> Any: logger.info(f'_get_default_value called with key "{key}", but it is not a known field') return None - def _get_domain_kwargs(self) -> Dict[str, Optional[str]]: - domain_kwargs: Dict[str, Optional[str]] = { + def _get_domain_kwargs(self) -> dict[str, Optional[str]]: + domain_kwargs: dict[str, Optional[str]] = { key: self.configuration.kwargs.get(key, self._get_default_value(key)) for key in self.domain_keys } - missing_kwargs: Union[set, Set[str]] = set(self.domain_keys) - set(domain_kwargs.keys()) + missing_kwargs: Union[set, set[str]] = set(self.domain_keys) - set(domain_kwargs.keys()) if missing_kwargs: raise InvalidExpectationKwargsError(f"Missing domain kwargs: {list(missing_kwargs)}") # noqa: TRY003 return domain_kwargs - def _get_success_kwargs(self) -> Dict[str, Any]: - domain_kwargs: Dict[str, Optional[str]] = self._get_domain_kwargs() - success_kwargs: Dict[str, Any] = { + def _get_success_kwargs(self) -> dict[str, Any]: + domain_kwargs: dict[str, Optional[str]] = self._get_domain_kwargs() + success_kwargs: dict[str, Any] = { key: self.configuration.kwargs.get(key, self._get_default_value(key)) for key in self.success_keys } @@ -1225,12 +1220,12 @@ def _get_runtime_kwargs( def _get_result_format( self, runtime_configuration: Optional[dict] = None, - ) -> Union[Dict[str, Union[str, int, bool, List[str], None]], str]: + ) -> Union[dict[str, Union[str, int, bool, list[str], None]], str]: default_result_format: Optional[Any] = self._get_default_value("result_format") configuration_result_format: Union[ - Dict[str, Union[str, int, bool, List[str], None]], str + dict[str, Union[str, int, bool, list[str], None]], str ] = self.configuration.kwargs.get("result_format", default_result_format) - result_format: Union[Dict[str, Union[str, int, bool, List[str], None]], str] + result_format: Union[dict[str, Union[str, int, bool, list[str], None]], str] if runtime_configuration: result_format = runtime_configuration.get( "result_format", @@ -1337,7 +1332,7 @@ def run_diagnostics( # noqa: PLR0913 ignore_only_for: bool = False, for_gallery: bool = False, debug_logger: Optional[logging.Logger] = None, - only_consider_these_backends: Optional[List[str]] = None, + only_consider_these_backends: Optional[list[str]] = None, context: Optional[AbstractDataContext] = None, ) -> ExpectationDiagnostics: """Produce a diagnostic report about this Expectation. @@ -1391,7 +1386,7 @@ def print_diagnostic_checklist( self, diagnostics: Optional[ExpectationDiagnostics] = None, show_failed_tests: bool = False, - backends: Optional[List[str]] = None, + backends: Optional[list[str]] = None, show_debug_messages: bool = False, ) -> str: """Runs self.run_diagnostics and generates a diagnostic checklist. @@ -1553,17 +1548,17 @@ class BatchExpectation(Expectation, ABC): batch_id: Union[str, None] = None - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "batch_id", "table", ) - metric_dependencies: ClassVar[Tuple[str, ...]] = () + metric_dependencies: ClassVar[tuple[str, ...]] = () domain_type: ClassVar[MetricDomainTypes] = MetricDomainTypes.TABLE - args_keys: ClassVar[Tuple[str, ...]] = () + args_keys: ClassVar[tuple[str, ...]] = () class Config: @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[BatchExpectation]) -> None: + def schema_extra(schema: dict[str, Any], model: type[BatchExpectation]) -> None: Expectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -1608,10 +1603,10 @@ def get_validation_dependencies( def _validate_metric_value_between( # noqa: C901, PLR0912 self, metric_name, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, - ) -> Dict[str, Union[bool, Dict[str, Any]]]: + ) -> dict[str, Union[bool, dict[str, Any]]]: metric_value: Optional[Any] = metrics.get(metric_name) if metric_value is None: @@ -1709,7 +1704,7 @@ class QueryExpectation(BatchExpectation, ABC): - https://docs.greatexpectations.io/docs/guides/expectations/creating_custom_expectations/how_to_create_custom_query_expectations """ # noqa: E501 - domain_keys: ClassVar[Tuple] = ("batch_id",) + domain_keys: ClassVar[tuple] = ("batch_id",) @override def validate_configuration( @@ -1739,7 +1734,7 @@ def validate_configuration( try: if not isinstance(query, str): raise TypeError(f"'query' must be a string, but your query is type: {type(query)}") # noqa: TRY003, TRY301 - parsed_query: Set[str] = { + parsed_query: set[str] = { x for x in re.split(", |\\(|\n|\\)| |/", query) if x.upper() and x.upper() not in valid_sql_tokens_and_types @@ -1783,7 +1778,7 @@ class ColumnAggregateExpectation(BatchExpectation, ABC): row_condition: Union[str, None] = None condition_parser: Union[ConditionParser, None] = None - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "batch_id", "table", "column", @@ -1794,7 +1789,7 @@ class ColumnAggregateExpectation(BatchExpectation, ABC): class Config: @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ColumnAggregateExpectation]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ColumnAggregateExpectation]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -1839,7 +1834,7 @@ class ColumnMapExpectation(BatchExpectation, ABC): catch_exceptions: bool = True map_metric: ClassVar[Optional[str]] = None - domain_keys: ClassVar[Tuple[str, ...]] = ( + domain_keys: ClassVar[tuple[str, ...]] = ( "batch_id", "table", "column", @@ -1847,11 +1842,11 @@ class ColumnMapExpectation(BatchExpectation, ABC): "condition_parser", ) domain_type: ClassVar[MetricDomainTypes] = MetricDomainTypes.COLUMN - success_keys: ClassVar[Tuple[str, ...]] = ("mostly",) + success_keys: ClassVar[tuple[str, ...]] = ("mostly",) class Config: @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ColumnMapExpectation]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ColumnMapExpectation]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -2005,7 +2000,7 @@ def get_validation_dependencies( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): @@ -2029,10 +2024,10 @@ def _validate( unexpected_count: Optional[int] = metrics.get( f"{self.map_metric}.{SummarizationMetricNameSuffixes.UNEXPECTED_COUNT.value}" ) - unexpected_values: Optional[List[Any]] = metrics.get( + unexpected_values: Optional[list[Any]] = metrics.get( f"{self.map_metric}.{SummarizationMetricNameSuffixes.UNEXPECTED_VALUES.value}" ) - unexpected_index_list: Optional[List[int]] = metrics.get( + unexpected_index_list: Optional[list[int]] = metrics.get( f"{self.map_metric}.{SummarizationMetricNameSuffixes.UNEXPECTED_INDEX_LIST.value}" ) unexpected_index_query: Optional[str] = metrics.get( @@ -2116,11 +2111,11 @@ class ColumnPairMapExpectation(BatchExpectation, ABC): "condition_parser", ) domain_type: ClassVar[MetricDomainTypes] = MetricDomainTypes.COLUMN_PAIR - success_keys: ClassVar[Tuple[str, ...]] = ("mostly",) + success_keys: ClassVar[tuple[str, ...]] = ("mostly",) class Config: @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[ColumnPairMapExpectation]) -> None: + def schema_extra(schema: dict[str, Any], model: type[ColumnPairMapExpectation]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -2273,11 +2268,11 @@ def get_validation_dependencies( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): - result_format: Union[Dict[str, Union[int, str, bool, List[str], None]], str] = ( + result_format: Union[dict[str, Union[int, str, bool, list[str], None]], str] = ( self._get_result_format(runtime_configuration=runtime_configuration) ) @@ -2291,7 +2286,7 @@ def _validate( unexpected_values: Optional[Any] = metrics.get( f"{self.map_metric}.{SummarizationMetricNameSuffixes.UNEXPECTED_VALUES.value}" ) - unexpected_index_list: Optional[List[int]] = metrics.get( + unexpected_index_list: Optional[list[int]] = metrics.get( f"{self.map_metric}.{SummarizationMetricNameSuffixes.UNEXPECTED_INDEX_LIST.value}" ) unexpected_index_query: Optional[str] = metrics.get( @@ -2355,7 +2350,7 @@ class MulticolumnMapExpectation(BatchExpectation, ABC): the expectation. """ # noqa: E501 - column_list: List[StrictStr] = pydantic.Field(description=COLUMN_LIST_DESCRIPTION) + column_list: list[StrictStr] = pydantic.Field(description=COLUMN_LIST_DESCRIPTION) mostly: MostlyField = 1 row_condition: Union[str, None] = None condition_parser: Union[ConditionParser, None] = None @@ -2378,7 +2373,7 @@ class MulticolumnMapExpectation(BatchExpectation, ABC): class Config: @staticmethod - def schema_extra(schema: Dict[str, Any], model: Type[MulticolumnMapExpectation]) -> None: + def schema_extra(schema: dict[str, Any], model: type[MulticolumnMapExpectation]) -> None: BatchExpectation.Config.schema_extra(schema, model) schema["properties"]["metadata"]["properties"].update( { @@ -2542,7 +2537,7 @@ def get_validation_dependencies( @override def _validate( self, - metrics: Dict, + metrics: dict, runtime_configuration: Optional[dict] = None, execution_engine: Optional[ExecutionEngine] = None, ): @@ -2558,7 +2553,7 @@ def _validate( unexpected_values: Optional[Any] = metrics.get( f"{self.map_metric}.{SummarizationMetricNameSuffixes.UNEXPECTED_VALUES.value}" ) - unexpected_index_list: Optional[List[int]] = metrics.get( + unexpected_index_list: Optional[list[int]] = metrics.get( f"{self.map_metric}.{SummarizationMetricNameSuffixes.UNEXPECTED_INDEX_LIST.value}" ) filtered_row_count: Optional[int] = metrics.get( @@ -2629,13 +2624,13 @@ def _format_map_output( # noqa: C901, PLR0912, PLR0913, PLR0915 element_count: Optional[int] = None, nonnull_count: Optional[int] = None, unexpected_count: Optional[int] = None, - unexpected_list: Optional[List[Any]] = None, - unexpected_index_list: Optional[List[int]] = None, + unexpected_list: Optional[list[Any]] = None, + unexpected_index_list: Optional[list[int]] = None, unexpected_index_query: Optional[str] = None, # Actually Optional[List[str]], but this is necessary to keep the typechecker happy - unexpected_index_column_names: Optional[Union[int, str, List[str]]] = None, + unexpected_index_column_names: Optional[Union[int, str, list[str]]] = None, unexpected_rows: pd.DataFrame | None = None, -) -> Dict: +) -> dict: """Helper function to construct expectation result objects for map_expectations (such as column_map_expectation and file_lines_map_expectation). @@ -2650,7 +2645,7 @@ def _format_map_output( # noqa: C901, PLR0912, PLR0913, PLR0915 # NB: unexpected_count parameter is explicit some implementing classes may limit the length of unexpected_list # noqa: E501 # Incrementally add to result and return when all values for the specified level are present - return_obj: Dict[str, Any] = {"success": success} + return_obj: dict[str, Any] = {"success": success} if result_format["result_format"] == ResultFormat.BOOLEAN_ONLY: return return_obj @@ -2728,7 +2723,7 @@ def _format_map_output( # noqa: C901, PLR0912, PLR0913, PLR0915 # Try to return the most common values, if possible. partial_unexpected_count: Optional[int] = result_format.get("partial_unexpected_count") - partial_unexpected_counts: Optional[List[Dict[str, Any]]] = None + partial_unexpected_counts: Optional[list[dict[str, Any]]] = None if partial_unexpected_count is not None and partial_unexpected_count > 0: try: if not exclude_unexpected_values: @@ -2774,7 +2769,7 @@ def _format_map_output( # noqa: C901, PLR0912, PLR0913, PLR0915 def _validate_dependencies_against_available_metrics( - validation_dependencies: List[MetricConfiguration], + validation_dependencies: list[MetricConfiguration], metrics: dict, ) -> None: """Check that validation_dependencies for current Expectations are available as Metrics. diff --git a/great_expectations/expectations/expectation_configuration.py b/great_expectations/expectations/expectation_configuration.py index 679c4fd84c44..0d8adfa77e0f 100644 --- a/great_expectations/expectations/expectation_configuration.py +++ b/great_expectations/expectations/expectation_configuration.py @@ -8,8 +8,6 @@ TYPE_CHECKING, Any, ClassVar, - Dict, - List, Mapping, Optional, Type, @@ -131,7 +129,7 @@ def __init__( # noqa: PLR0913 success_on_last_run: Optional[bool] = None, id: Optional[str] = None, expectation_context: Optional[ExpectationContext] = None, - rendered_content: Optional[List[RenderedAtomicContent]] = None, + rendered_content: Optional[list[RenderedAtomicContent]] = None, ) -> None: if not isinstance(type, str): raise InvalidExpectationConfigurationError("expectation_type must be a string") # noqa: TRY003 @@ -212,11 +210,11 @@ def kwargs(self, value: dict) -> None: self._kwargs = value @property - def rendered_content(self) -> Optional[List[RenderedAtomicContent]]: + def rendered_content(self) -> Optional[list[RenderedAtomicContent]]: return self._rendered_content @rendered_content.setter - def rendered_content(self, value: Optional[List[RenderedAtomicContent]]) -> None: + def rendered_content(self, value: Optional[list[RenderedAtomicContent]]) -> None: self._rendered_content = value def _get_default_custom_kwargs(self) -> KWargDetailsDict: @@ -417,7 +415,7 @@ def __str__(self): return json.dumps(self.to_json_dict(), indent=2) @override - def to_json_dict(self) -> Dict[str, JSONValues]: + def to_json_dict(self) -> dict[str, JSONValues]: """Returns a JSON-serializable dict representation of this ExpectationConfiguration. Returns: @@ -437,7 +435,7 @@ def to_json_dict(self) -> Dict[str, JSONValues]: myself["rendered_content"] = convert_to_json_serializable(myself["rendered_content"]) return myself - def _get_expectation_impl(self) -> Type[Expectation]: + def _get_expectation_impl(self) -> Type[Expectation]: # noqa: UP006 # needed to disambiguate from the type property return get_expectation_impl(self.type) def to_domain_obj(self) -> Expectation: diff --git a/great_expectations/expectations/metrics/column_aggregate_metric_provider.py b/great_expectations/expectations/metrics/column_aggregate_metric_provider.py index 50009490f9b7..654795522c2a 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metric_provider.py +++ b/great_expectations/expectations/metrics/column_aggregate_metric_provider.py @@ -2,7 +2,7 @@ import logging from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa from great_expectations.compatibility.typing_extensions import override @@ -38,7 +38,7 @@ def column_aggregate_value( - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], **kwargs, ): """Provides Pandas support for authoring a metric_fn with a simplified signature. @@ -67,7 +67,7 @@ def inner_func( # noqa: PLR0913 execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): filter_column_isnull = kwargs.get( @@ -102,7 +102,7 @@ def inner_func( # noqa: PLR0913 raise ValueError("column_aggregate_value decorator only supports PandasExecutionEngine") # noqa: TRY003, TRY004 -def column_aggregate_partial(engine: Type[ExecutionEngine], **kwargs): # noqa: C901 +def column_aggregate_partial(engine: type[ExecutionEngine], **kwargs): # noqa: C901 """Provides engine-specific support for authoring a metric_fn with a simplified signature. A column_aggregate_partial must provide an aggregate function; it will be executed with the specified engine @@ -136,7 +136,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): filter_column_isnull = kwargs.get( @@ -198,7 +198,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): filter_column_isnull = kwargs.get( diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_distinct_values.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_distinct_values.py index bafaaac49823..05fe0b5db239 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_distinct_values.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_distinct_values.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set +from typing import TYPE_CHECKING, Any, Optional from great_expectations.compatibility.pyspark import ( functions as F, @@ -35,23 +35,23 @@ class ColumnDistinctValues(ColumnAggregateMetricProvider): metric_name = "column.distinct_values" @column_aggregate_value(engine=PandasExecutionEngine) # type: ignore[misc] # untyped-decorator - def _pandas(cls, column: pd.Series, **kwargs) -> Set[Any]: + def _pandas(cls, column: pd.Series, **kwargs) -> set[Any]: return set(column.unique()) @metric_value(engine=SqlAlchemyExecutionEngine) def _sqlalchemy( cls, execution_engine: SqlAlchemyExecutionEngine, - metric_domain_kwargs: Dict[str, str], + metric_domain_kwargs: dict[str, str], **kwargs, - ) -> Set[Any]: + ) -> set[Any]: """ Past implementations of column.distinct_values depended on column.value_counts. This was causing performance issues due to the complex query used in column.value_counts and subsequent in-memory operations. """ # noqa: E501 selectable: sqlalchemy.Selectable - accessor_domain_kwargs: Dict[str, str] + accessor_domain_kwargs: dict[str, str] ( selectable, _, @@ -60,7 +60,7 @@ def _sqlalchemy( column_name: str = accessor_domain_kwargs["column"] column: sqlalchemy.ColumnClause = sa.column(column_name) - distinct_values: List[sqlalchemy.Row] + distinct_values: list[sqlalchemy.Row] if hasattr(column, "is_not"): distinct_values = execution_engine.execute_query( # type: ignore[assignment] sa.select(column).where(column.is_not(None)).distinct().select_from(selectable) # type: ignore[arg-type] @@ -76,23 +76,23 @@ def _sqlalchemy( def _spark( cls, execution_engine: SparkDFExecutionEngine, - metric_domain_kwargs: Dict[str, str], + metric_domain_kwargs: dict[str, str], **kwargs, - ) -> Set[Any]: + ) -> set[Any]: """ Past implementations of column.distinct_values depended on column.value_counts. This was causing performance issues due to the complex query used in column.value_counts and subsequent in-memory operations. """ # noqa: E501 df: pyspark.DataFrame - accessor_domain_kwargs: Dict[str, str] + accessor_domain_kwargs: dict[str, str] ( df, _, accessor_domain_kwargs, ) = execution_engine.get_compute_domain(metric_domain_kwargs, MetricDomainTypes.COLUMN) column_name: str = accessor_domain_kwargs["column"] - distinct_values: List[pyspark.Row] = ( + distinct_values: list[pyspark.Row] = ( df.select(F.col(column_name)) .distinct() .where(F.col(column_name).isNotNull()) @@ -147,8 +147,8 @@ def _pandas(cls, column: pd.Series, threshold: int, **kwargs) -> bool: @metric_value(engine=SqlAlchemyExecutionEngine) def _sqlalchemy( cls, - metric_value_kwargs: Dict[str, int], - metrics: Dict[str, int], + metric_value_kwargs: dict[str, int], + metrics: dict[str, int], **kwargs, ) -> bool: return metrics["column.distinct_values.count"] < metric_value_kwargs["threshold"] @@ -156,8 +156,8 @@ def _sqlalchemy( @metric_value(engine=SparkDFExecutionEngine) def _spark( cls, - metric_value_kwargs: Dict[str, int], - metrics: Dict[str, int], + metric_value_kwargs: dict[str, int], + metrics: dict[str, int], **kwargs, ) -> bool: return metrics["column.distinct_values.count"] < metric_value_kwargs["threshold"] @@ -169,7 +169,7 @@ def _get_evaluation_dependencies( metric: MetricConfiguration, configuration: Optional[ExpectationConfiguration] = None, execution_engine: Optional[ExecutionEngine] = None, - runtime_configuration: Optional[Dict] = None, + runtime_configuration: Optional[dict] = None, ): """Returns a dictionary of given metric names and their corresponding configuration, specifying the metric types and their respective domains""" diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_histogram.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_histogram.py index 8f874b02d5e8..4b33cbefdc91 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_histogram.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_histogram.py @@ -2,7 +2,7 @@ import copy import logging -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING, Any import numpy as np @@ -40,7 +40,7 @@ def _pandas( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): df, _, accessor_domain_kwargs = execution_engine.get_compute_domain( @@ -60,7 +60,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): """return a list of counts corresponding to bins @@ -213,7 +213,7 @@ def _spark( # noqa: C901 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): df, _, _accessor_domain_kwargs = execution_engine.get_compute_domain( diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_median.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_median.py index 0ffd6da68e08..d861b78e3699 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_median.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_median.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Optional import numpy as np @@ -46,7 +46,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): ( @@ -98,7 +98,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): ( diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_most_common_value.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_most_common_value.py index 3d73be625a15..ad0b4c33b2cf 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_most_common_value.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_most_common_value.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.execution_engine import ( @@ -36,7 +36,7 @@ def _spark( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): column_value_counts = metrics["column.value_counts"] @@ -48,7 +48,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): column_value_counts = metrics["column.value_counts"] @@ -61,7 +61,7 @@ def _get_evaluation_dependencies( metric: MetricConfiguration, configuration: Optional[ExpectationConfiguration] = None, execution_engine: Optional[ExecutionEngine] = None, - runtime_configuration: Optional[Dict] = None, + runtime_configuration: Optional[dict] = None, ): """Returns a dictionary of given metric names and their corresponding configuration, specifying the metric types and their respective domains""" diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_partition.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_partition.py index 79119c3e6386..b978bb773216 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_partition.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_partition.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Literal, Optional +from typing import TYPE_CHECKING, Any, Literal, Optional import numpy as np @@ -41,7 +41,7 @@ def _pandas( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): bins = metric_value_kwargs.get("bins", cls.default_kwarg_values["bins"]) @@ -54,7 +54,7 @@ def _sqlalchemy( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): bins = metric_value_kwargs.get("bins", cls.default_kwarg_values["bins"]) @@ -67,7 +67,7 @@ def _spark( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): bins = metric_value_kwargs.get("bins", cls.default_kwarg_values["bins"]) diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_value_counts.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_value_counts.py index ca3d5ba85237..a5ec01e9d009 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_value_counts.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_value_counts.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Iterable, Optional import pandas as pd @@ -31,8 +31,8 @@ class ColumnValueCounts(ColumnAggregateMetricProvider): def _pandas( cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict[str, str], - metric_value_kwargs: Dict[str, Optional[str]], + metric_domain_kwargs: dict[str, str], + metric_value_kwargs: dict[str, Optional[str]], **kwargs, ) -> pd.Series: sort: str = metric_value_kwargs.get("sort") or cls.default_kwarg_values["sort"] @@ -46,7 +46,7 @@ def _pandas( raise ValueError("collate parameter is not supported in PandasDataset") # noqa: TRY003 df: pd.DataFrame - accessor_domain_kwargs: Dict[str, str] + accessor_domain_kwargs: dict[str, str] df, _, accessor_domain_kwargs = execution_engine.get_compute_domain( metric_domain_kwargs, MetricDomainTypes.COLUMN ) @@ -73,8 +73,8 @@ def _pandas( def _sqlalchemy( cls, execution_engine: SqlAlchemyExecutionEngine, - metric_domain_kwargs: Dict[str, str], - metric_value_kwargs: Dict[str, Optional[str]], + metric_domain_kwargs: dict[str, str], + metric_value_kwargs: dict[str, Optional[str]], **kwargs, ) -> pd.Series: sort: str = metric_value_kwargs.get("sort") or cls.default_kwarg_values["sort"] @@ -88,7 +88,7 @@ def _sqlalchemy( raise ValueError("collate parameter is not supported in PandasDataset") # noqa: TRY003 selectable: sqlalchemy.Selectable - accessor_domain_kwargs: Dict[str, str] + accessor_domain_kwargs: dict[str, str] selectable, _, accessor_domain_kwargs = execution_engine.get_compute_domain( metric_domain_kwargs, MetricDomainTypes.COLUMN ) @@ -124,7 +124,7 @@ def _sqlalchemy( query = query.order_by(sa.column(column)) elif sort == "count": query = query.order_by(sa.column("count").desc()) - results: List[sqlalchemy.Row] = execution_engine.execute_query( # type: ignore[assignment] + results: list[sqlalchemy.Row] = execution_engine.execute_query( # type: ignore[assignment] query.select_from(selectable) # type: ignore[arg-type] ).fetchall() # Numpy does not always infer the correct DataTypes for SqlAlchemy Row, so we cannot use vectorized approach. # noqa: E501 @@ -139,8 +139,8 @@ def _sqlalchemy( def _spark( cls, execution_engine: SparkDFExecutionEngine, - metric_domain_kwargs: Dict[str, str], - metric_value_kwargs: Dict[str, Optional[str]], + metric_domain_kwargs: dict[str, str], + metric_value_kwargs: dict[str, Optional[str]], **kwargs, ) -> pd.Series: sort: str = metric_value_kwargs.get("sort") or cls.default_kwarg_values["sort"] @@ -154,7 +154,7 @@ def _spark( raise ValueError("collate parameter is not supported in SparkDFDataset") # noqa: TRY003 df: pyspark.DataFrame - accessor_domain_kwargs: Dict[str, str] + accessor_domain_kwargs: dict[str, str] df, _, accessor_domain_kwargs = execution_engine.get_compute_domain( metric_domain_kwargs, MetricDomainTypes.COLUMN ) @@ -169,7 +169,7 @@ def _spark( elif sort == "count": value_counts_df = value_counts_df.orderBy(F.desc("count")) - value_counts: List[pyspark.Row] = value_counts_df.collect() + value_counts: list[pyspark.Row] = value_counts_df.collect() # Numpy does not always infer the correct DataTypes for Spark df, so we cannot use vectorized approach. # noqa: E501 values: Iterable[Any] diff --git a/great_expectations/expectations/metrics/column_aggregate_metrics/column_values_between_count.py b/great_expectations/expectations/metrics/column_aggregate_metrics/column_values_between_count.py index 231e1b7ab1d2..c9d48f033827 100644 --- a/great_expectations/expectations/metrics/column_aggregate_metrics/column_values_between_count.py +++ b/great_expectations/expectations/metrics/column_aggregate_metrics/column_values_between_count.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any import numpy as np @@ -35,7 +35,7 @@ def _pandas( # noqa: C901, PLR0912 execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): min_value = metric_value_kwargs.get("min_value") @@ -89,7 +89,7 @@ def _sqlalchemy( # noqa: C901, PLR0912 execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): min_value = metric_value_kwargs.get("min_value") @@ -188,7 +188,7 @@ def _spark( # noqa: C901, PLR0912 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): min_value = metric_value_kwargs.get("min_value") diff --git a/great_expectations/expectations/metrics/column_map_metrics/column_values_decreasing.py b/great_expectations/expectations/metrics/column_map_metrics/column_values_decreasing.py index ef21daca6017..3498657d799a 100644 --- a/great_expectations/expectations/metrics/column_map_metrics/column_values_decreasing.py +++ b/great_expectations/expectations/metrics/column_map_metrics/column_values_decreasing.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any from great_expectations.compatibility import pyspark from great_expectations.compatibility.pyspark import functions as F @@ -50,7 +50,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): # check if column is any type that could have na (numeric types) diff --git a/great_expectations/expectations/metrics/column_map_metrics/column_values_increasing.py b/great_expectations/expectations/metrics/column_map_metrics/column_values_increasing.py index 4cc02cd36a6d..a233f1739283 100644 --- a/great_expectations/expectations/metrics/column_map_metrics/column_values_increasing.py +++ b/great_expectations/expectations/metrics/column_map_metrics/column_values_increasing.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any from great_expectations.compatibility import pyspark from great_expectations.compatibility.pyspark import functions as F @@ -51,7 +51,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): # check if column is any type that could have na (numeric types) diff --git a/great_expectations/expectations/metrics/map_metric_provider/column_condition_partial.py b/great_expectations/expectations/metrics/map_metric_provider/column_condition_partial.py index 17046a1c337b..2b4a280af38a 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/column_condition_partial.py +++ b/great_expectations/expectations/metrics/map_metric_provider/column_condition_partial.py @@ -6,9 +6,7 @@ TYPE_CHECKING, Any, Callable, - Dict, Optional, - Type, Union, ) @@ -39,7 +37,7 @@ def column_condition_partial( # noqa: C901, PLR0915 - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], partial_fn_type: Optional[MetricPartialFunctionTypes] = None, **kwargs, ): @@ -86,7 +84,7 @@ def inner_func( # noqa: PLR0913 execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -155,7 +153,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -240,7 +238,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( diff --git a/great_expectations/expectations/metrics/map_metric_provider/column_function_partial.py b/great_expectations/expectations/metrics/map_metric_provider/column_function_partial.py index 7b658f44fe8d..329c514be8f9 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/column_function_partial.py +++ b/great_expectations/expectations/metrics/map_metric_provider/column_function_partial.py @@ -6,9 +6,7 @@ TYPE_CHECKING, Any, Callable, - Dict, Optional, - Type, Union, ) @@ -37,7 +35,7 @@ def column_function_partial( # noqa: C901, PLR0915 - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], partial_fn_type: Optional[MetricPartialFunctionTypes] = None, **kwargs, ): @@ -80,7 +78,7 @@ def inner_func( # noqa: PLR0913 execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -140,7 +138,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): filter_column_isnull = kwargs.get( @@ -212,7 +210,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): filter_column_isnull = kwargs.get( diff --git a/great_expectations/expectations/metrics/map_metric_provider/column_map_condition_auxilliary_methods.py b/great_expectations/expectations/metrics/map_metric_provider/column_map_condition_auxilliary_methods.py index 6023943d644a..a8ddd27add6e 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/column_map_condition_auxilliary_methods.py +++ b/great_expectations/expectations/metrics/map_metric_provider/column_map_condition_auxilliary_methods.py @@ -4,9 +4,7 @@ from typing import ( TYPE_CHECKING, Any, - Dict, Sequence, - Tuple, Union, ) @@ -44,7 +42,7 @@ def _pandas_column_map_condition_values( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[dict]: """Return values from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -101,7 +99,7 @@ def _pandas_column_map_condition_value_counts( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> pd.Series[int]: """Returns respective value counts for distinct column values""" @@ -166,7 +164,7 @@ def _sqlalchemy_column_map_condition_values( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Tuple], + metrics: dict[str, tuple], **kwargs, ) -> list[dict]: """ @@ -225,7 +223,7 @@ def _sqlalchemy_column_map_condition_value_counts( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> Union[Sequence[sa.Row[Any]], Any]: """ @@ -266,7 +264,7 @@ def _spark_column_map_condition_values( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[dict]: """Return values from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -314,7 +312,7 @@ def _spark_column_map_condition_value_counts( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[pyspark.Row]: unexpected_condition, compute_domain_kwargs, accessor_domain_kwargs = metrics[ diff --git a/great_expectations/expectations/metrics/map_metric_provider/column_pair_condition_partial.py b/great_expectations/expectations/metrics/map_metric_provider/column_pair_condition_partial.py index 6bbdef7c2a86..ac40b057d91c 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/column_pair_condition_partial.py +++ b/great_expectations/expectations/metrics/map_metric_provider/column_pair_condition_partial.py @@ -6,9 +6,7 @@ TYPE_CHECKING, Any, Callable, - Dict, Optional, - Type, ) from great_expectations.compatibility.sqlalchemy import ( @@ -38,7 +36,7 @@ def column_pair_condition_partial( # noqa: C901 - 16 - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], partial_fn_type: Optional[MetricPartialFunctionTypes] = None, **kwargs, ): @@ -85,7 +83,7 @@ def inner_func( # noqa: PLR0913 execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -153,7 +151,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -227,7 +225,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( diff --git a/great_expectations/expectations/metrics/map_metric_provider/column_pair_function_partial.py b/great_expectations/expectations/metrics/map_metric_provider/column_pair_function_partial.py index 2c7949cb7e7e..96886a8d8e4b 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/column_pair_function_partial.py +++ b/great_expectations/expectations/metrics/map_metric_provider/column_pair_function_partial.py @@ -5,8 +5,6 @@ from typing import ( Any, Callable, - Dict, - Type, ) from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa @@ -31,7 +29,7 @@ def column_pair_function_partial( # noqa: C901 - 16 - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], partial_fn_type: MetricPartialFunctionTypes | None = None, **kwargs, ): @@ -75,7 +73,7 @@ def inner_func( # noqa: PLR0913 execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -135,7 +133,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -197,7 +195,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( diff --git a/great_expectations/expectations/metrics/map_metric_provider/column_pair_map_condition_auxilliary_methods.py b/great_expectations/expectations/metrics/map_metric_provider/column_pair_map_condition_auxilliary_methods.py index ae061aa13c66..0149109bbd84 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/column_pair_map_condition_auxilliary_methods.py +++ b/great_expectations/expectations/metrics/map_metric_provider/column_pair_map_condition_auxilliary_methods.py @@ -4,8 +4,6 @@ from typing import ( TYPE_CHECKING, Any, - Dict, - List, Union, ) @@ -39,7 +37,7 @@ def _pandas_column_pair_map_condition_values( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[tuple[Any, Any]]: """Return values from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -73,7 +71,7 @@ def _pandas_column_pair_map_condition_values( # noinspection PyPep8Naming column_B_name = accessor_domain_kwargs["column_B"] - column_names: List[Union[str, sqlalchemy.quoted_name]] = [ + column_names: list[Union[str, sqlalchemy.quoted_name]] = [ column_A_name, column_B_name, ] @@ -104,7 +102,7 @@ def _pandas_column_pair_map_condition_filtered_row_count( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> int: """Return record counts from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -137,7 +135,7 @@ def _sqlalchemy_column_pair_map_condition_values( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[tuple[Any, Any]]: """Return values from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -189,7 +187,7 @@ def _sqlalchemy_column_pair_map_condition_filtered_row_count( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> Any | None: """Return record counts from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -217,7 +215,7 @@ def _spark_column_pair_map_condition_values( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[tuple[Any, Any]]: """Return values from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -276,7 +274,7 @@ def _spark_column_pair_map_condition_filtered_row_count( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> int: """Return record counts from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 diff --git a/great_expectations/expectations/metrics/map_metric_provider/column_pair_map_metric_provider.py b/great_expectations/expectations/metrics/map_metric_provider/column_pair_map_metric_provider.py index ef02e45ee108..d55277661bdb 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/column_pair_map_metric_provider.py +++ b/great_expectations/expectations/metrics/map_metric_provider/column_pair_map_metric_provider.py @@ -4,7 +4,6 @@ from typing import ( TYPE_CHECKING, Optional, - Tuple, ) from great_expectations.compatibility.typing_extensions import override @@ -29,7 +28,7 @@ class ColumnPairMapMetricProvider(MapMetricProvider): `expect_column_pair_values_to_be_equal` is an example of an Expectation that uses this metric. """ # noqa: E501 - condition_domain_keys: Tuple[str, ...] = ( + condition_domain_keys: tuple[str, ...] = ( "batch_id", "table", "column_A", diff --git a/great_expectations/expectations/metrics/map_metric_provider/is_sqlalchemy_metric_selectable.py b/great_expectations/expectations/metrics/map_metric_provider/is_sqlalchemy_metric_selectable.py index a8a280f23171..40a55952349f 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/is_sqlalchemy_metric_selectable.py +++ b/great_expectations/expectations/metrics/map_metric_provider/is_sqlalchemy_metric_selectable.py @@ -3,7 +3,6 @@ import logging from typing import ( TYPE_CHECKING, - Set, ) if TYPE_CHECKING: @@ -11,7 +10,7 @@ logger = logging.getLogger(__name__) -SQLALCHEMY_SELECTABLE_METRICS: Set[str] = { +SQLALCHEMY_SELECTABLE_METRICS: set[str] = { "compound_columns.count", "compound_columns.unique", } diff --git a/great_expectations/expectations/metrics/map_metric_provider/map_condition_auxilliary_methods.py b/great_expectations/expectations/metrics/map_metric_provider/map_condition_auxilliary_methods.py index 7199105c87aa..9c8addea2af3 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/map_condition_auxilliary_methods.py +++ b/great_expectations/expectations/metrics/map_metric_provider/map_condition_auxilliary_methods.py @@ -4,8 +4,6 @@ from typing import ( TYPE_CHECKING, Any, - Dict, - List, Optional, Sequence, Union, @@ -58,7 +56,7 @@ def _pandas_map_condition_unexpected_count( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> int: """Returns unexpected count for MapExpectations""" @@ -70,9 +68,9 @@ def _pandas_map_condition_index( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, -) -> Union[List[int], List[Dict[str, Any]]]: +) -> Union[list[int], list[dict[str, Any]]]: ( boolean_mapped_unexpected_values, compute_domain_kwargs, @@ -92,7 +90,7 @@ def _pandas_map_condition_index( domain_records_df: pd.DataFrame = execution_engine.get_domain_records( domain_kwargs=domain_kwargs ) - domain_column_name_list: List[str] = list() + domain_column_name_list: list[str] = list() # column map expectations if "column" in accessor_domain_kwargs: column_name: Union[str, sqlalchemy.quoted_name] = accessor_domain_kwargs["column"] @@ -112,7 +110,7 @@ def _pandas_map_condition_index( # column pair expectations elif "column_A" in accessor_domain_kwargs and "column_B" in accessor_domain_kwargs: - column_list: List[Union[str, sqlalchemy.quoted_name]] = list() + column_list: list[Union[str, sqlalchemy.quoted_name]] = list() column_list.append(accessor_domain_kwargs["column_A"]) column_list.append(accessor_domain_kwargs["column_B"]) domain_column_name_list = column_list @@ -125,7 +123,7 @@ def _pandas_map_condition_index( result_format = metric_value_kwargs["result_format"] domain_records_df = domain_records_df[boolean_mapped_unexpected_values] - unexpected_index_list: Union[List[int], List[Dict[str, Any]]] = ( + unexpected_index_list: Union[list[int], list[dict[str, Any]]] = ( compute_unexpected_pandas_indices( domain_records_df=domain_records_df, result_format=result_format, @@ -142,9 +140,9 @@ def _pandas_map_condition_index( def _pandas_map_condition_query( cls, execution_engine: PandasExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], **kwargs, ) -> Optional[str]: """ @@ -202,7 +200,7 @@ def _pandas_map_condition_rows( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> pd.DataFrame: """Return values from the specified domain (ignoring the column constraint) that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -254,7 +252,7 @@ def _sqlalchemy_map_condition_unexpected_count_aggregate_fn( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> tuple[Any, Any, Any]: """Returns unexpected count for MapExpectations""" @@ -279,7 +277,7 @@ def _sqlalchemy_map_condition_unexpected_count_value( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> float | int: """Returns unexpected count for MapExpectations. This is a *value* metric, which is useful for @@ -296,7 +294,7 @@ def _sqlalchemy_map_condition_unexpected_count_value( selectable = execution_engine.get_domain_records(domain_kwargs=domain_kwargs) # The integral values are cast to SQL Numeric in order to avoid a bug in AWS Redshift (converted to integer later). # noqa: E501 - count_case_statement: List[sqlalchemy.Label] = sa.case( # type: ignore[assignment] + count_case_statement: list[sqlalchemy.Label] = sa.case( # type: ignore[assignment] ( unexpected_condition, sa.sql.expression.cast(1, sa.Numeric), @@ -369,7 +367,7 @@ def _sqlalchemy_map_condition_rows( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> Union[Sequence[sa.Row[Any]], Any]: """ @@ -407,9 +405,9 @@ def _sqlalchemy_map_condition_rows( def _sqlalchemy_map_condition_query( # noqa: C901 - too complex cls, execution_engine: SqlAlchemyExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], **kwargs, ) -> Optional[str]: """ @@ -433,14 +431,14 @@ def _sqlalchemy_map_condition_query( # noqa: C901 - too complex if return_unexpected_index_query is False: return None - domain_column_name_list: List[str] = list() + domain_column_name_list: list[str] = list() # column map expectations if "column" in accessor_domain_kwargs: column_name: Union[str, sqlalchemy.quoted_name] = accessor_domain_kwargs["column"] domain_column_name_list.append(column_name) # multi-column map expectations elif "column_list" in accessor_domain_kwargs: - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] domain_column_name_list = column_list @@ -451,10 +449,10 @@ def _sqlalchemy_map_condition_query( # noqa: C901 - too complex column_list.append(accessor_domain_kwargs["column_B"]) domain_column_name_list = column_list - column_selector: List[sa.Column] = [] + column_selector: list[sa.Column] = [] - all_table_columns: List[str] = metrics.get("table.columns", []) - unexpected_index_column_names: List[str] | None = result_format.get( + all_table_columns: list[str] = metrics.get("table.columns", []) + unexpected_index_column_names: list[str] | None = result_format.get( "unexpected_index_column_names" ) if unexpected_index_column_names: @@ -493,9 +491,9 @@ def _sqlalchemy_map_condition_query( # noqa: C901 - too complex def _sqlalchemy_map_condition_index( # noqa: C901 - too complex cls, execution_engine: SqlAlchemyExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], **kwargs, ) -> list[dict[str, Any]] | None: """ @@ -515,14 +513,14 @@ def _sqlalchemy_map_condition_index( # noqa: C901 - too complex if "unexpected_index_column_names" not in result_format: return None - domain_column_name_list: List[str] = list() + domain_column_name_list: list[str] = list() # column map expectations if "column" in accessor_domain_kwargs: column_name: Union[str, sqlalchemy.quoted_name] = accessor_domain_kwargs["column"] domain_column_name_list.append(column_name) # multi-column map expectations elif "column_list" in accessor_domain_kwargs: - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] domain_column_name_list = column_list @@ -534,13 +532,13 @@ def _sqlalchemy_map_condition_index( # noqa: C901 - too complex domain_column_name_list = column_list domain_kwargs: dict = dict(**compute_domain_kwargs, **accessor_domain_kwargs) - all_table_columns: List[str] = metrics.get("table.columns", []) + all_table_columns: list[str] = metrics.get("table.columns", []) - unexpected_index_column_names: List[str] = result_format.get( + unexpected_index_column_names: list[str] = result_format.get( "unexpected_index_column_names", [] ) - column_selector: List[sa.Column] = [] + column_selector: list[sa.Column] = [] for column_name in unexpected_index_column_names: if column_name not in all_table_columns: raise gx_exceptions.InvalidMetricAccessorDomainKwargsKeyError( @@ -567,7 +565,7 @@ def _sqlalchemy_map_condition_index( # noqa: C901 - too complex final_query: sa.select = unexpected_condition_query_with_selected_columns.select_from( # type: ignore[valid-type,attr-defined] domain_records_as_selectable ).limit(result_format["partial_unexpected_count"]) - query_result: List[sqlalchemy.Row] = execution_engine.execute_query(final_query).fetchall() # type: ignore[assignment] + query_result: list[sqlalchemy.Row] = execution_engine.execute_query(final_query).fetchall() # type: ignore[assignment] exclude_unexpected_values: bool = result_format.get("exclude_unexpected_values", False) @@ -584,7 +582,7 @@ def _spark_map_condition_unexpected_count_aggregate_fn( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> tuple[Any, Any, Any]: unexpected_condition, compute_domain_kwargs, accessor_domain_kwargs = metrics[ @@ -602,7 +600,7 @@ def _spark_map_condition_unexpected_count_value( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> int: # fn_domain_kwargs maybe updated to reflect null filtering @@ -630,7 +628,7 @@ def _spark_map_condition_rows( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[pyspark.Row]: unexpected_condition, compute_domain_kwargs, accessor_domain_kwargs = metrics[ @@ -661,11 +659,11 @@ def _spark_map_condition_rows( def _spark_map_condition_index( # noqa: C901 - too complex cls, execution_engine: SparkDFExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], **kwargs, -) -> Union[List[Dict[str, Any]], None]: +) -> Union[list[dict[str, Any]], None]: """ Returns indices of the metric values which do not meet an expected Expectation condition for instances of ColumnMapExpectation. @@ -686,7 +684,7 @@ def _spark_map_condition_index( # noqa: C901 - too complex if "unexpected_index_column_names" not in result_format: return None - domain_column_name_list: List[str] = list() + domain_column_name_list: list[str] = list() # column map expectations if "column" in accessor_domain_kwargs: column_name: Union[str, sqlalchemy.quoted_name] = accessor_domain_kwargs["column"] @@ -694,7 +692,7 @@ def _spark_map_condition_index( # noqa: C901 - too complex # multi-column map expectations elif "column_list" in accessor_domain_kwargs: - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] domain_column_name_list = column_list @@ -722,8 +720,8 @@ def _spark_map_condition_index( # noqa: C901 - too complex ) exclude_unexpected_values: bool = result_format.get("exclude_unexpected_values", False) - unexpected_index_column_names: List[str] = result_format["unexpected_index_column_names"] - columns_to_keep: List[str] = [column for column in unexpected_index_column_names] + unexpected_index_column_names: list[str] = result_format["unexpected_index_column_names"] + columns_to_keep: list[str] = [column for column in unexpected_index_column_names] columns_to_keep += domain_column_name_list # check that column name is in row @@ -750,9 +748,9 @@ def _spark_map_condition_index( # noqa: C901 - too complex def _spark_map_condition_query( cls, execution_engine: SparkDFExecutionEngine, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], **kwargs, ) -> Union[str, None]: """ @@ -791,9 +789,9 @@ def _spark_map_condition_query( def _generate_temp_table( connection: sa.engine.base.Connection, - metric_domain_kwargs: Dict, - metric_value_kwargs: Dict, - metrics: Dict[str, Any], + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], **kwargs, ) -> sa.Table: temp_table_name: str = generate_temporary_table_name(default_table_name_prefix="#ge_temp_") @@ -810,18 +808,18 @@ def _generate_temp_table( def _get_sqlalchemy_customized_unexpected_index_list( exclude_unexpected_values: bool, - unexpected_index_column_names: List[str], - query_result: List[sqlalchemy.Row], - domain_column_name_list: List[Union[str, sqlalchemy.quoted_name]], -) -> Union[List[Dict[str, Any]], None]: - unexpected_index_list: List[Dict[str, Any]] = [] + unexpected_index_column_names: list[str], + query_result: list[sqlalchemy.Row], + domain_column_name_list: list[Union[str, sqlalchemy.quoted_name]], +) -> Union[list[dict[str, Any]], None]: + unexpected_index_list: list[dict[str, Any]] = [] if ( exclude_unexpected_values and len(query_result) != 0 and len(unexpected_index_column_names) != 0 ): - primary_key_dict_list: dict[str, List[Any]] = { + primary_key_dict_list: dict[str, list[Any]] = { idx_col: [] for idx_col in unexpected_index_column_names } for row in query_result: @@ -831,7 +829,7 @@ def _get_sqlalchemy_customized_unexpected_index_list( else: for row in query_result: - primary_key_dict: Dict[str, Any] = {} + primary_key_dict: dict[str, Any] = {} # add the actual unexpected value all_columns = unexpected_index_column_names + domain_column_name_list for index in range(len(all_columns)): @@ -844,14 +842,14 @@ def _get_sqlalchemy_customized_unexpected_index_list( def _get_spark_customized_unexpected_index_list( exclude_unexpected_values: bool, - unexpected_index_column_names: List[str], + unexpected_index_column_names: list[str], filtered: pyspark.sql.dataframe.DataFrame, - columns_to_keep: List[str], -) -> Union[List[Dict[str, Any]], None]: - unexpected_index_list: List[Dict[str, Any]] = [] + columns_to_keep: list[str], +) -> Union[list[dict[str, Any]], None]: + unexpected_index_list: list[dict[str, Any]] = [] if exclude_unexpected_values and not filtered.isEmpty(): - dict_list_to_add: dict[str, List[Any]] = { + dict_list_to_add: dict[str, list[Any]] = { idx_col: [] for idx_col in unexpected_index_column_names } for row in filtered.collect(): diff --git a/great_expectations/expectations/metrics/map_metric_provider/multicolumn_condition_partial.py b/great_expectations/expectations/metrics/map_metric_provider/multicolumn_condition_partial.py index 098968e28ef0..36df2661790c 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/multicolumn_condition_partial.py +++ b/great_expectations/expectations/metrics/map_metric_provider/multicolumn_condition_partial.py @@ -6,10 +6,7 @@ TYPE_CHECKING, Any, Callable, - Dict, - List, Optional, - Type, Union, ) @@ -40,7 +37,7 @@ def multicolumn_condition_partial( # noqa: C901 - 16 - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], partial_fn_type: Optional[MetricPartialFunctionTypes] = None, **kwargs, ): @@ -87,7 +84,7 @@ def inner_func( # noqa: PLR0913 execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -103,7 +100,7 @@ def inner_func( # noqa: PLR0913 domain_kwargs=metric_domain_kwargs, domain_type=domain_type ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] @@ -153,7 +150,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -169,7 +166,7 @@ def inner_func( # noqa: PLR0913 domain_kwargs=metric_domain_kwargs, domain_type=domain_type ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] @@ -226,7 +223,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -242,7 +239,7 @@ def inner_func( # noqa: PLR0913 domain_kwargs=metric_domain_kwargs, domain_type=domain_type ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] diff --git a/great_expectations/expectations/metrics/map_metric_provider/multicolumn_function_partial.py b/great_expectations/expectations/metrics/map_metric_provider/multicolumn_function_partial.py index 8034d25102bd..c2808429c5a6 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/multicolumn_function_partial.py +++ b/great_expectations/expectations/metrics/map_metric_provider/multicolumn_function_partial.py @@ -6,10 +6,7 @@ TYPE_CHECKING, Any, Callable, - Dict, - List, Optional, - Type, Union, ) @@ -40,7 +37,7 @@ def multicolumn_function_partial( # noqa: C901 - 16 - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], partial_fn_type: Optional[MetricPartialFunctionTypes] = None, **kwargs, ): @@ -82,7 +79,7 @@ def inner_func( # noqa: PLR0913 execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -98,7 +95,7 @@ def inner_func( # noqa: PLR0913 domain_kwargs=metric_domain_kwargs, domain_type=domain_type ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] @@ -140,7 +137,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -156,7 +153,7 @@ def inner_func( # noqa: PLR0913 domain_kwargs=metric_domain_kwargs, domain_type=domain_type ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] @@ -211,7 +208,7 @@ def inner_func( # noqa: PLR0913 execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): metric_domain_kwargs = get_dbms_compatible_metric_domain_kwargs( @@ -227,7 +224,7 @@ def inner_func( # noqa: PLR0913 domain_kwargs=metric_domain_kwargs, domain_type=domain_type ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs[ "column_list" ] diff --git a/great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_condition_auxilliary_methods.py b/great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_condition_auxilliary_methods.py index c78fecc85ad6..1f75671faab3 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_condition_auxilliary_methods.py +++ b/great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_condition_auxilliary_methods.py @@ -4,8 +4,6 @@ from typing import ( TYPE_CHECKING, Any, - Dict, - List, Union, ) @@ -39,7 +37,7 @@ def _pandas_multicolumn_map_condition_values( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[dict]: """Return values from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -68,7 +66,7 @@ def _pandas_multicolumn_map_condition_values( """ # noqa: E501 ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs["column_list"] + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs["column_list"] domain_values = df[column_list] @@ -90,7 +88,7 @@ def _pandas_multicolumn_map_condition_filtered_row_count( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> int: """Return record counts from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -123,7 +121,7 @@ def _sqlalchemy_multicolumn_map_condition_values( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[dict]: """Return values from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -152,7 +150,7 @@ def _sqlalchemy_multicolumn_map_condition_values( """ # noqa: E501 ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs["column_list"] + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs["column_list"] column_selector = [sa.column(column_name) for column_name in column_list] # type: ignore[var-annotated] query = sa.select(*column_selector).where(boolean_mapped_unexpected_values) @@ -175,7 +173,7 @@ def _sqlalchemy_multicolumn_map_condition_filtered_row_count( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> Any | None: """Return record counts from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -212,7 +210,7 @@ def _spark_multicolumn_map_condition_values( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> list[dict]: """Return values from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 @@ -241,7 +239,7 @@ def _spark_multicolumn_map_condition_values( """ # noqa: E501 ) - column_list: List[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs["column_list"] + column_list: list[Union[str, sqlalchemy.quoted_name]] = accessor_domain_kwargs["column_list"] # withColumn is required to transform window functions returned by some metrics to boolean mask data = df.withColumn("__unexpected", unexpected_condition) @@ -268,7 +266,7 @@ def _spark_multicolumn_map_condition_filtered_row_count( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], **kwargs, ) -> int: """Return record counts from the specified domain that match the map-style metric in the metrics dictionary.""" # noqa: E501 diff --git a/great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_metric_provider.py b/great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_metric_provider.py index 5d9751e1550b..8bb8a392221c 100644 --- a/great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_metric_provider.py +++ b/great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_metric_provider.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.expectations.metrics.map_metric_provider import ( @@ -25,7 +25,7 @@ class MulticolumnMapMetricProvider(MapMetricProvider): `expect_compound_columns_to_be_unique` is an example of an Expectation that uses this metric. """ # noqa: E501 - condition_domain_keys: Tuple[str, ...] = ( + condition_domain_keys: tuple[str, ...] = ( "batch_id", "table", "column_list", diff --git a/great_expectations/expectations/metrics/metric_provider.py b/great_expectations/expectations/metrics/metric_provider.py index 568b5e031958..f25406e71a1c 100644 --- a/great_expectations/expectations/metrics/metric_provider.py +++ b/great_expectations/expectations/metrics/metric_provider.py @@ -2,7 +2,7 @@ import logging from functools import wraps -from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type, TypeVar, Union +from typing import TYPE_CHECKING, Callable, Optional, TypeVar, Union from typing_extensions import ParamSpec @@ -34,7 +34,7 @@ def metric_value( - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], metric_fn_type: Union[str, MetricFunctionTypes] = MetricFunctionTypes.VALUE, **kwargs, ) -> Callable[[Callable[P, T]], Callable[P, T]]: @@ -68,7 +68,7 @@ def inner_func(*args: P.args, **kwargs: P.kwargs): def metric_partial( - engine: Type[ExecutionEngine], + engine: type[ExecutionEngine], partial_fn_type: MetricPartialFunctionTypes, domain_type: Union[str, MetricDomainTypes], **kwargs, @@ -136,8 +136,8 @@ class MetricProvider(metaclass=MetaMetricProvider): """ # noqa: E501 - domain_keys: Tuple[str, ...] = tuple() - value_keys: Tuple[str, ...] = tuple() + domain_keys: tuple[str, ...] = tuple() + value_keys: tuple[str, ...] = tuple() default_kwarg_values: dict = {} @classmethod @@ -272,7 +272,7 @@ def _get_evaluation_dependencies( execution_engine: Optional[ExecutionEngine] = None, runtime_configuration: Optional[dict] = None, ): - dependencies: Dict[str, MetricConfiguration] = {} + dependencies: dict[str, MetricConfiguration] = {} if execution_engine is None: return dependencies diff --git a/great_expectations/expectations/metrics/query_metrics/query_column.py b/great_expectations/expectations/metrics/query_metrics/query_column.py index 13678d3db81e..f792f3df345e 100644 --- a/great_expectations/expectations/metrics/query_metrics/query_column.py +++ b/great_expectations/expectations/metrics/query_metrics/query_column.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Optional from great_expectations.core.metric_domain_types import MetricDomainTypes from great_expectations.execution_engine import ( @@ -31,7 +31,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ) -> list[dict]: batch_selectable, _, _ = execution_engine.get_compute_domain( @@ -56,7 +56,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ) -> list[dict]: query = cls._get_query_from_metric_value_kwargs(metric_value_kwargs) @@ -71,6 +71,6 @@ def _spark( query = query.format(col=column, batch="tmp_view") engine: pyspark.SparkSession = execution_engine.spark - result: List[pyspark.Row] = engine.sql(query).limit(MAX_RESULT_RECORDS).collect() + result: list[pyspark.Row] = engine.sql(query).limit(MAX_RESULT_RECORDS).collect() return [element.asDict() for element in result] diff --git a/great_expectations/expectations/metrics/query_metrics/query_column_pair.py b/great_expectations/expectations/metrics/query_metrics/query_column_pair.py index 8e39b56faee7..e92cc2648c69 100644 --- a/great_expectations/expectations/metrics/query_metrics/query_column_pair.py +++ b/great_expectations/expectations/metrics/query_metrics/query_column_pair.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Optional from great_expectations.core.metric_domain_types import MetricDomainTypes from great_expectations.execution_engine import ( @@ -31,7 +31,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ) -> list[dict]: batch_selectable, _, _ = execution_engine.get_compute_domain( @@ -60,9 +60,9 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, - ) -> List[dict]: + ) -> list[dict]: query = cls._get_query_from_metric_value_kwargs(metric_value_kwargs) df: pyspark.DataFrame @@ -76,6 +76,6 @@ def _spark( query = query.format(column_A=column_A, column_B=column_B, batch="tmp_view") engine: pyspark.SparkSession = execution_engine.spark - result: List[pyspark.Row] = engine.sql(query).collect() + result: list[pyspark.Row] = engine.sql(query).collect() return [element.asDict() for element in result] diff --git a/great_expectations/expectations/metrics/query_metrics/query_multiple_columns.py b/great_expectations/expectations/metrics/query_metrics/query_multiple_columns.py index c9c7c49b23cb..b35a5646a3ed 100644 --- a/great_expectations/expectations/metrics/query_metrics/query_multiple_columns.py +++ b/great_expectations/expectations/metrics/query_metrics/query_multiple_columns.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from great_expectations.core.metric_domain_types import MetricDomainTypes from great_expectations.execution_engine import ( @@ -31,7 +31,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ) -> list[dict]: batch_selectable, _, _ = execution_engine.get_compute_domain( @@ -54,7 +54,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ) -> list[dict]: query = cls._get_query_from_metric_value_kwargs(metric_value_kwargs) @@ -79,6 +79,6 @@ def _spark( ) engine: pyspark.SparkSession = execution_engine.spark - result: List[pyspark.Row] = engine.sql(query).limit(MAX_RESULT_RECORDS).collect() + result: list[pyspark.Row] = engine.sql(query).limit(MAX_RESULT_RECORDS).collect() return [element.asDict() for element in result] diff --git a/great_expectations/expectations/metrics/query_metrics/query_table.py b/great_expectations/expectations/metrics/query_metrics/query_table.py index 678986dce0fd..84a4c225e5a8 100644 --- a/great_expectations/expectations/metrics/query_metrics/query_table.py +++ b/great_expectations/expectations/metrics/query_metrics/query_table.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from great_expectations.core.metric_domain_types import MetricDomainTypes from great_expectations.execution_engine import ( @@ -27,7 +27,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ) -> list[dict]: batch_selectable, _, _ = execution_engine.get_compute_domain( @@ -46,7 +46,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ) -> list[dict]: query = cls._get_query_from_metric_value_kwargs(metric_value_kwargs) @@ -60,6 +60,6 @@ def _spark( query = query.format(batch="tmp_view") engine: pyspark.SparkSession = execution_engine.spark - result: List[pyspark.Row] = engine.sql(query).limit(MAX_RESULT_RECORDS).collect() + result: list[pyspark.Row] = engine.sql(query).limit(MAX_RESULT_RECORDS).collect() return [element.asDict() for element in result] diff --git a/great_expectations/expectations/metrics/query_metrics/query_template_values.py b/great_expectations/expectations/metrics/query_metrics/query_template_values.py index 91c176f4d1ec..d58f2dbe5141 100644 --- a/great_expectations/expectations/metrics/query_metrics/query_template_values.py +++ b/great_expectations/expectations/metrics/query_metrics/query_template_values.py @@ -1,7 +1,7 @@ from __future__ import annotations import numbers -from typing import TYPE_CHECKING, Any, Dict, List, Union +from typing import TYPE_CHECKING, Any, Union from great_expectations.compatibility.sqlalchemy import ( sqlalchemy as sa, @@ -45,9 +45,9 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, - ) -> List[dict]: + ) -> list[dict]: query = cls._get_query_from_metric_value_kwargs(metric_value_kwargs) selectable: Union[sa.sql.Selectable, str] @@ -86,7 +86,7 @@ def _sqlalchemy( query = cls.get_query(query, template_dict, f"({selectable})") try: - result: List[sqlalchemy.Row] = execution_engine.execute_query(sa.text(query)).fetchall() # type: ignore[assignment,arg-type] + result: list[sqlalchemy.Row] = execution_engine.execute_query(sa.text(query)).fetchall() # type: ignore[assignment,arg-type] except Exception as e: if hasattr(e, "_query_id"): # query_id removed because it duplicates the validation_results @@ -101,9 +101,9 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, - ) -> List[dict]: + ) -> list[dict]: query = cls._get_query_from_metric_value_kwargs(metric_value_kwargs) df: pyspark.DataFrame @@ -121,6 +121,6 @@ def _spark( query = query.format(**template_dict, batch="tmp_view") engine: pyspark.SparkSession = execution_engine.spark - result: List[pyspark.Row] = engine.sql(query).collect() + result: list[pyspark.Row] = engine.sql(query).collect() return [element.asDict() for element in result] diff --git a/great_expectations/expectations/metrics/table_metric_provider.py b/great_expectations/expectations/metrics/table_metric_provider.py index 30598ff14f54..ff0ca2760f93 100644 --- a/great_expectations/expectations/metrics/table_metric_provider.py +++ b/great_expectations/expectations/metrics/table_metric_provider.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from typing import Tuple from great_expectations.expectations.metrics.metric_provider import MetricProvider @@ -27,7 +26,7 @@ class TableMetricProvider(MetricProvider): - https://docs.greatexpectations.io/docs/guides/expectations/custom_expectations_lp """ # noqa: E501 - domain_keys: Tuple[str, ...] = ( + domain_keys: tuple[str, ...] = ( "batch_id", "table", "row_condition", diff --git a/great_expectations/expectations/metrics/table_metrics/table_column_count.py b/great_expectations/expectations/metrics/table_metrics/table_column_count.py index 44a454b23633..6d00e5532a2f 100644 --- a/great_expectations/expectations/metrics/table_metrics/table_column_count.py +++ b/great_expectations/expectations/metrics/table_metrics/table_column_count.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.execution_engine import ( @@ -30,7 +30,7 @@ def _pandas( execution_engine: ExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): columns = metrics.get("table.columns") @@ -42,7 +42,7 @@ def _sqlalchemy( execution_engine: ExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): columns = metrics.get("table.columns") @@ -54,7 +54,7 @@ def _spark( execution_engine: ExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): columns = metrics.get("table.columns") diff --git a/great_expectations/expectations/metrics/table_metrics/table_column_types.py b/great_expectations/expectations/metrics/table_metrics/table_column_types.py index 729487d4a443..7306b7683843 100644 --- a/great_expectations/expectations/metrics/table_metrics/table_column_types.py +++ b/great_expectations/expectations/metrics/table_metrics/table_column_types.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict, Optional, cast +from typing import Any, Optional, cast from great_expectations.compatibility import pyspark, sqlalchemy from great_expectations.core.metric_domain_types import MetricDomainTypes @@ -31,7 +31,7 @@ def _pandas( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): df, _, _ = execution_engine.get_compute_domain( @@ -45,7 +45,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): batch_id: Optional[str] = metric_domain_kwargs.get("batch_id") @@ -75,7 +75,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): df, _, _ = execution_engine.get_compute_domain( diff --git a/great_expectations/expectations/metrics/table_metrics/table_columns.py b/great_expectations/expectations/metrics/table_metrics/table_columns.py index adffa3317637..e463eda176b1 100644 --- a/great_expectations/expectations/metrics/table_metrics/table_columns.py +++ b/great_expectations/expectations/metrics/table_metrics/table_columns.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.execution_engine import ( @@ -30,7 +30,7 @@ def _pandas( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): column_metadata = metrics["table.column_types"] @@ -42,7 +42,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): column_metadata = metrics["table.column_types"] @@ -54,7 +54,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): column_metadata = metrics["table.column_types"] diff --git a/great_expectations/expectations/metrics/table_metrics/table_row_count.py b/great_expectations/expectations/metrics/table_metrics/table_row_count.py index 806309b0ef4a..7d8c1533182d 100644 --- a/great_expectations/expectations/metrics/table_metrics/table_row_count.py +++ b/great_expectations/expectations/metrics/table_metrics/table_row_count.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any from great_expectations.compatibility.pyspark import functions as F from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa @@ -29,7 +29,7 @@ def _pandas( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): df, _, _ = execution_engine.get_compute_domain( @@ -47,7 +47,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): return sa.func.count(), metric_domain_kwargs, {} @@ -62,7 +62,7 @@ def _spark( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): return F.count(F.lit(1)), metric_domain_kwargs, {} diff --git a/great_expectations/expectations/metrics/util.py b/great_expectations/expectations/metrics/util.py index 206b7f93080f..3aa4c5cb3f4b 100644 --- a/great_expectations/expectations/metrics/util.py +++ b/great_expectations/expectations/metrics/util.py @@ -7,15 +7,11 @@ from typing import ( TYPE_CHECKING, Any, - Dict, Final, Iterable, - List, Mapping, Optional, Sequence, - Tuple, - Type, overload, ) @@ -91,10 +87,10 @@ MAX_RESULT_RECORDS: Final[int] = 200 -UnexpectedIndexList: TypeAlias = List[Dict[str, Any]] +UnexpectedIndexList: TypeAlias = list[dict[str, Any]] -def _is_databricks_dialect(dialect: ModuleType | sa.Dialect | Type[sa.Dialect]) -> bool: +def _is_databricks_dialect(dialect: ModuleType | sa.Dialect | type[sa.Dialect]) -> bool: """ Check if the Databricks dialect is being provided. """ @@ -115,7 +111,7 @@ def _is_databricks_dialect(dialect: ModuleType | sa.Dialect | Type[sa.Dialect]) def get_dialect_regex_expression( # noqa: C901, PLR0911, PLR0912, PLR0915 column: sa.Column, regex: str, - dialect: ModuleType | Type[sa.Dialect] | sa.Dialect, + dialect: ModuleType | type[sa.Dialect] | sa.Dialect, positive: bool = True, ) -> sa.SQLColumnExpression | None: try: @@ -300,8 +296,8 @@ def get_dialect_regex_expression( # noqa: C901, PLR0911, PLR0912, PLR0915 def _get_dialect_type_module( - dialect: ModuleType | Type[sa.Dialect] | sa.Dialect | None = None, -) -> ModuleType | Type[sa.Dialect] | sa.Dialect: + dialect: ModuleType | type[sa.Dialect] | sa.Dialect | None = None, +) -> ModuleType | type[sa.Dialect] | sa.Dialect: if dialect is None: logger.warning("No sqlalchemy dialect found; relying in top-level sqlalchemy types.") return sa @@ -412,7 +408,7 @@ def get_sqlalchemy_column_metadata( schema_name: Optional[str] = None, ) -> Sequence[Mapping[str, Any]] | None: try: - columns: Sequence[Dict[str, Any]] + columns: Sequence[dict[str, Any]] engine = execution_engine.engine inspector = execution_engine.get_inspector() @@ -477,7 +473,7 @@ def column_reflection_fallback( # noqa: C901, PLR0912, PLR0915 selectable: sqlalchemy.Select, dialect: sqlalchemy.Dialect, sqlalchemy_engine: sqlalchemy.Engine, -) -> List[Dict[str, str]]: +) -> list[dict[str, str]]: """If we can't reflect the table, use a query to at least get column names.""" if isinstance(sqlalchemy_engine.engine, sqlalchemy.Engine): @@ -487,7 +483,7 @@ def column_reflection_fallback( # noqa: C901, PLR0912, PLR0915 # with sqlalchemy_engine.begin() as connection: with connection: - col_info_dict_list: List[Dict[str, str]] + col_info_dict_list: list[dict[str, str]] # noinspection PyUnresolvedReferences if dialect.name.lower() == "mssql": # Get column names and types from the database @@ -581,7 +577,7 @@ def column_reflection_fallback( # noqa: C901, PLR0912, PLR0915 columns_table_query.c.column_id.asc(), ) ) - col_info_tuples_list: List[tuple] = connection.execute(col_info_query).fetchall() # type: ignore[assignment] + col_info_tuples_list: list[tuple] = connection.execute(col_info_query).fetchall() # type: ignore[assignment] # type_module = _get_dialect_type_module(dialect=dialect) col_info_dict_list = [ { @@ -694,7 +690,7 @@ def column_reflection_fallback( # noqa: C901, PLR0912, PLR0915 result_object = connection.execute(query) # noinspection PyProtectedMember - col_names: List[str] = result_object._metadata.keys # type: ignore[assignment] + col_names: list[str] = result_object._metadata.keys # type: ignore[assignment] col_info_dict_list = [{"name": col_name} for col_name in col_names] return col_info_dict_list @@ -714,7 +710,7 @@ def get_dbms_compatible_metric_domain_kwargs( Returns: metric_domain_kwargs: Updated "metric_domain_kwargs" dictionary with quoted column names, where appropriate. """ # noqa: E501 - column_names: List[str | sqlalchemy.quoted_name] + column_names: list[str | sqlalchemy.quoted_name] if "column" in metric_domain_kwargs: column_name: str | sqlalchemy.quoted_name = get_dbms_compatible_column_names( column_names=metric_domain_kwargs["column"], @@ -757,17 +753,17 @@ def get_dbms_compatible_column_names( @overload def get_dbms_compatible_column_names( - column_names: List[str], + column_names: list[str], batch_columns_list: Sequence[str | sqlalchemy.quoted_name], error_message_template: str = ..., -) -> List[str | sqlalchemy.quoted_name]: ... +) -> list[str | sqlalchemy.quoted_name]: ... def get_dbms_compatible_column_names( - column_names: List[str] | str, + column_names: list[str] | str, batch_columns_list: Sequence[str | sqlalchemy.quoted_name], error_message_template: str = 'Error: The column "{column_name:s}" in BatchData does not exist.', # noqa: E501 -) -> List[str | sqlalchemy.quoted_name] | str | sqlalchemy.quoted_name: +) -> list[str | sqlalchemy.quoted_name] | str | sqlalchemy.quoted_name: """ Case non-sensitivity is expressed in upper case by common DBMS backends and in lower case by SQLAlchemy, with any deviations enclosed with double quotes. @@ -785,7 +781,7 @@ def get_dbms_compatible_column_names( Returns: Single property-typed column name object or list of property-typed column name objects (depending on input). """ # noqa: E501 - normalized_typed_batch_columns_mappings: List[Tuple[str, str | sqlalchemy.quoted_name]] = ( + normalized_typed_batch_columns_mappings: list[tuple[str, str | sqlalchemy.quoted_name]] = ( _verify_column_names_exist_and_get_normalized_typed_column_names_map( column_names=column_names, batch_columns_list=batch_columns_list, @@ -794,8 +790,8 @@ def get_dbms_compatible_column_names( or [] ) - element: Tuple[str, str | sqlalchemy.quoted_name] - typed_batch_column_names_list: List[str | sqlalchemy.quoted_name] = [ + element: tuple[str, str | sqlalchemy.quoted_name] + typed_batch_column_names_list: list[str | sqlalchemy.quoted_name] = [ element[1] for element in normalized_typed_batch_columns_mappings ] if isinstance(column_names, list): @@ -805,8 +801,8 @@ def get_dbms_compatible_column_names( def verify_column_names_exist( - column_names: List[str] | str, - batch_columns_list: List[str | sqlalchemy.quoted_name], + column_names: list[str] | str, + batch_columns_list: list[str | sqlalchemy.quoted_name], error_message_template: str = 'Error: The column "{column_name:s}" in BatchData does not exist.', # noqa: E501 ) -> None: _ = _verify_column_names_exist_and_get_normalized_typed_column_names_map( @@ -818,11 +814,11 @@ def verify_column_names_exist( def _verify_column_names_exist_and_get_normalized_typed_column_names_map( # noqa: C901 - column_names: List[str] | str, + column_names: list[str] | str, batch_columns_list: Sequence[str | sqlalchemy.quoted_name], error_message_template: str = 'Error: The column "{column_name:s}" in BatchData does not exist.', # noqa: E501 verify_only: bool = False, -) -> List[Tuple[str, str | sqlalchemy.quoted_name]] | None: +) -> list[tuple[str, str | sqlalchemy.quoted_name]] | None: """ Insures that column name or column names (supplied as argument using "str" representation) exist in "Batch" object. @@ -835,7 +831,7 @@ def _verify_column_names_exist_and_get_normalized_typed_column_names_map( # noq Returns: List of tuples having mapping from string-valued column name to typed column name; None if "verify_only" is set. """ # noqa: E501 - column_names_list: List[str] + column_names_list: list[str] if isinstance(column_names, list): column_names_list = column_names else: @@ -843,7 +839,7 @@ def _verify_column_names_exist_and_get_normalized_typed_column_names_map( # noq def _get_normalized_column_name_mapping_if_exists( column_name: str, - ) -> Tuple[str, str | sqlalchemy.quoted_name] | None: + ) -> tuple[str, str | sqlalchemy.quoted_name] | None: typed_column_name_cursor: str | sqlalchemy.quoted_name for typed_column_name_cursor in batch_columns_list: if ( @@ -862,9 +858,9 @@ def _get_normalized_column_name_mapping_if_exists( return None - normalized_batch_columns_mappings: List[Tuple[str, str | sqlalchemy.quoted_name]] = [] + normalized_batch_columns_mappings: list[tuple[str, str | sqlalchemy.quoted_name]] = [] - normalized_column_name_mapping: Tuple[str, str | sqlalchemy.quoted_name] | None + normalized_column_name_mapping: tuple[str, str | sqlalchemy.quoted_name] | None column_name: str for column_name in column_names_list: normalized_column_name_mapping = _get_normalized_column_name_mapping_if_exists( @@ -1251,8 +1247,8 @@ def get_sqlalchemy_source_table_and_schema( def get_unexpected_indices_for_multiple_pandas_named_indices( # noqa: C901 domain_records_df: pd.DataFrame, - unexpected_index_column_names: List[str], - expectation_domain_column_list: List[str], + unexpected_index_column_names: list[str], + expectation_domain_column_list: list[str], exclude_unexpected_values: bool = False, ) -> UnexpectedIndexList: """ @@ -1272,10 +1268,10 @@ def get_unexpected_indices_for_multiple_pandas_named_indices( # noqa: C901 failed_metrics=["unexpected_index_list"], ) - domain_records_df_index_names: List[str] = domain_records_df.index.names - unexpected_indices: List[tuple[int | str, ...]] = list(domain_records_df.index) + domain_records_df_index_names: list[str] = domain_records_df.index.names + unexpected_indices: list[tuple[int | str, ...]] = list(domain_records_df.index) - tuple_index: Dict[str, int] = dict() + tuple_index: dict[str, int] = dict() for column_name in unexpected_index_column_names: if column_name not in domain_records_df_index_names: raise gx_exceptions.MetricResolutionError( @@ -1289,7 +1285,7 @@ def get_unexpected_indices_for_multiple_pandas_named_indices( # noqa: C901 unexpected_index_list: UnexpectedIndexList = [] if exclude_unexpected_values and len(unexpected_indices) != 0: - primary_key_dict_list: dict[str, List[Any]] = { + primary_key_dict_list: dict[str, list[Any]] = { idx_col: [] for idx_col in unexpected_index_column_names } for index in unexpected_indices: @@ -1300,7 +1296,7 @@ def get_unexpected_indices_for_multiple_pandas_named_indices( # noqa: C901 else: for index in unexpected_indices: - primary_key_dict: Dict[str, Any] = dict() + primary_key_dict: dict[str, Any] = dict() for domain_column_name in expectation_domain_column_list: primary_key_dict[domain_column_name] = domain_records_df.at[ index, domain_column_name @@ -1314,8 +1310,8 @@ def get_unexpected_indices_for_multiple_pandas_named_indices( # noqa: C901 def get_unexpected_indices_for_single_pandas_named_index( domain_records_df: pd.DataFrame, - unexpected_index_column_names: List[str], - expectation_domain_column_list: List[str], + unexpected_index_column_names: list[str], + expectation_domain_column_list: list[str], exclude_unexpected_values: bool = False, ) -> UnexpectedIndexList: """ @@ -1332,7 +1328,7 @@ def get_unexpected_indices_for_single_pandas_named_index( """ # noqa: E501 if not expectation_domain_column_list: return [] - unexpected_index_values_by_named_index: List[int | str] = list(domain_records_df.index) + unexpected_index_values_by_named_index: list[int | str] = list(domain_records_df.index) unexpected_index_list: UnexpectedIndexList = [] if not ( len(unexpected_index_column_names) == 1 @@ -1344,14 +1340,14 @@ def get_unexpected_indices_for_single_pandas_named_index( ) if exclude_unexpected_values and len(unexpected_index_values_by_named_index) != 0: - primary_key_dict_list: dict[str, List[Any]] = {unexpected_index_column_names[0]: []} + primary_key_dict_list: dict[str, list[Any]] = {unexpected_index_column_names[0]: []} for index in unexpected_index_values_by_named_index: primary_key_dict_list[unexpected_index_column_names[0]].append(index) unexpected_index_list.append(primary_key_dict_list) else: for index in unexpected_index_values_by_named_index: - primary_key_dict: Dict[str, Any] = dict() + primary_key_dict: dict[str, Any] = dict() for domain_column in expectation_domain_column_list: primary_key_dict[domain_column] = domain_records_df.at[index, domain_column] column_name: str = unexpected_index_column_names[0] @@ -1363,10 +1359,10 @@ def get_unexpected_indices_for_single_pandas_named_index( def compute_unexpected_pandas_indices( # noqa: C901 domain_records_df: pd.DataFrame, - expectation_domain_column_list: List[str], - result_format: Dict[str, Any], + expectation_domain_column_list: list[str], + result_format: dict[str, Any], execution_engine: PandasExecutionEngine, - metrics: Dict[str, Any], + metrics: dict[str, Any], ) -> UnexpectedIndexList: """ Helper method to compute unexpected_index_list for PandasExecutionEngine. Handles logic needed for named indices. @@ -1383,7 +1379,7 @@ def compute_unexpected_pandas_indices( # noqa: C901 list of unexpected_index_list values. It can either be a list of dicts or a list of numbers (if using default index). """ # noqa: E501 - unexpected_index_column_names: List[str] + unexpected_index_column_names: list[str] unexpected_index_list: UnexpectedIndexList exclude_unexpected_values: bool = result_format.get("exclude_unexpected_values", False) @@ -1412,14 +1408,14 @@ def compute_unexpected_pandas_indices( # noqa: C901 elif result_format.get("unexpected_index_column_names"): unexpected_index_column_names = result_format["unexpected_index_column_names"] unexpected_index_list = [] - unexpected_indices: List[int | str] = list(domain_records_df.index) + unexpected_indices: list[int | str] = list(domain_records_df.index) if ( exclude_unexpected_values and len(unexpected_indices) != 0 and len(unexpected_index_column_names) != 0 ): - primary_key_dict_list: dict[str, List[Any]] = { + primary_key_dict_list: dict[str, list[Any]] = { idx_col: [] for idx_col in unexpected_index_column_names } for index in unexpected_indices: @@ -1436,7 +1432,7 @@ def compute_unexpected_pandas_indices( # noqa: C901 else: for index in unexpected_indices: - primary_key_dict: Dict[str, Any] = dict() + primary_key_dict: dict[str, Any] = dict() assert ( expectation_domain_column_list ), "`expectation_domain_column_list` was not provided" diff --git a/great_expectations/expectations/registry.py b/great_expectations/expectations/registry.py index 04dba6b69ad8..8b738289cbae 100644 --- a/great_expectations/expectations/registry.py +++ b/great_expectations/expectations/registry.py @@ -4,12 +4,8 @@ from typing import ( TYPE_CHECKING, Callable, - Dict, - List, NamedTuple, Optional, - Tuple, - Type, Union, ) @@ -61,7 +57,7 @@ class RendererImpl(NamedTuple): def register_renderer( object_name: str, - parent_class: Union[Type[Expectation], Type[MetricProvider]], + parent_class: type[Union[Expectation, MetricProvider]], renderer_fn: Callable[..., Union[RenderedAtomicContent, RenderedContent]], ): # noinspection PyUnresolvedReferences @@ -97,7 +93,7 @@ def register_renderer( return -def get_renderer_names(expectation_or_metric_type: str) -> List[str]: +def get_renderer_names(expectation_or_metric_type: str) -> list[str]: """Gets renderer names for a given Expectation or Metric. Args: @@ -111,8 +107,8 @@ def get_renderer_names(expectation_or_metric_type: str) -> List[str]: def get_renderer_names_with_renderer_types( expectation_or_metric_type: str, - renderer_types: List[AtomicRendererType], -) -> List[Union[str, AtomicDiagnosticRendererType, AtomicPrescriptiveRendererType]]: + renderer_types: list[AtomicRendererType], +) -> list[Union[str, AtomicDiagnosticRendererType, AtomicPrescriptiveRendererType]]: """Gets renderer names of a given type, for a given Expectation or Metric. Args: @@ -131,7 +127,7 @@ def get_renderer_names_with_renderer_types( ] -def get_renderer_impls(object_name: str) -> List[str]: +def get_renderer_impls(object_name: str) -> list[str]: return list(_registered_renderers.get(object_name, {}).values()) @@ -143,7 +139,7 @@ def get_renderer_impl(object_name: str, renderer_type: str) -> Optional[Renderer return renderer_impl -def register_expectation(expectation: Type[Expectation]) -> None: +def register_expectation(expectation: type[Expectation]) -> None: expectation_type = expectation.expectation_type # TODO: add version to key if expectation_type in _registered_expectations: @@ -218,10 +214,10 @@ def _add_response_key(res, key, value): def register_metric( # noqa: PLR0913 metric_name: str, - metric_domain_keys: Tuple[str, ...], - metric_value_keys: Tuple[str, ...], - execution_engine: Type[ExecutionEngine], - metric_class: Type[MetricProvider], + metric_domain_keys: tuple[str, ...], + metric_value_keys: tuple[str, ...], + execution_engine: type[ExecutionEngine], + metric_class: type[MetricProvider], metric_provider: Optional[Callable], metric_fn_type: Optional[Union[MetricFunctionTypes, MetricPartialFunctionTypes]] = None, ) -> dict: @@ -309,7 +305,7 @@ def register_metric( # noqa: PLR0913 def get_metric_provider( metric_name: str, execution_engine: ExecutionEngine -) -> Tuple[MetricProvider, Callable]: +) -> tuple[MetricProvider, Callable]: try: metric_definition = _registered_metrics[metric_name] return metric_definition["providers"][type(execution_engine).__name__] @@ -381,7 +377,7 @@ def get_metric_kwargs( def get_domain_metrics_dict_by_name( - metrics: Dict[Tuple[str, str, str], MetricValue], metric_domain_kwargs: IDDict + metrics: dict[tuple[str, str, str], MetricValue], metric_domain_kwargs: IDDict ): return { metric_edge_key_id_tuple[0]: metric_value @@ -390,8 +386,8 @@ def get_domain_metrics_dict_by_name( } -def get_expectation_impl(expectation_name: str) -> Type[Expectation]: - expectation: Type[Expectation] | None = _registered_expectations.get(expectation_name) +def get_expectation_impl(expectation_name: str) -> type[Expectation]: + expectation: type[Expectation] | None = _registered_expectations.get(expectation_name) if not expectation: raise gx_exceptions.ExpectationNotFoundError(f"{expectation_name} not found") # noqa: TRY003 @@ -399,8 +395,8 @@ def get_expectation_impl(expectation_name: str) -> Type[Expectation]: def list_registered_expectation_implementations( - expectation_root: Optional[Type[Expectation]] = None, -) -> List[str]: + expectation_root: Optional[type[Expectation]] = None, +) -> list[str]: registered_expectation_implementations = [] for ( expectation_name, diff --git a/great_expectations/expectations/sql_tokens_and_types.py b/great_expectations/expectations/sql_tokens_and_types.py index e5734432a946..9300c7ee0dcf 100644 --- a/great_expectations/expectations/sql_tokens_and_types.py +++ b/great_expectations/expectations/sql_tokens_and_types.py @@ -2,7 +2,6 @@ from enum import Enum from itertools import chain -from typing import Set class ValidSqlTokens(str, Enum): @@ -260,7 +259,7 @@ class ValidSparkSqlTokens(str, Enum): FOR = "FOR" -valid_sql_tokens_and_types: Set[str] = set( +valid_sql_tokens_and_types: set[str] = set( chain.from_iterable( [ list(map(lambda i: i.upper(), ValidSqlTokens.__members__.keys())), diff --git a/great_expectations/experimental/metric_repository/batch_inspector.py b/great_expectations/experimental/metric_repository/batch_inspector.py index 044d366fba19..2bf7fd293c50 100644 --- a/great_expectations/experimental/metric_repository/batch_inspector.py +++ b/great_expectations/experimental/metric_repository/batch_inspector.py @@ -1,7 +1,7 @@ from __future__ import annotations import uuid -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from great_expectations.experimental.metric_repository.metrics import ( Metric, @@ -31,7 +31,7 @@ def compute_metric_list_run( self, data_asset_id: uuid.UUID, batch_request: BatchRequest, - metric_list: Optional[List[MetricTypes]], + metric_list: Optional[list[MetricTypes]], ) -> MetricRun: """Method that computes a MetricRun for a list of metrics. Called by GX Agent to compute a MetricRun as part of a RunMetricsEvent. diff --git a/great_expectations/experimental/metric_repository/cloud_data_store.py b/great_expectations/experimental/metric_repository/cloud_data_store.py index 3960c15a1c89..dfb150f0b1f9 100644 --- a/great_expectations/experimental/metric_repository/cloud_data_store.py +++ b/great_expectations/experimental/metric_repository/cloud_data_store.py @@ -3,7 +3,7 @@ import urllib.parse import uuid import weakref -from typing import TYPE_CHECKING, Any, Dict, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from great_expectations.compatibility.pydantic import BaseModel from great_expectations.compatibility.typing_extensions import override @@ -41,7 +41,7 @@ def orjson_loads(v, *args, **kwargs): class Payload(BaseModel): - data: Dict[str, Any] + data: dict[str, Any] class Config: extra = "forbid" diff --git a/great_expectations/experimental/metric_repository/metric_list_metric_retriever.py b/great_expectations/experimental/metric_repository/metric_list_metric_retriever.py index 9ea8f49df326..7fbac323ad9e 100644 --- a/great_expectations/experimental/metric_repository/metric_list_metric_retriever.py +++ b/great_expectations/experimental/metric_repository/metric_list_metric_retriever.py @@ -2,7 +2,7 @@ import logging from itertools import chain -from typing import TYPE_CHECKING, List, Optional, Sequence +from typing import TYPE_CHECKING, Optional, Sequence from great_expectations.compatibility.typing_extensions import override from great_expectations.experimental.metric_repository.metric_retriever import ( @@ -33,9 +33,9 @@ def __init__(self, context: AbstractDataContext): def get_metrics( self, batch_request: BatchRequest, - metric_list: Optional[List[MetricTypes]] = None, + metric_list: Optional[list[MetricTypes]] = None, ) -> Sequence[Metric]: - metrics_result: List[Metric] = [] + metrics_result: list[Metric] = [] if not metric_list: raise ValueError("metric_list cannot be empty") # noqa: TRY003 @@ -79,7 +79,7 @@ def get_metrics( timestamp_column_metrics = self._get_timestamp_column_metrics( metric_list, batch_request, timestamp_column_names ) - all_column_names: List[str] = self._get_all_column_names(table_metrics) + all_column_names: list[str] = self._get_all_column_names(table_metrics) non_numeric_column_metrics = self._get_non_numeric_column_metrics( metric_list, batch_request, all_column_names ) @@ -97,9 +97,9 @@ def get_metrics( def _get_non_numeric_column_metrics( self, - metrics_list: List[MetricTypes], + metrics_list: list[MetricTypes], batch_request: BatchRequest, - column_list: List[str], + column_list: list[str], ) -> Sequence[Metric]: """Calculate column metrics for non-numeric columns. @@ -129,9 +129,9 @@ def _get_non_numeric_column_metrics( def _get_numeric_column_metrics( self, - metrics_list: List[MetricTypes], + metrics_list: list[MetricTypes], batch_request: BatchRequest, - column_list: List[str], + column_list: list[str], ) -> Sequence[Metric]: """Calculate column metrics for numeric columns. @@ -164,9 +164,9 @@ def _get_numeric_column_metrics( def _get_timestamp_column_metrics( self, - metrics_list: List[MetricTypes], + metrics_list: list[MetricTypes], batch_request: BatchRequest, - column_list: List[str], + column_list: list[str], ) -> Sequence[Metric]: """Calculate column metrics for timestamp columns. @@ -200,8 +200,8 @@ def _get_timestamp_column_metrics( ) def _calculate_table_metrics( - self, batch_request: BatchRequest, metric_list: List[MetricTypes] - ) -> List[Metric]: + self, batch_request: BatchRequest, metric_list: list[MetricTypes] + ) -> list[Metric]: """Calculate table metrics, which include row_count, column names and types. Args: @@ -211,7 +211,7 @@ def _calculate_table_metrics( Returns: Sequence[Metric]: List of table metrics. """ - metrics: List[Metric] = [] + metrics: list[Metric] = [] if MetricTypes.TABLE_ROW_COUNT in metric_list: metrics.append(self._get_table_row_count(batch_request=batch_request)) if MetricTypes.TABLE_COLUMNS in metric_list: @@ -220,7 +220,7 @@ def _calculate_table_metrics( metrics.append(self._get_table_column_types(batch_request=batch_request)) return metrics - def _check_valid_metric_types(self, metric_list: List[MetricTypes]) -> bool: + def _check_valid_metric_types(self, metric_list: list[MetricTypes]) -> bool: """Check whether all the metric types in the list are valid. Args: @@ -231,7 +231,7 @@ def _check_valid_metric_types(self, metric_list: List[MetricTypes]) -> bool: """ # noqa: E501 return all(metric in MetricTypes for metric in metric_list) - def _column_metrics_in_metric_list(self, metric_list: List[MetricTypes]) -> bool: + def _column_metrics_in_metric_list(self, metric_list: list[MetricTypes]) -> bool: """Helper method to check whether any column metrics are present in the metric list. Args: @@ -240,7 +240,7 @@ def _column_metrics_in_metric_list(self, metric_list: List[MetricTypes]) -> bool Returns: bool: True if any column metrics are present in the metric list, False otherwise. """ # noqa: E501 - column_metrics: List[MetricTypes] = [ + column_metrics: list[MetricTypes] = [ MetricTypes.COLUMN_MIN, MetricTypes.COLUMN_MAX, MetricTypes.COLUMN_MEDIAN, diff --git a/great_expectations/experimental/metric_repository/metric_retriever.py b/great_expectations/experimental/metric_repository/metric_retriever.py index 89def6bb601c..990bf924bfce 100644 --- a/great_expectations/experimental/metric_repository/metric_retriever.py +++ b/great_expectations/experimental/metric_repository/metric_retriever.py @@ -5,7 +5,6 @@ from typing import ( TYPE_CHECKING, Any, - List, Optional, Sequence, ) @@ -54,7 +53,7 @@ def get_validator(self, batch_request: BatchRequest) -> Validator: def get_metrics( self, batch_request: BatchRequest, - metric_list: Optional[List[MetricTypes]] = None, + metric_list: Optional[list[MetricTypes]] = None, ) -> Sequence[Metric]: raise NotImplementedError @@ -132,11 +131,11 @@ def _compute_metrics( batch_id = validator.active_batch.id return batch_id, computed_metrics, aborted_metrics - def _get_columns_to_exclude(self, table_column_types: Metric) -> List[str]: + def _get_columns_to_exclude(self, table_column_types: Metric) -> list[str]: """ Excludes columns that are unsupported or missing type metadata """ - columns_to_skip: List[str] = [] + columns_to_skip: list[str] = [] UNSUPPORTED_COLUMN_TYPES = ["TIME"] for column_type in table_column_types.value: if not column_type.get("type"): @@ -148,7 +147,7 @@ def _get_columns_to_exclude(self, table_column_types: Metric) -> List[str]: def _get_numeric_column_names( self, batch_request: BatchRequest, - exclude_column_names: List[str], + exclude_column_names: list[str], ) -> list[str]: """Get the names of all numeric columns in the batch.""" return self._get_column_names_for_semantic_types( @@ -160,7 +159,7 @@ def _get_numeric_column_names( def _get_timestamp_column_names( self, batch_request: BatchRequest, - exclude_column_names: List[str], + exclude_column_names: list[str], ) -> list[str]: """Get the names of all timestamp columns in the batch.""" return self._get_column_names_for_semantic_types( @@ -172,8 +171,8 @@ def _get_timestamp_column_names( def _get_column_names_for_semantic_types( self, batch_request: BatchRequest, - include_semantic_types: List[SemanticDomainTypes], - exclude_column_names: List[str], + include_semantic_types: list[SemanticDomainTypes], + exclude_column_names: list[str], ) -> list[str]: """Get the names of all columns matching semantic types in the batch.""" validator = self.get_validator(batch_request=batch_request) @@ -213,8 +212,8 @@ def _get_table_metrics( def _get_column_metrics( self, batch_request: BatchRequest, - column_list: List[str], - column_metric_names: List[MetricTypes | str], + column_list: list[str], + column_metric_names: list[MetricTypes | str], column_metric_type: type[ColumnMetric[Any]], ) -> Sequence[Metric]: column_metric_configs = self._generate_column_metric_configurations( @@ -252,7 +251,7 @@ def _get_column_metrics( def _generate_column_metric_configurations( self, column_list: list[str], column_metric_names: list[str | MetricTypes] ) -> list[MetricConfiguration]: - column_metric_configs: List[MetricConfiguration] = list() + column_metric_configs: list[MetricConfiguration] = list() for metric_name in column_metric_names: for column in column_list: column_metric_configs.append( @@ -264,8 +263,8 @@ def _generate_column_metric_configurations( ) return column_metric_configs - def _get_all_column_names(self, metrics: Sequence[Metric]) -> List[str]: - column_list: List[str] = [] + def _get_all_column_names(self, metrics: Sequence[Metric]) -> list[str]: + column_list: list[str] = [] for metric in metrics: if metric.metric_name == MetricTypes.TABLE_COLUMNS: column_list = metric.value @@ -282,7 +281,7 @@ def _get_table_columns(self, batch_request: BatchRequest) -> Metric: return self._get_table_metrics( batch_request=batch_request, metric_name=MetricTypes.TABLE_COLUMNS, - metric_type=TableMetric[List[str]], + metric_type=TableMetric[list[str]], ) def _get_table_column_types(self, batch_request: BatchRequest) -> Metric: @@ -315,7 +314,7 @@ def _get_table_column_types(self, batch_request: BatchRequest) -> Metric: else: column_types_converted_to_str.append({"name": raw_column_type["name"]}) - return TableMetric[List[str]]( + return TableMetric[list[str]]( batch_id=batch_id, metric_name=metric_name, value=column_types_converted_to_str, diff --git a/great_expectations/experimental/metric_repository/metrics.py b/great_expectations/experimental/metric_repository/metrics.py index d607628018f6..c3feaca0cee4 100644 --- a/great_expectations/experimental/metric_repository/metrics.py +++ b/great_expectations/experimental/metric_repository/metrics.py @@ -6,9 +6,7 @@ TYPE_CHECKING, AbstractSet, Any, - Dict, Generic, - List, Mapping, Optional, Sequence, @@ -124,7 +122,7 @@ def dict( # noqa: PLR0913 exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """Override the dict function to include @property fields, in pydandic v2 we can use computed_field. https://docs.pydantic.dev/latest/usage/computed_fields/ """ # noqa: E501 @@ -202,8 +200,8 @@ def metric_type(self) -> str: # ColumnQuantileValuesMetric is an example of a metric that has parameters -class ColumnQuantileValuesMetric(ColumnMetric[List[float]]): - quantiles: List[float] = Field(description="Quantiles to compute") +class ColumnQuantileValuesMetric(ColumnMetric[list[float]]): + quantiles: list[float] = Field(description="Quantiles to compute") allow_relative_error: Union[float, str] = Field( description="Relative error interpolation type (pandas) or limit (e.g. spark) depending on data source" # noqa: E501 ) diff --git a/great_expectations/experimental/rule_based_profiler/attributed_resolved_metrics.py b/great_expectations/experimental/rule_based_profiler/attributed_resolved_metrics.py index 5c2ac56a7a22..29546dedc87e 100644 --- a/great_expectations/experimental/rule_based_profiler/attributed_resolved_metrics.py +++ b/great_expectations/experimental/rule_based_profiler/attributed_resolved_metrics.py @@ -2,7 +2,7 @@ import logging from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Sized +from typing import TYPE_CHECKING, Iterator, Optional, Sized import numpy as np import pandas as pd @@ -91,14 +91,14 @@ class AttributedResolvedMetrics(SerializableDictDot): with uniquely identifiable attribution object so that receivers can filter them from overall resolved metrics. """ # noqa: E501 - batch_ids: Optional[List[str]] = None + batch_ids: Optional[list[str]] = None metric_attributes: Optional[Attributes] = None - metric_values_by_batch_id: Optional[Dict[str, MetricValue]] = None + metric_values_by_batch_id: Optional[dict[str, MetricValue]] = None @staticmethod def get_conditioned_attributed_metric_values_from_attributed_metric_values( - attributed_metric_values: Dict[str, MetricValues], - ) -> Dict[str, MetricValues]: + attributed_metric_values: dict[str, MetricValues], + ) -> dict[str, MetricValues]: """ Converts "attributed_metric_values" to Numpy array for each "batch_id" key (recursively, wherever possible). """ # noqa: E501 @@ -114,7 +114,7 @@ def get_conditioned_attributed_metric_values_from_attributed_metric_values( @staticmethod def get_conditioned_metric_values_from_attributed_metric_values( - attributed_metric_values: Dict[str, MetricValue], + attributed_metric_values: dict[str, MetricValue], ) -> Optional[MetricValues]: """ Converts all "attributed_metric_values" as list (together) to Numpy array (recursively, wherever possible). @@ -142,7 +142,7 @@ def id(self) -> str: return self.metric_attributes.to_id() @property - def attributed_metric_values(self) -> Optional[Dict[str, MetricValue]]: + def attributed_metric_values(self) -> Optional[dict[str, MetricValue]]: if self.metric_values_by_batch_id is None or self.batch_ids is None: return None @@ -154,7 +154,7 @@ def attributed_metric_values(self) -> Optional[Dict[str, MetricValue]]: } @property - def conditioned_attributed_metric_values(self) -> Dict[str, MetricValues]: + def conditioned_attributed_metric_values(self) -> dict[str, MetricValues]: if self.attributed_metric_values is None: return {} diff --git a/great_expectations/experimental/rule_based_profiler/builder.py b/great_expectations/experimental/rule_based_profiler/builder.py index f798b38f1975..b303adb3a14e 100644 --- a/great_expectations/experimental/rule_based_profiler/builder.py +++ b/great_expectations/experimental/rule_based_profiler/builder.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Any, ClassVar, List, Optional, Set, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.batch import ( @@ -30,7 +30,7 @@ class Builder(SerializableDictDot): A Builder provides methods to serialize any builder object of a rule generically. """ - exclude_field_names: ClassVar[Set[str]] = { + exclude_field_names: ClassVar[set[str]] = { "batch_list", "batch_request", "data_context", @@ -44,7 +44,7 @@ def __init__( Args: data_context: AbstractDataContext associated with this Builder """ - self._batch_list: Optional[List[Batch]] = None + self._batch_list: Optional[list[Batch]] = None self._batch_request: Union[BatchRequestBase, dict, None] = None self._data_context: Optional[AbstractDataContext] = data_context @@ -53,11 +53,11 @@ def __init__( """ # noqa: E501 @property - def batch_list(self) -> Optional[List[Batch]]: + def batch_list(self) -> Optional[list[Batch]]: return self._batch_list @batch_list.setter - def batch_list(self, value: List[Batch]) -> None: + def batch_list(self, value: list[Batch]) -> None: self._batch_list = value @property @@ -77,7 +77,7 @@ def data_context(self) -> Optional[AbstractDataContext]: def set_batch_list_if_null_batch_request( self, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, ) -> None: """ @@ -92,7 +92,7 @@ def set_batch_list_if_null_batch_request( def set_batch_data( self, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, ) -> None: arg: Any diff --git a/great_expectations/experimental/rule_based_profiler/config/base.py b/great_expectations/experimental/rule_based_profiler/config/base.py index dc91e94138a4..fcd2747e5173 100644 --- a/great_expectations/experimental/rule_based_profiler/config/base.py +++ b/great_expectations/experimental/rule_based_profiler/config/base.py @@ -2,7 +2,7 @@ import json import logging -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Union from marshmallow import INCLUDE, Schema, ValidationError, fields, post_dump, post_load @@ -49,7 +49,7 @@ class NotNullSchema(Schema): # noinspection PyUnusedLocal @post_load - def make_config(self, data: dict, **kwargs) -> Type[DictDot]: + def make_config(self, data: dict, **kwargs) -> type[DictDot]: """Hook to convert the schema object into its respective config type. Args: @@ -150,7 +150,7 @@ def __repr__(self) -> str: inplace=True, ) - keys: List[str] = sorted(list(json_dict.keys())) + keys: list[str] = sorted(list(json_dict.keys())) key: str sorted_json_dict: dict = {key: json_dict[key] for key in keys} @@ -236,7 +236,7 @@ def __repr__(self) -> str: inplace=True, ) - keys: List[str] = sorted(list(json_dict.keys())) + keys: list[str] = sorted(list(json_dict.keys())) key: str sorted_json_dict: dict = {key: json_dict[key] for key in keys} @@ -338,7 +338,7 @@ def __repr__(self) -> str: inplace=True, ) - keys: List[str] = sorted(list(json_dict.keys())) + keys: list[str] = sorted(list(json_dict.keys())) key: str sorted_json_dict: dict = {key: json_dict[key] for key in keys} @@ -400,11 +400,11 @@ class Meta: class RuleConfig(SerializableDictDot): def __init__( self, - variables: Optional[Dict[str, Any]] = None, + variables: Optional[dict[str, Any]] = None, domain_builder: Optional[dict] = None, # see DomainBuilderConfig - parameter_builders: Optional[List[dict]] = None, # see ParameterBuilderConfig + parameter_builders: Optional[list[dict]] = None, # see ParameterBuilderConfig expectation_configuration_builders: Optional[ - List[dict] + list[dict] ] = None, # see ExpectationConfigurationBuilderConfig ) -> None: self.variables = variables @@ -440,7 +440,7 @@ def __repr__(self) -> str: inplace=True, ) - keys: List[str] = sorted(list(json_dict.keys())) + keys: list[str] = sorted(list(json_dict.keys())) key: str sorted_json_dict: dict = {key: json_dict[key] for key in keys} @@ -503,9 +503,9 @@ def __init__( # noqa: PLR0913 self, name: str, config_version: float, - rules: Dict[str, dict], # see RuleConfig + rules: dict[str, dict], # see RuleConfig id: Optional[str] = None, - variables: Optional[Dict[str, Any]] = None, + variables: Optional[dict[str, Any]] = None, commented_map: Optional[CommentedMap] = None, ) -> None: self.module_name = "great_expectations.rule_based_profiler" @@ -543,12 +543,12 @@ def from_commented_map(cls, commented_map: CommentedMap): # type: ignore[overri @classmethod @override - def get_config_class(cls) -> Type[RuleBasedProfilerConfig]: + def get_config_class(cls) -> type[RuleBasedProfilerConfig]: return cls @classmethod @override - def get_schema_class(cls) -> Type[RuleBasedProfilerConfigSchema]: + def get_schema_class(cls) -> type[RuleBasedProfilerConfigSchema]: return RuleBasedProfilerConfigSchema @override @@ -579,7 +579,7 @@ def __repr__(self) -> str: inplace=True, ) - keys: List[str] = sorted(list(json_dict.keys())) + keys: list[str] = sorted(list(json_dict.keys())) key: str sorted_json_dict: dict = {key: json_dict[key] for key in keys} @@ -601,8 +601,8 @@ def __str__(self) -> str: def resolve_config_using_acceptable_arguments( cls, profiler: RuleBasedProfiler, - variables: Optional[Dict[str, Any]] = None, - rules: Optional[Dict[str, Dict[str, Any]]] = None, + variables: Optional[dict[str, Any]] = None, + rules: Optional[dict[str, dict[str, Any]]] = None, ) -> RuleBasedProfilerConfig: """Reconciles variables/rules by taking into account runtime overrides and variable substitution. @@ -621,17 +621,17 @@ def resolve_config_using_acceptable_arguments( effective_variables: Optional[ParameterContainer] = profiler.reconcile_profiler_variables( variables=variables, ) - runtime_variables: Optional[Dict[str, Any]] = convert_variables_to_dict( + runtime_variables: Optional[dict[str, Any]] = convert_variables_to_dict( variables=effective_variables ) - effective_rules: List[Rule] = profiler.reconcile_profiler_rules( + effective_rules: list[Rule] = profiler.reconcile_profiler_rules( rules=rules, ) rule: Rule - effective_rules_dict: Dict[str, Rule] = {rule.name: rule for rule in effective_rules} - runtime_rules: Dict[str, dict] = { + effective_rules_dict: dict[str, Rule] = {rule.name: rule for rule in effective_rules} + runtime_rules: dict[str, dict] = { name: RuleBasedProfilerConfig._substitute_variables_in_config( rule=rule, variables_container=effective_variables, # type: ignore[arg-type] # could be None diff --git a/great_expectations/experimental/rule_based_profiler/domain_builder/categorical_column_domain_builder.py b/great_expectations/experimental/rule_based_profiler/domain_builder/categorical_column_domain_builder.py index 3193d8a90a87..5f5343204e38 100644 --- a/great_expectations/experimental/rule_based_profiler/domain_builder/categorical_column_domain_builder.py +++ b/great_expectations/experimental/rule_based_profiler/domain_builder/categorical_column_domain_builder.py @@ -3,13 +3,8 @@ from typing import ( TYPE_CHECKING, ClassVar, - Dict, Iterable, - List, Optional, - Set, - Tuple, - Type, Union, ) @@ -48,27 +43,27 @@ class CategoricalColumnDomainBuilder(ColumnDomainBuilder): This DomainBuilder uses column cardinality to identify domains. """ - exclude_field_names: ClassVar[Set[str]] = ColumnDomainBuilder.exclude_field_names | { + exclude_field_names: ClassVar[set[str]] = ColumnDomainBuilder.exclude_field_names | { "cardinality_checker", } - cardinality_limit_modes: Type[CardinalityLimitMode] = CardinalityLimitMode + cardinality_limit_modes: type[CardinalityLimitMode] = CardinalityLimitMode def __init__( # noqa: PLR0913 self, - include_column_names: Optional[Union[str, Optional[List[str]]]] = None, - exclude_column_names: Optional[Union[str, Optional[List[str]]]] = None, - include_column_name_suffixes: Optional[Union[str, Iterable, List[str]]] = None, - exclude_column_name_suffixes: Optional[Union[str, Iterable, List[str]]] = None, + include_column_names: Optional[Union[str, Optional[list[str]]]] = None, + exclude_column_names: Optional[Union[str, Optional[list[str]]]] = None, + include_column_name_suffixes: Optional[Union[str, Iterable, list[str]]] = None, + exclude_column_name_suffixes: Optional[Union[str, Iterable, list[str]]] = None, semantic_type_filter_module_name: Optional[str] = None, semantic_type_filter_class_name: Optional[str] = None, include_semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, exclude_semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, allowed_semantic_types_passthrough: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, cardinality_limit_mode: Optional[Union[str, CardinalityLimitMode, dict]] = None, max_unique_values: Optional[Union[str, int]] = None, @@ -137,7 +132,7 @@ def __init__( # noqa: PLR0913 ] self._allowed_semantic_types_passthrough: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = allowed_semantic_types_passthrough super().__init__( @@ -166,13 +161,13 @@ def domain_type(self) -> MetricDomainTypes: @property def allowed_semantic_types_passthrough( self, - ) -> Optional[Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]]]: + ) -> Optional[Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]]]: return self._allowed_semantic_types_passthrough @allowed_semantic_types_passthrough.setter def allowed_semantic_types_passthrough( self, - value: Optional[Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]]], + value: Optional[Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]]], ) -> None: self._allowed_semantic_types_passthrough = value @@ -206,7 +201,7 @@ def _get_domains( rule_name: str, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: """Return domains matching the selected cardinality_limit_mode. Args: @@ -217,11 +212,11 @@ def _get_domains( Returns: List of domains that match the desired cardinality. """ # noqa: E501 - batch_ids: Optional[List[str]] = self.get_batch_ids(variables=variables) + batch_ids: Optional[list[str]] = self.get_batch_ids(variables=variables) validator: Optional[Validator] = self.get_validator(variables=variables) - effective_column_names: List[str] = self.get_effective_column_names( + effective_column_names: list[str] = self.get_effective_column_names( batch_ids=batch_ids, validator=validator, variables=variables, @@ -270,7 +265,7 @@ def _get_domains( # Obtain allowed_semantic_types_passthrough from "rule state" (i.e., variables and parameters); from instance variable otherwise. # noqa: E501 allowed_semantic_types_passthrough: Union[ - str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]] + str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]] ] = get_parameter_value_and_validate_return_type( domain=None, parameter_reference=self.allowed_semantic_types_passthrough, @@ -286,7 +281,7 @@ def _get_domains( column_name: str - allowed_column_names_passthrough: List[str] = [ + allowed_column_names_passthrough: list[str] = [ column_name for column_name in effective_column_names if self.semantic_type_filter.table_column_name_to_inferred_semantic_domain_type_map[ # type: ignore[operator,union-attr] # could be None @@ -301,7 +296,7 @@ def _get_domains( if column_name not in allowed_column_names_passthrough ] - metrics_for_cardinality_check: Dict[str, List[MetricConfiguration]] = ( + metrics_for_cardinality_check: dict[str, list[MetricConfiguration]] = ( self._generate_metric_configurations_to_check_cardinality( column_names=effective_column_names, batch_ids=batch_ids ) @@ -313,14 +308,14 @@ def _get_domains( " (Validator is required for cardinality checks)." ) - candidate_column_names: List[str] = self._column_names_meeting_cardinality_limit( + candidate_column_names: list[str] = self._column_names_meeting_cardinality_limit( validator=validator, metrics_for_cardinality_check=metrics_for_cardinality_check, runtime_configuration=runtime_configuration, ) candidate_column_names.extend(allowed_column_names_passthrough) - domains: List[Domain] = build_domains_from_column_names( + domains: list[Domain] = build_domains_from_column_names( rule_name=rule_name, column_names=candidate_column_names, domain_type=self.domain_type, @@ -331,9 +326,9 @@ def _get_domains( def _generate_metric_configurations_to_check_cardinality( self, - column_names: List[str], - batch_ids: Optional[List[str]] = None, - ) -> Dict[str, List[MetricConfiguration]]: + column_names: list[str], + batch_ids: Optional[list[str]] = None, + ) -> dict[str, list[MetricConfiguration]]: """Generate metric configurations used to compute metrics for checking cardinality. Args: @@ -352,7 +347,7 @@ def _generate_metric_configurations_to_check_cardinality( ) batch_id: str - metric_configurations: Dict[str, List[MetricConfiguration]] = { + metric_configurations: dict[str, list[MetricConfiguration]] = { column_name: [ MetricConfiguration( metric_name=cardinality_limit_mode.metric_name_defining_limit, @@ -372,9 +367,9 @@ def _generate_metric_configurations_to_check_cardinality( def _column_names_meeting_cardinality_limit( self, validator: Validator, - metrics_for_cardinality_check: Dict[str, List[MetricConfiguration]], + metrics_for_cardinality_check: dict[str, list[MetricConfiguration]], runtime_configuration: Optional[dict] = None, - ) -> List[str]: + ) -> list[str]: """Compute cardinality and return column names meeting cardinality limit. Args: @@ -386,10 +381,10 @@ def _column_names_meeting_cardinality_limit( List of column names meeting cardinality. """ # noqa: E501 column_name: str - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] + resolved_metrics: dict[tuple[str, str, str], MetricValue] metric_value: MetricValue - resolved_metrics_by_column_name: Dict[str, Dict[Tuple[str, str, str], MetricValue]] = ( + resolved_metrics_by_column_name: dict[str, dict[tuple[str, str, str], MetricValue]] = ( get_resolved_metrics_by_key( validator=validator, metric_configurations_by_key=metrics_for_cardinality_check, @@ -397,7 +392,7 @@ def _column_names_meeting_cardinality_limit( ) ) - candidate_column_names: List[str] = [ + candidate_column_names: list[str] = [ column_name for column_name, resolved_metrics in resolved_metrics_by_column_name.items() if all( diff --git a/great_expectations/experimental/rule_based_profiler/domain_builder/column_domain_builder.py b/great_expectations/experimental/rule_based_profiler/domain_builder/column_domain_builder.py index c15932cbbf90..8e755dcafc54 100644 --- a/great_expectations/experimental/rule_based_profiler/domain_builder/column_domain_builder.py +++ b/great_expectations/experimental/rule_based_profiler/domain_builder/column_domain_builder.py @@ -4,10 +4,7 @@ TYPE_CHECKING, ClassVar, Iterable, - List, Optional, - Set, - Tuple, Union, cast, ) @@ -42,24 +39,24 @@ class ColumnDomainBuilder(DomainBuilder): This DomainBuilder emits "Domain" object for every column in table and can serve as parent of other column-focused DomainBuilder implementations. """ # noqa: E501 - exclude_field_names: ClassVar[Set[str]] = DomainBuilder.exclude_field_names | { + exclude_field_names: ClassVar[set[str]] = DomainBuilder.exclude_field_names | { "table_column_names", "semantic_type_filter", } def __init__( # noqa: PLR0913 self, - include_column_names: Optional[Union[str, Optional[List[str]]]] = None, - exclude_column_names: Optional[Union[str, Optional[List[str]]]] = None, - include_column_name_suffixes: Optional[Union[str, Iterable, List[str]]] = None, - exclude_column_name_suffixes: Optional[Union[str, Iterable, List[str]]] = None, + include_column_names: Optional[Union[str, Optional[list[str]]]] = None, + exclude_column_names: Optional[Union[str, Optional[list[str]]]] = None, + include_column_name_suffixes: Optional[Union[str, Iterable, list[str]]] = None, + exclude_column_name_suffixes: Optional[Union[str, Iterable, list[str]]] = None, semantic_type_filter_module_name: Optional[str] = None, semantic_type_filter_class_name: Optional[str] = None, include_semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, exclude_semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: @@ -99,7 +96,7 @@ def __init__( # noqa: PLR0913 self._semantic_type_filter = None - self._table_column_names: List[str] = [] + self._table_column_names: list[str] = [] @property @override @@ -112,42 +109,42 @@ def domain_type(self) -> MetricDomainTypes: """ # noqa: E501 @property - def include_column_names(self) -> Optional[Union[str, Optional[List[str]]]]: + def include_column_names(self) -> Optional[Union[str, Optional[list[str]]]]: return self._include_column_names @include_column_names.setter - def include_column_names(self, value: Optional[Union[str, Optional[List[str]]]]) -> None: + def include_column_names(self, value: Optional[Union[str, Optional[list[str]]]]) -> None: self._include_column_names = value @property - def exclude_column_names(self) -> Optional[Union[str, Optional[List[str]]]]: + def exclude_column_names(self) -> Optional[Union[str, Optional[list[str]]]]: return self._exclude_column_names @exclude_column_names.setter - def exclude_column_names(self, value: Optional[Union[str, Optional[List[str]]]]) -> None: + def exclude_column_names(self, value: Optional[Union[str, Optional[list[str]]]]) -> None: self._exclude_column_names = value @property def include_column_name_suffixes( self, - ) -> Optional[Union[str, Iterable, List[str]]]: + ) -> Optional[Union[str, Iterable, list[str]]]: return self._include_column_name_suffixes @include_column_name_suffixes.setter def include_column_name_suffixes( - self, value: Optional[Union[str, Iterable, List[str]]] + self, value: Optional[Union[str, Iterable, list[str]]] ) -> None: self._include_column_name_suffixes = value @property def exclude_column_name_suffixes( self, - ) -> Optional[Union[str, Iterable, List[str]]]: + ) -> Optional[Union[str, Iterable, list[str]]]: return self._exclude_column_name_suffixes @exclude_column_name_suffixes.setter def exclude_column_name_suffixes( - self, value: Optional[Union[str, Iterable, List[str]]] + self, value: Optional[Union[str, Iterable, list[str]]] ) -> None: self._exclude_column_name_suffixes = value @@ -162,26 +159,26 @@ def semantic_type_filter_class_name(self) -> Optional[str]: @property def include_semantic_types( self, - ) -> Optional[Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]]]: + ) -> Optional[Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]]]: return self._include_semantic_types @include_semantic_types.setter def include_semantic_types( self, - value: Optional[Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]]], + value: Optional[Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]]], ) -> None: self._include_semantic_types = value @property def exclude_semantic_types( self, - ) -> Optional[Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]]]: + ) -> Optional[Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]]]: return self._exclude_semantic_types @exclude_semantic_types.setter def exclude_semantic_types( self, - value: Optional[Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]]], + value: Optional[Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]]], ) -> None: self._exclude_semantic_types = value @@ -191,10 +188,10 @@ def semantic_type_filter(self) -> Optional[SemanticTypeFilter]: def get_table_column_names( self, - batch_ids: Optional[List[str]] = None, + batch_ids: Optional[list[str]] = None, validator: Optional[Validator] = None, variables: Optional[ParameterContainer] = None, - ) -> List[str]: + ) -> list[str]: """ This method returns all column names available (i.e., prior to any inclusions/exclusions filtering is applied). """ # noqa: E501 @@ -202,12 +199,12 @@ def get_table_column_names( return self._table_column_names if batch_ids is None: - batch_ids: List[str] = self.get_batch_ids(variables=variables) # type: ignore[no-redef] + batch_ids: list[str] = self.get_batch_ids(variables=variables) # type: ignore[no-redef] if validator is None: validator = self.get_validator(variables=variables) - table_columns: List[str] = validator.get_metric( # type: ignore[union-attr] # could be None + table_columns: list[str] = validator.get_metric( # type: ignore[union-attr] # could be None metric=MetricConfiguration( metric_name="table.columns", metric_domain_kwargs={ @@ -225,16 +222,16 @@ def get_table_column_names( def get_filtered_column_names( # noqa: C901 self, - column_names: List[str], - batch_ids: Optional[List[str]] = None, + column_names: list[str], + batch_ids: Optional[list[str]] = None, validator: Optional[Validator] = None, variables: Optional[ParameterContainer] = None, - ) -> List[str]: + ) -> list[str]: """ This method returns list of column names, filtered according to directives supplied via instance attributes. """ # noqa: E501 - include_column_names: List[str] = cast( - List[str], + include_column_names: list[str] = cast( + list[str], self._resolve_list_type_property( property_name="include_column_names", property_value_type=list, @@ -242,10 +239,10 @@ def get_filtered_column_names( # noqa: C901 ), ) - filtered_column_names: List[str] = include_column_names or column_names + filtered_column_names: list[str] = include_column_names or column_names - exclude_column_names: List[str] = cast( - List[str], + exclude_column_names: list[str] = cast( + list[str], self._resolve_list_type_property( property_name="exclude_column_names", property_value_type=list, @@ -267,8 +264,8 @@ def get_filtered_column_names( # noqa: C901 message=f'Error: The column "{column_name}" in BatchData does not exist.' ) - include_column_name_suffixes: List[str] = cast( - List[str], + include_column_name_suffixes: list[str] = cast( + list[str], self._resolve_list_type_property( property_name="include_column_name_suffixes", property_value_type=(str, Iterable, list), @@ -285,8 +282,8 @@ def get_filtered_column_names( # noqa: C901 ) ) - exclude_column_name_suffixes: List[str] = cast( - List[str], + exclude_column_name_suffixes: list[str] = cast( + list[str], self._resolve_list_type_property( property_name="exclude_column_name_suffixes", property_value_type=(str, Iterable, list), @@ -343,8 +340,8 @@ def get_filtered_column_names( # noqa: C901 ) self._semantic_type_filter = semantic_type_filter # type: ignore[assignment] # could be None - include_semantic_types: List[Union[str, SemanticDomainTypes]] = cast( - List[Union[str, SemanticDomainTypes]], + include_semantic_types: list[Union[str, SemanticDomainTypes]] = cast( + list[Union[str, SemanticDomainTypes]], self._resolve_list_type_property( property_name="include_semantic_types", property_value_type=(str, SemanticDomainTypes, list), @@ -366,8 +363,8 @@ def get_filtered_column_names( # noqa: C901 ) ) - exclude_semantic_types: List[Union[str, SemanticDomainTypes]] = cast( - List[Union[str, SemanticDomainTypes]], + exclude_semantic_types: list[Union[str, SemanticDomainTypes]] = cast( + list[Union[str, SemanticDomainTypes]], self._resolve_list_type_property( property_name="exclude_semantic_types", property_value_type=(str, SemanticDomainTypes, list), @@ -393,26 +390,26 @@ def get_filtered_column_names( # noqa: C901 def get_effective_column_names( self, - batch_ids: Optional[List[str]] = None, + batch_ids: Optional[list[str]] = None, validator: Optional[Validator] = None, variables: Optional[ParameterContainer] = None, - ) -> List[str]: + ) -> list[str]: """ This method applies multiple directives to obtain columns to be included as part of returned "Domain" objects. """ # noqa: E501 if batch_ids is None: - batch_ids: List[str] = self.get_batch_ids(variables=variables) # type: ignore[no-redef] + batch_ids: list[str] = self.get_batch_ids(variables=variables) # type: ignore[no-redef] if validator is None: validator = self.get_validator(variables=variables) - table_columns: List[str] = self.get_table_column_names( + table_columns: list[str] = self.get_table_column_names( batch_ids=batch_ids, validator=validator, variables=variables, ) - effective_column_names: List[str] = self.get_filtered_column_names( + effective_column_names: list[str] = self.get_filtered_column_names( column_names=table_columns, batch_ids=batch_ids, validator=validator, @@ -427,7 +424,7 @@ def _get_domains( rule_name: str, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: """ Obtains and returns domains for all columns of a table (or for configured columns, if they exist in the table). @@ -439,17 +436,17 @@ def _get_domains( Returns: List of domains that match the desired columns and filtering criteria. """ # noqa: E501 - batch_ids: List[str] = self.get_batch_ids(variables=variables) # type: ignore[assignment] # could be None + batch_ids: list[str] = self.get_batch_ids(variables=variables) # type: ignore[assignment] # could be None validator: Validator = self.get_validator(variables=variables) # type: ignore[assignment] # could be None - effective_column_names: List[str] = self.get_effective_column_names( + effective_column_names: list[str] = self.get_effective_column_names( batch_ids=batch_ids, validator=validator, variables=variables, ) - domains: List[Domain] = build_domains_from_column_names( + domains: list[Domain] = build_domains_from_column_names( rule_name=rule_name, column_names=effective_column_names, domain_type=self.domain_type, @@ -461,9 +458,9 @@ def _get_domains( def _resolve_list_type_property( self, property_name: str, - property_value_type: Union[type, Tuple[type, ...]], + property_value_type: Union[type, tuple[type, ...]], variables: Optional[ParameterContainer] = None, - ) -> List[type]: + ) -> list[type]: property_value = getattr(self, property_name, []) if property_value is None: property_value = [] diff --git a/great_expectations/experimental/rule_based_profiler/domain_builder/column_pair_domain_builder.py b/great_expectations/experimental/rule_based_profiler/domain_builder/column_pair_domain_builder.py index ea9726784ebf..62471bf062bf 100644 --- a/great_expectations/experimental/rule_based_profiler/domain_builder/column_pair_domain_builder.py +++ b/great_expectations/experimental/rule_based_profiler/domain_builder/column_pair_domain_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, ClassVar, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.domain import ( @@ -27,7 +27,7 @@ class ColumnPairDomainBuilder(ColumnDomainBuilder): This DomainBuilder uses "include_column_names" property of its parent class to specify "column_A" and "column_B" (order-preserving). """ # noqa: E501 - exclude_field_names: ClassVar[Set[str]] = ColumnDomainBuilder.exclude_field_names | { + exclude_field_names: ClassVar[set[str]] = ColumnDomainBuilder.exclude_field_names | { "exclude_column_names", "include_column_name_suffixes", "exclude_column_name_suffixes", @@ -39,7 +39,7 @@ class ColumnPairDomainBuilder(ColumnDomainBuilder): def __init__( self, - include_column_names: Optional[Union[str, Optional[List[str]]]] = None, + include_column_names: Optional[Union[str, Optional[list[str]]]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -70,7 +70,7 @@ def _get_domains( rule_name: str, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: """Obtains and returns Domain object, whose domain_kwargs consists of "column_A" and "column_B" (order-preserving) column-pair. Args: @@ -81,11 +81,11 @@ def _get_domains( Returns: List of domains that match the desired tolerance limits. """ # noqa: E501 - batch_ids: List[str] = self.get_batch_ids(variables=variables) # type: ignore[assignment] # could be None + batch_ids: list[str] = self.get_batch_ids(variables=variables) # type: ignore[assignment] # could be None validator: Validator = self.get_validator(variables=variables) # type: ignore[assignment] # could be None - effective_column_names: List[str] = self.get_effective_column_names( + effective_column_names: list[str] = self.get_effective_column_names( batch_ids=batch_ids, validator=validator, variables=variables, @@ -100,7 +100,7 @@ def _get_domains( """ # noqa: E501 ) - domain_kwargs: Dict[str, str] = dict( + domain_kwargs: dict[str, str] = dict( zip( [ "column_A", @@ -111,14 +111,14 @@ def _get_domains( ) column_name: str - semantic_types_by_column_name: Dict[str, SemanticDomainTypes] = { + semantic_types_by_column_name: dict[str, SemanticDomainTypes] = { column_name: self.semantic_type_filter.table_column_name_to_inferred_semantic_domain_type_map[ # type: ignore[union-attr] # could be None # noqa: E501 column_name ] for column_name in effective_column_names } - domains: List[Domain] = [ + domains: list[Domain] = [ Domain( domain_type=self.domain_type, domain_kwargs=domain_kwargs, diff --git a/great_expectations/experimental/rule_based_profiler/domain_builder/domain_builder.py b/great_expectations/experimental/rule_based_profiler/domain_builder/domain_builder.py index 4dfd6ceee874..25a48c82fd3f 100644 --- a/great_expectations/experimental/rule_based_profiler/domain_builder/domain_builder.py +++ b/great_expectations/experimental/rule_based_profiler/domain_builder/domain_builder.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from great_expectations.core.batch import Batch, BatchRequestBase # noqa: TCH001 from great_expectations.core.domain import Domain # noqa: TCH001 @@ -50,10 +50,10 @@ def get_domains( self, rule_name: str, variables: Optional[ParameterContainer] = None, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: """ Args: rule_name: name of Rule object, for which "Domain" objects are obtained. @@ -90,7 +90,7 @@ def _get_domains( rule_name: str, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: """ _get_domains is the primary workhorse for the DomainBuilder """ @@ -100,10 +100,10 @@ def _get_domains( def get_table_row_counts( self, validator: Optional[Validator] = None, - batch_ids: Optional[List[str]] = None, + batch_ids: Optional[list[str]] = None, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> Dict[str, int]: + ) -> dict[str, int]: if validator is None: validator = self.get_validator(variables=variables) @@ -112,7 +112,7 @@ def get_table_row_counts( batch_id: str - metric_configurations_by_batch_id: Dict[str, List[MetricConfiguration]] = { + metric_configurations_by_batch_id: dict[str, list[MetricConfiguration]] = { batch_id: [ MetricConfiguration( metric_name="table.row_count", @@ -127,7 +127,7 @@ def get_table_row_counts( for batch_id in batch_ids # type: ignore[union-attr] # could be None } - resolved_metrics_by_batch_id: Dict[str, Dict[Tuple[str, str, str], MetricValue]] = ( + resolved_metrics_by_batch_id: dict[str, dict[tuple[str, str, str], MetricValue]] = ( get_resolved_metrics_by_key( validator=validator, # type: ignore[arg-type] # could be None metric_configurations_by_key=metric_configurations_by_batch_id, @@ -135,13 +135,13 @@ def get_table_row_counts( ) ) - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] + resolved_metrics: dict[tuple[str, str, str], MetricValue] metric_value: Any - table_row_count_lists_by_batch_id: Dict[str, List[int]] = { + table_row_count_lists_by_batch_id: dict[str, list[int]] = { batch_id: [metric_value for metric_value in resolved_metrics.values()] # type: ignore[misc] # incompatible values for batch_id, resolved_metrics in resolved_metrics_by_batch_id.items() } - table_row_counts_by_batch_id: Dict[str, int] = { + table_row_counts_by_batch_id: dict[str, int] = { batch_id: metric_value[0] for batch_id, metric_value in table_row_count_lists_by_batch_id.items() } @@ -165,7 +165,7 @@ def get_validator( def get_batch_ids( self, variables: Optional[ParameterContainer] = None, - ) -> Optional[List[str]]: + ) -> Optional[list[str]]: return get_batch_ids_from_batch_list_or_batch_request( data_context=self.data_context, batch_list=self.batch_list, diff --git a/great_expectations/experimental/rule_based_profiler/domain_builder/map_metric_column_domain_builder.py b/great_expectations/experimental/rule_based_profiler/domain_builder/map_metric_column_domain_builder.py index b75c12b515fe..46d7af2cc20f 100644 --- a/great_expectations/experimental/rule_based_profiler/domain_builder/map_metric_column_domain_builder.py +++ b/great_expectations/experimental/rule_based_profiler/domain_builder/map_metric_column_domain_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Iterable, Optional, Union from great_expectations.core.domain import Domain, SemanticDomainTypes # noqa: TCH001 from great_expectations.core.metric_function_types import ( @@ -34,17 +34,17 @@ class MapMetricColumnDomainBuilder(ColumnDomainBuilder): def __init__( # noqa: PLR0913 self, map_metric_name: str, - include_column_names: Optional[Union[str, Optional[List[str]]]] = None, - exclude_column_names: Optional[Union[str, Optional[List[str]]]] = None, - include_column_name_suffixes: Optional[Union[str, Iterable, List[str]]] = None, - exclude_column_name_suffixes: Optional[Union[str, Iterable, List[str]]] = None, + include_column_names: Optional[Union[str, Optional[list[str]]]] = None, + exclude_column_names: Optional[Union[str, Optional[list[str]]]] = None, + include_column_name_suffixes: Optional[Union[str, Iterable, list[str]]] = None, + exclude_column_name_suffixes: Optional[Union[str, Iterable, list[str]]] = None, semantic_type_filter_module_name: Optional[str] = None, semantic_type_filter_class_name: Optional[str] = None, include_semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, exclude_semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, max_unexpected_values: Union[str, int] = 0, max_unexpected_ratio: Optional[Union[str, float]] = None, @@ -137,7 +137,7 @@ def _get_domains( rule_name: str, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: """Return domains matching the specified tolerance limits. Args: @@ -184,18 +184,18 @@ def _get_domains( parameters=None, ) - batch_ids: List[str] = self.get_batch_ids(variables=variables) + batch_ids: list[str] = self.get_batch_ids(variables=variables) num_batch_ids: int = len(batch_ids) validator: Validator = self.get_validator(variables=variables) - table_column_names: List[str] = self.get_effective_column_names( + table_column_names: list[str] = self.get_effective_column_names( batch_ids=batch_ids, validator=validator, variables=variables, ) - table_row_counts: Dict[str, int] = self.get_table_row_counts( + table_row_counts: dict[str, int] = self.get_table_row_counts( validator=validator, batch_ids=batch_ids, variables=variables, @@ -209,7 +209,7 @@ def _get_domains( if max_unexpected_ratio is None: max_unexpected_ratio = max_unexpected_values / mean_table_row_count_as_float - metric_configurations_by_column_name: Dict[str, List[MetricConfiguration]] = ( + metric_configurations_by_column_name: dict[str, list[MetricConfiguration]] = ( self._generate_metric_configurations( map_metric_name=map_metric_name, batch_ids=batch_ids, @@ -217,7 +217,7 @@ def _get_domains( ) ) - candidate_column_names: List[str] = self._get_column_names_satisfying_tolerance_limits( + candidate_column_names: list[str] = self._get_column_names_satisfying_tolerance_limits( validator=validator, num_batch_ids=num_batch_ids, metric_configurations_by_column_name=metric_configurations_by_column_name, @@ -227,7 +227,7 @@ def _get_domains( ) column_name: str - domains: List[Domain] = build_domains_from_column_names( + domains: list[Domain] = build_domains_from_column_names( rule_name=rule_name, column_names=candidate_column_names, domain_type=self.domain_type, @@ -239,9 +239,9 @@ def _get_domains( @staticmethod def _generate_metric_configurations( map_metric_name: str, - batch_ids: List[str], - column_names: List[str], - ) -> Dict[str, List[MetricConfiguration]]: + batch_ids: list[str], + column_names: list[str], + ) -> dict[str, list[MetricConfiguration]]: """ Generate metric configurations used to compute "unexpected_count" values for "map_metric_name". @@ -258,7 +258,7 @@ def _generate_metric_configurations( """ # noqa: E501 column_name: str batch_id: str - metric_configurations: Dict[str, List[MetricConfiguration]] = { + metric_configurations: dict[str, list[MetricConfiguration]] = { column_name: [ MetricConfiguration( metric_name=f"{map_metric_name}.{SummarizationMetricNameSuffixes.UNEXPECTED_COUNT.value}", @@ -279,12 +279,12 @@ def _generate_metric_configurations( def _get_column_names_satisfying_tolerance_limits( # noqa: PLR0913 validator: Validator, num_batch_ids: int, - metric_configurations_by_column_name: Dict[str, List[MetricConfiguration]], + metric_configurations_by_column_name: dict[str, list[MetricConfiguration]], mean_table_row_count_as_float: float, max_unexpected_ratio: float, min_max_unexpected_values_proportion: float, runtime_configuration: Optional[dict] = None, - ) -> List[str]: + ) -> list[str]: """ Compute figures of merit and return column names satisfying tolerance limits. @@ -300,9 +300,9 @@ def _get_column_names_satisfying_tolerance_limits( # noqa: PLR0913 List of column names satisfying tolerance limits. """ # noqa: E501 column_name: str - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] + resolved_metrics: dict[tuple[str, str, str], MetricValue] - resolved_metrics_by_column_name: Dict[str, Dict[Tuple[str, str, str], MetricValue]] = ( + resolved_metrics_by_column_name: dict[str, dict[tuple[str, str, str], MetricValue]] = ( get_resolved_metrics_by_key( validator=validator, metric_configurations_by_key=metric_configurations_by_column_name, @@ -311,7 +311,7 @@ def _get_column_names_satisfying_tolerance_limits( # noqa: PLR0913 ) metric_value: Any - intra_batch_unexpected_ratios_by_column_name: Dict[str, List[float]] = { + intra_batch_unexpected_ratios_by_column_name: dict[str, list[float]] = { column_name: [ (metric_value or 0.0) / mean_table_row_count_as_float for metric_value in resolved_metrics.values() @@ -319,9 +319,9 @@ def _get_column_names_satisfying_tolerance_limits( # noqa: PLR0913 for column_name, resolved_metrics in resolved_metrics_by_column_name.items() } - metric_value_ratios: List[float] + metric_value_ratios: list[float] metric_value_ratio: float - intra_batch_adherence_by_column_name: Dict[str, List[bool]] = { + intra_batch_adherence_by_column_name: dict[str, list[bool]] = { column_name: [ metric_value_ratio <= max_unexpected_ratio for metric_value_ratio in metric_value_ratios @@ -329,7 +329,7 @@ def _get_column_names_satisfying_tolerance_limits( # noqa: PLR0913 for column_name, metric_value_ratios in intra_batch_unexpected_ratios_by_column_name.items() # noqa: E501 } - inter_batch_adherence_by_column_name: Dict[str, float] = { + inter_batch_adherence_by_column_name: dict[str, float] = { column_name: 1.0 * sum(intra_batch_adherence_by_column_name[column_name]) / num_batch_ids @@ -337,7 +337,7 @@ def _get_column_names_satisfying_tolerance_limits( # noqa: PLR0913 } inter_batch_unexpected_values_proportion: float - candidate_column_names: List[str] = [ + candidate_column_names: list[str] = [ column_name for column_name, inter_batch_unexpected_values_proportion in inter_batch_adherence_by_column_name.items() # noqa: E501 if inter_batch_unexpected_values_proportion >= min_max_unexpected_values_proportion diff --git a/great_expectations/experimental/rule_based_profiler/domain_builder/multi_column_domain_builder.py b/great_expectations/experimental/rule_based_profiler/domain_builder/multi_column_domain_builder.py index be320c054440..aa32aefab671 100644 --- a/great_expectations/experimental/rule_based_profiler/domain_builder/multi_column_domain_builder.py +++ b/great_expectations/experimental/rule_based_profiler/domain_builder/multi_column_domain_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, ClassVar, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.domain import ( @@ -27,7 +27,7 @@ class MultiColumnDomainBuilder(ColumnDomainBuilder): This DomainBuilder uses "include_column_names" property of its parent class to specify "column_list" (order-non-preserving). """ # noqa: E501 - exclude_field_names: ClassVar[Set[str]] = ColumnDomainBuilder.exclude_field_names | { + exclude_field_names: ClassVar[set[str]] = ColumnDomainBuilder.exclude_field_names | { "exclude_column_names", "include_column_name_suffixes", "exclude_column_name_suffixes", @@ -39,7 +39,7 @@ class MultiColumnDomainBuilder(ColumnDomainBuilder): def __init__( self, - include_column_names: Optional[Union[str, Optional[List[str]]]] = None, + include_column_names: Optional[Union[str, Optional[list[str]]]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -70,7 +70,7 @@ def _get_domains( rule_name: str, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: """Obtains and returns Domain object, whose domain_kwargs consists of "column_list" (order-non-preserving). Args: @@ -81,11 +81,11 @@ def _get_domains( Returns: List of domains that match the desired tolerance limits. """ # noqa: E501 - batch_ids: List[str] = self.get_batch_ids(variables=variables) # type: ignore[assignment] # could be None + batch_ids: list[str] = self.get_batch_ids(variables=variables) # type: ignore[assignment] # could be None validator: Validator = self.get_validator(variables=variables) # type: ignore[assignment] # could be None - effective_column_names: List[str] = self.get_effective_column_names( + effective_column_names: list[str] = self.get_effective_column_names( batch_ids=batch_ids, validator=validator, variables=variables, @@ -97,14 +97,14 @@ def _get_domains( ) column_name: str - semantic_types_by_column_name: Dict[str, SemanticDomainTypes] = { + semantic_types_by_column_name: dict[str, SemanticDomainTypes] = { column_name: self.semantic_type_filter.table_column_name_to_inferred_semantic_domain_type_map[ # type: ignore[union-attr] # could be None # noqa: E501 column_name ] for column_name in effective_column_names } - domains: List[Domain] = [ + domains: list[Domain] = [ Domain( domain_type=self.domain_type, domain_kwargs={ diff --git a/great_expectations/experimental/rule_based_profiler/domain_builder/table_domain_builder.py b/great_expectations/experimental/rule_based_profiler/domain_builder/table_domain_builder.py index d103b5d3f2e9..0cc060cb0570 100644 --- a/great_expectations/experimental/rule_based_profiler/domain_builder/table_domain_builder.py +++ b/great_expectations/experimental/rule_based_profiler/domain_builder/table_domain_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.core.domain import Domain @@ -49,7 +49,7 @@ def _get_domains( rule_name: str, variables: Optional[ParameterContainer] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: + ) -> list[Domain]: other_table_name: Optional[str] try: # Obtain table from "rule state" (i.e., variables and parameters); from instance variable otherwise. # noqa: E501 @@ -63,7 +63,7 @@ def _get_domains( except KeyError: other_table_name = None - domains: List[Domain] + domains: list[Domain] if other_table_name: domains = [ Domain( diff --git a/great_expectations/experimental/rule_based_profiler/estimators/bootstrap_numeric_range_estimator.py b/great_expectations/experimental/rule_based_profiler/estimators/bootstrap_numeric_range_estimator.py index 7e72439f4804..4a62921db12e 100644 --- a/great_expectations/experimental/rule_based_profiler/estimators/bootstrap_numeric_range_estimator.py +++ b/great_expectations/experimental/rule_based_profiler/estimators/bootstrap_numeric_range_estimator.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Optional from great_expectations.experimental.rule_based_profiler.estimators.numeric_range_estimator import ( NumericRangeEstimator, @@ -57,7 +57,7 @@ def _get_numeric_range_estimate( metric_values: np.ndarray, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> NumericRangeEstimationResult: if is_ndarray_datetime_dtype( data=metric_values, diff --git a/great_expectations/experimental/rule_based_profiler/estimators/exact_numeric_range_estimator.py b/great_expectations/experimental/rule_based_profiler/estimators/exact_numeric_range_estimator.py index 1cce9e97901a..af91fc376a8b 100644 --- a/great_expectations/experimental/rule_based_profiler/estimators/exact_numeric_range_estimator.py +++ b/great_expectations/experimental/rule_based_profiler/estimators/exact_numeric_range_estimator.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Optional import numpy as np @@ -50,7 +50,7 @@ def _get_numeric_range_estimate( metric_values: npt.NDArray, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> NumericRangeEstimationResult: datetime_detected: bool = datetime_semantic_domain_type(domain=domain) metric_values_converted: npt.NDArray diff --git a/great_expectations/experimental/rule_based_profiler/estimators/kde_numeric_range_estimator.py b/great_expectations/experimental/rule_based_profiler/estimators/kde_numeric_range_estimator.py index 8604f84f1a5a..95f402179cf6 100644 --- a/great_expectations/experimental/rule_based_profiler/estimators/kde_numeric_range_estimator.py +++ b/great_expectations/experimental/rule_based_profiler/estimators/kde_numeric_range_estimator.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Callable, Dict, Optional, Union +from typing import TYPE_CHECKING, Callable, Optional, Union from great_expectations.experimental.rule_based_profiler.estimators.numeric_range_estimator import ( NumericRangeEstimator, @@ -56,7 +56,7 @@ def _get_numeric_range_estimate( metric_values: np.ndarray, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> NumericRangeEstimationResult: if is_ndarray_datetime_dtype( data=metric_values, diff --git a/great_expectations/experimental/rule_based_profiler/estimators/numeric_range_estimation_result.py b/great_expectations/experimental/rule_based_profiler/estimators/numeric_range_estimation_result.py index cb34fece1236..c96dfdb74532 100644 --- a/great_expectations/experimental/rule_based_profiler/estimators/numeric_range_estimation_result.py +++ b/great_expectations/experimental/rule_based_profiler/estimators/numeric_range_estimation_result.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.types import DictDot @@ -27,7 +27,7 @@ class NumericRangeEstimationResult(DictDot): """ # noqa: E501 estimation_histogram: np.ndarray - value_range: Union[np.ndarray, List[np.float64]] + value_range: Union[np.ndarray, list[np.float64]] @override def to_dict(self) -> dict: diff --git a/great_expectations/experimental/rule_based_profiler/estimators/numeric_range_estimator.py b/great_expectations/experimental/rule_based_profiler/estimators/numeric_range_estimator.py index d7c205548a52..97900a7313cf 100644 --- a/great_expectations/experimental/rule_based_profiler/estimators/numeric_range_estimator.py +++ b/great_expectations/experimental/rule_based_profiler/estimators/numeric_range_estimator.py @@ -2,7 +2,7 @@ import logging from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.types import SerializableDictDot @@ -62,7 +62,7 @@ def get_numeric_range_estimate( metric_values: np.ndarray, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> NumericRangeEstimationResult: """ Method that invokes implementation of the estimation algorithm that is the subject of the inherited class. @@ -88,7 +88,7 @@ def _get_numeric_range_estimate( metric_values: np.ndarray, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> NumericRangeEstimationResult: """ Essentials of the estimation algorithm (all subclasses must implement this method). diff --git a/great_expectations/experimental/rule_based_profiler/estimators/quantiles_numeric_range_estimator.py b/great_expectations/experimental/rule_based_profiler/estimators/quantiles_numeric_range_estimator.py index 9403964bd17d..fcb7076717fc 100644 --- a/great_expectations/experimental/rule_based_profiler/estimators/quantiles_numeric_range_estimator.py +++ b/great_expectations/experimental/rule_based_profiler/estimators/quantiles_numeric_range_estimator.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Dict, Final, Optional +from typing import TYPE_CHECKING, Final, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.core.domain import Domain # noqa: TCH001 @@ -56,7 +56,7 @@ def _get_numeric_range_estimate( metric_values: np.ndarray, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> NumericRangeEstimationResult: false_positive_rate: np.float64 = get_false_positive_rate_from_rule_state( # type: ignore[assignment] # could be float false_positive_rate=self.configuration.false_positive_rate, # type: ignore[union-attr] # configuration could be None diff --git a/great_expectations/experimental/rule_based_profiler/expectation_configuration_builder/default_expectation_configuration_builder.py b/great_expectations/experimental/rule_based_profiler/expectation_configuration_builder/default_expectation_configuration_builder.py index 260497856cce..a00d3f985c92 100644 --- a/great_expectations/experimental/rule_based_profiler/expectation_configuration_builder/default_expectation_configuration_builder.py +++ b/great_expectations/experimental/rule_based_profiler/expectation_configuration_builder/default_expectation_configuration_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union from pyparsing import ( Combine, @@ -70,7 +70,7 @@ class DefaultExpectationConfigurationBuilder(ExpectationConfigurationBuilder): ExpectationConfigurations can be optionally filtered if a supplied condition is met. """ # noqa: E501 - exclude_field_names: ClassVar[Set[str]] = ( + exclude_field_names: ClassVar[set[str]] = ( ExpectationConfigurationBuilder.exclude_field_names | { "kwargs", @@ -80,9 +80,9 @@ class DefaultExpectationConfigurationBuilder(ExpectationConfigurationBuilder): def __init__( self, expectation_type: str, - meta: Optional[Dict[str, Any]] = None, + meta: Optional[dict[str, Any]] = None, condition: Optional[str] = None, - validation_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + validation_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, **kwargs, ) -> None: @@ -142,7 +142,7 @@ def condition(self) -> Optional[str]: @property def validation_parameter_builder_configs( self, - ) -> Optional[List[ParameterBuilderConfig]]: + ) -> Optional[list[ParameterBuilderConfig]]: return self._validation_parameter_builder_configs @property @@ -173,7 +173,7 @@ def _substitute_parameters_and_variables( term_list: Union[str, ParseResults], domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> ParseResults: """Recursively substitute all parameters and variables in term list @@ -219,7 +219,7 @@ def _substitute_parameters_and_variables( token: Union[str, ParseResults] for idx, token in enumerate(term_list): if isinstance(token, str) and token.startswith("$"): - term_list[idx]: Dict[str, Any] = get_parameter_value_and_validate_return_type( + term_list[idx]: dict[str, Any] = get_parameter_value_and_validate_return_type( domain=domain, parameter_reference=token, expected_return_type=None, @@ -279,7 +279,7 @@ def _build_binary_list( def _build_boolean_result( self, - binary_list: Union[ParseResults, List[Union[bool, str]]], + binary_list: Union[ParseResults, list[Union[bool, str]]], ) -> bool: """Recursively build boolean result from binary list @@ -322,7 +322,7 @@ def _evaluate_condition( parsed_condition: ParseResults, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> bool: """Evaluates the parsed condition to True/False and returns the boolean result""" substituted_term_list: ParseResults = self._substitute_parameters_and_variables( @@ -343,13 +343,13 @@ def _build_expectation_configuration( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Optional[ExpectationConfiguration]: """Returns either and ExpectationConfiguration object or None depending on evaluation of condition""" # noqa: E501 parameter_name: str fully_qualified_parameter_name: str - expectation_kwargs: Dict[str, Any] = { + expectation_kwargs: dict[str, Any] = { parameter_name: get_parameter_value_and_validate_return_type( domain=domain, parameter_reference=fully_qualified_parameter_name, @@ -359,7 +359,7 @@ def _build_expectation_configuration( ) for parameter_name, fully_qualified_parameter_name in self.kwargs.items() } - meta: Dict[str, Any] = get_parameter_value_and_validate_return_type( + meta: dict[str, Any] = get_parameter_value_and_validate_return_type( domain=domain, parameter_reference=self.meta, expected_return_type=dict, diff --git a/great_expectations/experimental/rule_based_profiler/expectation_configuration_builder/expectation_configuration_builder.py b/great_expectations/experimental/rule_based_profiler/expectation_configuration_builder/expectation_configuration_builder.py index 20aa7333556f..e672cca4b416 100644 --- a/great_expectations/experimental/rule_based_profiler/expectation_configuration_builder/expectation_configuration_builder.py +++ b/great_expectations/experimental/rule_based_profiler/expectation_configuration_builder/expectation_configuration_builder.py @@ -2,7 +2,7 @@ import logging from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, ClassVar, Optional, Union from great_expectations.core.batch import Batch, BatchRequestBase # noqa: TCH001 from great_expectations.core.domain import Domain # noqa: TCH001 @@ -33,14 +33,14 @@ class ExpectationConfigurationBuilder(ABC, Builder): - exclude_field_names: ClassVar[Set[str]] = Builder.exclude_field_names | { + exclude_field_names: ClassVar[set[str]] = Builder.exclude_field_names | { "validation_parameter_builders", } def __init__( self, expectation_type: str, - validation_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + validation_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, **kwargs, ) -> None: @@ -80,8 +80,8 @@ def build_expectation_configuration( # noqa: PLR0913 self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, - batch_list: Optional[List[Batch]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, runtime_configuration: Optional[dict] = None, ) -> ExpectationConfiguration | None: @@ -128,12 +128,12 @@ def resolve_validation_dependencies( # noqa: PLR0913 self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, - batch_list: Optional[List[Batch]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, runtime_configuration: Optional[dict] = None, ) -> None: - validation_parameter_builders: List[ParameterBuilder] = ( + validation_parameter_builders: list[ParameterBuilder] = ( self.validation_parameter_builders or [] ) @@ -154,7 +154,7 @@ def _build_expectation_configuration( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> ExpectationConfiguration | None: pass @@ -164,14 +164,14 @@ def expectation_type(self) -> str: return self._expectation_type @property - def validation_parameter_builders(self) -> Optional[List[ParameterBuilder]]: + def validation_parameter_builders(self) -> Optional[list[ParameterBuilder]]: return self._validation_parameter_builders def init_rule_expectation_configuration_builders( - expectation_configuration_builder_configs: List[dict], + expectation_configuration_builder_configs: list[dict], data_context: Optional[AbstractDataContext] = None, -) -> List[ExpectationConfigurationBuilder]: +) -> list[ExpectationConfigurationBuilder]: expectation_configuration_builder_config: dict return [ init_expectation_configuration_builder( diff --git a/great_expectations/experimental/rule_based_profiler/helpers/runtime_environment.py b/great_expectations/experimental/rule_based_profiler/helpers/runtime_environment.py index 2698db973e05..aaace68ff60d 100644 --- a/great_expectations/experimental/rule_based_profiler/helpers/runtime_environment.py +++ b/great_expectations/experimental/rule_based_profiler/helpers/runtime_environment.py @@ -2,7 +2,7 @@ import logging from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.core.metric_domain_types import MetricDomainTypes @@ -22,7 +22,7 @@ @dataclass class RuntimeEnvironmentVariablesDirectives(SerializableDictDot): rule_name: str - variables: Optional[Dict[str, Any]] = None + variables: Optional[dict[str, Any]] = None @override def to_dict(self) -> dict: @@ -42,7 +42,7 @@ def to_json_dict(self) -> dict: @dataclass class RuntimeEnvironmentDomainTypeDirectives(SerializableDictDot): domain_type: MetricDomainTypes - directives: Dict[str, Any] + directives: dict[str, Any] @override def to_dict(self) -> dict: @@ -61,21 +61,21 @@ def to_json_dict(self) -> dict: def build_variables_directives( exact_estimation: bool, - rules: List[Rule], + rules: list[Rule], **kwargs: dict, -) -> List[RuntimeEnvironmentVariablesDirectives]: +) -> list[RuntimeEnvironmentVariablesDirectives]: """ This method makes best-effort attempt to identify directives, supplied in "kwargs", as "variables", referenced by components of "Rule" objects, identified by respective "rule_name" property as indicated, and return each of these directives as part of dedicated "RuntimeEnvironmentVariablesDirectives" typed object for every "rule_name" (string). """ # noqa: E501 # Implementation relies on assumption that "kwargs" contains "variables"-level arguments/directives only. # noqa: E501 - directives: Dict[ - str, Dict[str, Any] + directives: dict[ + str, dict[str, Any] ] # key is "rule_name"; value is "variables" in corresponding "Rule" object if exact_estimation: directives = {} - rule_variables_configs: Optional[Dict[str, Any]] + rule_variables_configs: Optional[dict[str, Any]] rule: Rule for rule in rules: rule_variables_configs = convert_variables_to_dict(variables=rule.variables) @@ -106,7 +106,7 @@ def build_variables_directives( def build_domain_type_directives( **kwargs: dict, -) -> List[RuntimeEnvironmentDomainTypeDirectives]: +) -> list[RuntimeEnvironmentDomainTypeDirectives]: """ This method makes best-effort attempt to identify directives, supplied in "kwargs", as supported properties, corresponnding to "DomainBuilder" classes, associated with every "MetricDomainTypes", and return each of these @@ -119,7 +119,7 @@ def build_domain_type_directives( "domain_type_directives_list" is declared as "List" and a single "RuntimeEnvironmentDomainTypeDirectives" element is appended, instead of setting "domain_type_directives_list" to contain that element explicitly. """ # noqa: E501 - domain_type_directives_list: List[RuntimeEnvironmentDomainTypeDirectives] = [] + domain_type_directives_list: list[RuntimeEnvironmentDomainTypeDirectives] = [] column_domain_type_directives: RuntimeEnvironmentDomainTypeDirectives = ( RuntimeEnvironmentDomainTypeDirectives( diff --git a/great_expectations/experimental/rule_based_profiler/helpers/simple_semantic_type_filter.py b/great_expectations/experimental/rule_based_profiler/helpers/simple_semantic_type_filter.py index a0915c79c465..c0fe9411dfed 100644 --- a/great_expectations/experimental/rule_based_profiler/helpers/simple_semantic_type_filter.py +++ b/great_expectations/experimental/rule_based_profiler/helpers/simple_semantic_type_filter.py @@ -3,11 +3,8 @@ from typing import ( TYPE_CHECKING, Any, - Dict, - List, Optional, Sequence, - Type, TypeVar, Union, ) @@ -32,7 +29,7 @@ T = TypeVar("T") -def _is_sequence_of(sequence: Sequence, type_: Type[T]) -> TypeGuard[Sequence[T]]: +def _is_sequence_of(sequence: Sequence, type_: type[T]) -> TypeGuard[Sequence[T]]: return all(isinstance(x, type_) for x in sequence) @@ -43,9 +40,9 @@ class SimpleSemanticTypeFilter(SemanticTypeFilter): def __init__( self, - batch_ids: List[str], + batch_ids: list[str], validator: Validator, - column_names: Optional[List[str]] = None, + column_names: Optional[list[str]] = None, ) -> None: self._build_table_column_name_to_inferred_semantic_domain_type_map( batch_ids=batch_ids, @@ -57,16 +54,16 @@ def __init__( @override def table_column_name_to_inferred_semantic_domain_type_map( self, - ) -> Dict[str, SemanticDomainTypes]: + ) -> dict[str, SemanticDomainTypes]: return self._table_column_name_to_inferred_semantic_domain_type_map # type: ignore[return-value] # could be None @override def parse_semantic_domain_type_argument( self, semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, - ) -> List[SemanticDomainTypes]: + ) -> list[SemanticDomainTypes]: if semantic_types is None: return [] @@ -95,11 +92,11 @@ def parse_semantic_domain_type_argument( def _build_table_column_name_to_inferred_semantic_domain_type_map( self, - batch_ids: List[str], + batch_ids: list[str], validator: Validator, - column_names: Optional[List[str]] = None, + column_names: Optional[list[str]] = None, ) -> None: - column_types_dict_list: List[Dict[str, Any]] = validator.get_metric( + column_types_dict_list: list[dict[str, Any]] = validator.get_metric( metric=MetricConfiguration( metric_name="table.column_types", metric_domain_kwargs={ @@ -135,7 +132,7 @@ def _build_table_column_name_to_inferred_semantic_domain_type_map( @staticmethod def _infer_semantic_domain_type_from_table_column_type( # noqa: C901 - column_types_dict_list: List[Dict[str, Any]], + column_types_dict_list: list[dict[str, Any]], column_name: str, ) -> InferredSemanticDomainType: # Note: As of Python 3.8, specifying argument type in Lambda functions is not supported by Lambda syntax. # noqa: E501 diff --git a/great_expectations/experimental/rule_based_profiler/helpers/util.py b/great_expectations/experimental/rule_based_profiler/helpers/util.py index 0b0a4e638f52..8ef96e938661 100644 --- a/great_expectations/experimental/rule_based_profiler/helpers/util.py +++ b/great_expectations/experimental/rule_based_profiler/helpers/util.py @@ -13,13 +13,10 @@ TYPE_CHECKING, Any, Callable, - Dict, Final, Iterable, - List, Optional, Protocol, - Tuple, Union, ) @@ -102,11 +99,11 @@ def get_validator( # noqa: PLR0913 purpose: str, *, data_context: Optional[AbstractDataContext] = None, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[str, BatchRequestBase, dict]] = None, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> Optional[Validator]: validator: Optional[Validator] @@ -155,13 +152,13 @@ def get_validator( # noqa: PLR0913 def get_batch_ids( # noqa: PLR0913 data_context: Optional[AbstractDataContext] = None, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[str, BatchRequestBase, dict]] = None, limit: Optional[int] = None, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, -) -> Optional[List[str]]: + parameters: Optional[dict[str, ParameterContainer]] = None, +) -> Optional[list[str]]: batch: Batch if batch_list is None or all(batch is None for batch in batch_list): if batch_request is None: @@ -176,7 +173,7 @@ def get_batch_ids( # noqa: PLR0913 batch_list = [data_context.get_last_batch(batch_request=batch_request)] - batch_ids: List[str] = [batch.id for batch in batch_list] + batch_ids: list[str] = [batch.id for batch in batch_list] num_batch_ids: int = len(batch_ids) @@ -203,7 +200,7 @@ def build_batch_request( batch_request: Optional[Union[str, BatchRequestBase, dict]] = None, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> Optional[Union[BatchRequest, RuntimeBatchRequest]]: if batch_request is None: return None @@ -230,7 +227,7 @@ def build_metric_domain_kwargs( metric_domain_kwargs: Optional[Union[str, dict]] = None, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ): # Obtain domain kwargs from "rule state" (i.e., variables and parameters); from instance variable otherwise. # noqa: E501 metric_domain_kwargs = get_parameter_value_and_validate_return_type( @@ -256,7 +253,7 @@ def get_parameter_value_and_validate_return_type( parameter_reference: Optional[Union[Any, str]] = None, expected_return_type: Optional[Union[type, tuple]] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> Any: """ This method allows for the parameter_reference to be specified as an object (literal, dict, any typed object, etc.) @@ -287,7 +284,7 @@ def get_parameter_value( domain: Optional[Domain] = None, parameter_reference: Optional[Union[Any, str]] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> Optional[Any]: """ This method allows for the parameter_reference to be specified as an object (literal, dict, any typed object, etc.) @@ -339,9 +336,9 @@ def get_parameter_value( def get_resolved_metrics_by_key( validator: Validator, - metric_configurations_by_key: Dict[str, List[MetricConfiguration]], + metric_configurations_by_key: dict[str, list[MetricConfiguration]], runtime_configuration: Optional[dict] = None, -) -> Dict[str, Dict[Tuple[str, str, str], MetricValue]]: +) -> dict[str, dict[tuple[str, str, str], MetricValue]]: """ Compute (resolve) metrics for every column name supplied on input. @@ -360,7 +357,7 @@ def get_resolved_metrics_by_key( """ # noqa: E501 key: str metric_configuration: MetricConfiguration - metric_configurations_for_key: List[MetricConfiguration] + metric_configurations_for_key: list[MetricConfiguration] # Step 1: Gather "MetricConfiguration" objects corresponding to all possible key values/combinations. # noqa: E501 # and compute all metric values (resolve "MetricConfiguration" objects ) using a single method call. # noqa: E501 @@ -376,14 +373,14 @@ def get_resolved_metrics_by_key( ) # Step 2: Gather "MetricConfiguration" ID values for each key (one element per batch_id in every list). # noqa: E501 - metric_configuration_ids_by_key: Dict[str, List[Tuple[str, str, str]]] = { + metric_configuration_ids_by_key: dict[str, list[tuple[str, str, str]]] = { key: [metric_configuration.id for metric_configuration in metric_configurations_for_key] for key, metric_configurations_for_key in metric_configurations_by_key.items() } - metric_configuration_ids: List[Tuple[str, str, str]] + metric_configuration_ids: list[tuple[str, str, str]] # Step 3: Obtain flattened list of "MetricConfiguration" ID values across all key values/combinations. # noqa: E501 - metric_configuration_ids_all_keys: List[Tuple[str, str, str]] = list( + metric_configuration_ids_all_keys: list[tuple[str, str, str]] = list( itertools.chain( *[ metric_configuration_ids @@ -394,7 +391,7 @@ def get_resolved_metrics_by_key( # Step 4: Retain only those metric computation results that both, correspond to "MetricConfiguration" objects of # noqa: E501 # interest (reflecting specified key values/combinations). - metric_configuration_id: Tuple[str, str, str] + metric_configuration_id: tuple[str, str, str] metric_value: Any resolved_metrics = { metric_configuration_id: metric_value @@ -403,12 +400,12 @@ def get_resolved_metrics_by_key( } # Step 5: Gather "MetricConfiguration" ID values for effective collection of resolved metrics. - metric_configuration_ids_resolved_metrics: List[Tuple[str, str, str]] = list( + metric_configuration_ids_resolved_metrics: list[tuple[str, str, str]] = list( resolved_metrics.keys() ) # Step 6: Produce "key" list, corresponding to effective "MetricConfiguration" ID values. - candidate_keys: List[str] = [ + candidate_keys: list[str] = [ key for key, metric_configuration_ids in metric_configuration_ids_by_key.items() if all( @@ -417,7 +414,7 @@ def get_resolved_metrics_by_key( ) ] - resolved_metrics_by_key: Dict[str, Dict[Tuple[str, str, str], MetricValue]] = { + resolved_metrics_by_key: dict[str, dict[tuple[str, str, str], MetricValue]] = { key: { metric_configuration.id: resolved_metrics[metric_configuration.id] for metric_configuration in metric_configurations_by_key[key] @@ -430,12 +427,12 @@ def get_resolved_metrics_by_key( def build_domains_from_column_names( rule_name: str, - column_names: List[str], + column_names: list[str], domain_type: MetricDomainTypes, table_column_name_to_inferred_semantic_domain_type_map: Optional[ - Dict[str, SemanticDomainTypes] + dict[str, SemanticDomainTypes] ] = None, -) -> List[Domain]: +) -> list[Domain]: """ This utility method builds "simple" Domain objects (i.e., required fields only, no "details" metadata accepted). @@ -446,7 +443,7 @@ def build_domains_from_column_names( :return: list of resulting Domain objects """ # noqa: E501 column_name: str - domains: List[Domain] = [ + domains: list[Domain] = [ Domain( domain_type=domain_type, domain_kwargs={ @@ -471,8 +468,8 @@ def build_domains_from_column_names( def convert_variables_to_dict( variables: Optional[ParameterContainer] = None, -) -> Dict[str, Any]: - variables_as_dict: Optional[Union[ParameterNode, Dict[str, Any]]] = ( +) -> dict[str, Any]: + variables_as_dict: Optional[Union[ParameterNode, dict[str, Any]]] = ( get_parameter_value_and_validate_return_type( domain=None, parameter_reference=VARIABLES_PREFIX, @@ -508,7 +505,7 @@ def integer_semantic_domain_type(domain: Domain) -> bool: """ # noqa: E501 - inferred_semantic_domain_type: Dict[str, SemanticDomainTypes] = domain.details.get( + inferred_semantic_domain_type: dict[str, SemanticDomainTypes] = domain.details.get( INFERRED_SEMANTIC_TYPE_KEY ) @@ -537,7 +534,7 @@ def datetime_semantic_domain_type(domain: Domain) -> bool: Boolean value indicating whether or not specified "Domain" is inferred as "SemanticDomainTypes.DATETIME" """ # noqa: E501 - inferred_semantic_domain_type: Dict[str, SemanticDomainTypes] = domain.details.get( + inferred_semantic_domain_type: dict[str, SemanticDomainTypes] = domain.details.get( INFERRED_SEMANTIC_TYPE_KEY ) @@ -552,7 +549,7 @@ def get_false_positive_rate_from_rule_state( false_positive_rate: Union[str, float], domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> Union[float, np.float64]: """ This method obtains false_positive_rate from "rule state" (i.e., variables and parameters) and validates the result. @@ -597,7 +594,7 @@ def get_quantile_statistic_interpolation_method_from_rule_state( round_decimals: int, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> str: """ This method obtains quantile_statistic_interpolation_method from "rule state" (i.e., variables and parameters) and @@ -859,7 +856,7 @@ def build_numeric_range_estimation_result( metric_values_converted, ) = convert_metric_values_to_float_dtype_best_effort(metric_values=metric_values) - histogram: Tuple[np.ndarray, np.ndarray] + histogram: tuple[np.ndarray, np.ndarray] bin_edges: np.ndarray if ndarray_is_datetime_type: histogram = np.histogram(a=metric_values_converted, bins=NUM_HISTOGRAM_BINS) @@ -926,7 +923,7 @@ def _determine_quantile_bias_corrected_point_estimate( # noqa: PLR0913 def convert_metric_values_to_float_dtype_best_effort( metric_values: np.ndarray, -) -> Tuple[bool, np.ndarray]: +) -> tuple[bool, np.ndarray]: """ Makes best effort attempt to discern element type of 1-D "np.ndarray" and convert it to "float" "np.ndarray" type. @@ -963,7 +960,7 @@ def convert_metric_values_to_float_dtype_best_effort( def get_validator_with_expectation_suite( # noqa: PLR0913 data_context: AbstractDataContext, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, expectation_suite: Optional[ExpectationSuite] = None, expectation_suite_name: Optional[str] = None, diff --git a/great_expectations/experimental/rule_based_profiler/metric_computation_result.py b/great_expectations/experimental/rule_based_profiler/metric_computation_result.py index 7af22c76eeb5..8aa5aba83617 100644 --- a/great_expectations/experimental/rule_based_profiler/metric_computation_result.py +++ b/great_expectations/experimental/rule_based_profiler/metric_computation_result.py @@ -1,12 +1,12 @@ from __future__ import annotations from dataclasses import make_dataclass -from typing import Any, Dict, List, Set, Tuple, Union +from typing import Any, Union from great_expectations.validator.computed_metric import MetricValue -MetricValues = Union[MetricValue, List[MetricValue], Set[MetricValue], Tuple[MetricValue, ...]] -MetricComputationDetails = Dict[str, Any] +MetricValues = Union[MetricValue, list[MetricValue], set[MetricValue], tuple[MetricValue, ...]] +MetricComputationDetails = dict[str, Any] MetricComputationResult = make_dataclass( "MetricComputationResult", ["attributed_resolved_metrics", "details"] ) diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/histogram_single_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/histogram_single_batch_parameter_builder.py index a15b348891c1..bbf0191b463e 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/histogram_single_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/histogram_single_batch_parameter_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Set +from typing import TYPE_CHECKING, ClassVar, Optional import numpy as np @@ -39,7 +39,7 @@ class HistogramSingleBatchParameterBuilder(MetricSingleBatchParameterBuilder): Compute histogram using specified metric for one Batch of data. """ - exclude_field_names: ClassVar[Set[str]] = ( + exclude_field_names: ClassVar[set[str]] = ( MetricSingleBatchParameterBuilder.exclude_field_names | { "column_partition_metric_single_batch_parameter_builder_config", @@ -58,7 +58,7 @@ def __init__( # noqa: PLR0913 bins: str = "uniform", n_bins: int = 10, allow_relative_error: bool = False, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -117,7 +117,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/mean_table_columns_set_match_multi_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/mean_table_columns_set_match_multi_batch_parameter_builder.py index e261fd396f8a..e0ffdc3b425d 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/mean_table_columns_set_match_multi_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/mean_table_columns_set_match_multi_batch_parameter_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, ClassVar, Optional, Union import numpy as np @@ -43,7 +43,7 @@ class MeanTableColumnsSetMatchMultiBatchParameterBuilder(MetricMultiBatchParamet Step-4: Compute mean value of match scores as "success_ratio" (divide sum of scores by number of Batch objects). """ # noqa: E501 - exclude_field_names: ClassVar[Set[str]] = ( + exclude_field_names: ClassVar[set[str]] = ( MetricMultiBatchParameterBuilder.exclude_field_names | { "metric_name", @@ -59,7 +59,7 @@ def __init__( name: str, metric_domain_kwargs: Optional[Union[str, dict]] = None, metric_value_kwargs: Optional[Union[str, dict]] = None, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -91,7 +91,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ @@ -122,16 +122,16 @@ def _build_parameters( ] one_batch_table_columns_names_value: MetricValue - multi_batch_table_columns_names_sets_as_list: List[Set[str]] = [ + multi_batch_table_columns_names_sets_as_list: list[set[str]] = [ set(one_batch_table_columns_names_value) # type: ignore[arg-type] # could be dict for one_batch_table_columns_names_value in table_columns_names_multi_batch_value # type: ignore[union-attr] # not all iterable ] - multi_batch_table_columns_names_as_set: Set[str] = set().union( + multi_batch_table_columns_names_as_set: set[str] = set().union( *multi_batch_table_columns_names_sets_as_list ) - one_batch_table_columns_names_set: Set[str] + one_batch_table_columns_names_set: set[str] mean_table_columns_set_match: np.float64 = np.mean( np.asarray( [ diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/mean_unexpected_map_metric_multi_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/mean_unexpected_map_metric_multi_batch_parameter_builder.py index 3d01b12bf0ac..e1a5737372bf 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/mean_unexpected_map_metric_multi_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/mean_unexpected_map_metric_multi_batch_parameter_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, ClassVar, Optional, Union import numpy as np @@ -41,7 +41,7 @@ class MeanUnexpectedMapMetricMultiBatchParameterBuilder(MetricMultiBatchParamete Compute mean unexpected count ratio (as a fraction) of specified map-style metric across every Batch of data given. """ # noqa: E501 - exclude_field_names: ClassVar[Set[str]] = ( + exclude_field_names: ClassVar[set[str]] = ( MetricMultiBatchParameterBuilder.exclude_field_names | { "metric_name", @@ -60,7 +60,7 @@ def __init__( # noqa: PLR0913 null_count_parameter_builder_name: Optional[str] = None, metric_domain_kwargs: Optional[Union[str, dict]] = None, metric_value_kwargs: Optional[Union[str, dict]] = None, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -111,7 +111,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ @@ -155,7 +155,7 @@ def _build_parameters( ) ) - batch_ids: Optional[List[str]] = self.get_batch_ids( + batch_ids: Optional[list[str]] = self.get_batch_ids( domain=domain, variables=variables, parameters=parameters, diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/metric_multi_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/metric_multi_batch_parameter_builder.py index 199696cbfc9e..8f6d403c4e87 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/metric_multi_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/metric_multi_batch_parameter_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union import numpy as np @@ -46,7 +46,7 @@ def __init__( # noqa: PLR0913 enforce_numeric_metric: Union[str, bool] = False, replace_nan_with_zero: Union[str, bool] = False, reduce_scalar_metric: Union[str, bool] = True, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -133,7 +133,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/metric_single_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/metric_single_batch_parameter_builder.py index c8df3df656a8..2e96ea065caf 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/metric_single_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/metric_single_batch_parameter_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, ClassVar, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.domain import Domain # noqa: TCH001 @@ -33,7 +33,7 @@ class MetricSingleBatchParameterBuilder(MetricMultiBatchParameterBuilder): and metric_name as arguments. """ # noqa: E501 - exclude_field_names: ClassVar[Set[str]] = ( + exclude_field_names: ClassVar[set[str]] = ( MetricMultiBatchParameterBuilder.exclude_field_names | { "single_batch_mode", @@ -49,7 +49,7 @@ def __init__( # noqa: PLR0913 enforce_numeric_metric: Union[str, bool] = False, replace_nan_with_zero: Union[str, bool] = False, reduce_scalar_metric: Union[str, bool] = True, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -87,7 +87,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py index a28d777c58fb..bf4b87972398 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py @@ -10,10 +10,7 @@ Any, Callable, ClassVar, - Dict, - List, Optional, - Set, Union, ) @@ -116,7 +113,7 @@ class NumericMetricRangeMultiBatchParameterBuilder(MetricMultiBatchParameterBuil "column.unique_proportion", } - exclude_field_names: ClassVar[Set[str]] = ( + exclude_field_names: ClassVar[set[str]] = ( MetricMultiBatchParameterBuilder.exclude_field_names | { "single_batch_mode", @@ -143,10 +140,10 @@ def __init__( # noqa: PLR0913 bw_method: Optional[Union[str, float, Callable]] = None, include_estimator_samples_histogram_in_details: Union[str, bool] = False, truncate_values: Optional[ - Union[str, Dict[str, Union[Optional[int], Optional[float]]]] + Union[str, dict[str, Union[Optional[int], Optional[float]]]] ] = None, round_decimals: Optional[Union[str, int]] = None, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -297,7 +294,7 @@ def include_estimator_samples_histogram_in_details(self) -> Union[str, bool]: @property def truncate_values( self, - ) -> Optional[Union[str, Dict[str, Union[Optional[int], Optional[float]]]]]: + ) -> Optional[Union[str, dict[str, Union[Optional[int], Optional[float]]]]]: return self._truncate_values @property @@ -308,7 +305,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ @@ -397,7 +394,7 @@ def _build_parameters( ) value_range: np.ndarray = numeric_range_estimation_result.value_range - details: Dict[str, Any] = copy.deepcopy( + details: dict[str, Any] = copy.deepcopy( parameter_node[FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY] ) @@ -427,7 +424,7 @@ def _build_numeric_range_estimator( round_decimals: int, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> NumericRangeEstimator: """ Determines "estimator" name and returns appropriate configured "NumericRangeEstimator" subclass instance. @@ -517,7 +514,7 @@ def _estimate_metric_value_range( # noqa: C901, PLR0912, PLR0913, PLR0915 round_decimals: int, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> NumericRangeEstimationResult: """ This method accepts "NumericRangeEstimator" and data samples in format "N x R^m", where "N" (most significant @@ -526,7 +523,7 @@ def _estimate_metric_value_range( # noqa: C901, PLR0912, PLR0913, PLR0915 vector of sample measurements is constructed and given to the estimator to apply its specific algorithm for computing the range of values in this vector. Estimator algorithms differ based on their use of data samples. """ # noqa: E501 - truncate_values: Dict[str, Number] = self._get_truncate_values_using_heuristics( + truncate_values: dict[str, Number] = self._get_truncate_values_using_heuristics( metric_values=metric_values, domain=domain, variables=variables, @@ -543,16 +540,16 @@ def _estimate_metric_value_range( # noqa: C901, PLR0912, PLR0913, PLR0915 # Generate all permutations of indexes for accessing every element of the multi-dimensional metric. # noqa: E501 metric_value_shape_idx: int - axes: List[np.ndarray] = [ + axes: list[np.ndarray] = [ np.indices(dimensions=(metric_value_shape_idx,))[0] for metric_value_shape_idx in metric_value_shape ] - metric_value_indices: List[tuple] = list(itertools.product(*tuple(axes))) + metric_value_indices: list[tuple] = list(itertools.product(*tuple(axes))) # Generate all permutations of indexes for accessing estimates of every element of the multi-dimensional metric. # noqa: E501 # Prefixing multi-dimensional index with "(slice(None, None, None),)" is equivalent to "[:,]" access. # noqa: E501 metric_value_idx: tuple - metric_value_vector_indices: List[tuple] = [ + metric_value_vector_indices: list[tuple] = [ (slice(None, None, None),) + metric_value_idx for metric_value_idx in metric_value_indices ] @@ -681,7 +678,7 @@ def _estimate_metric_value_range( # noqa: C901, PLR0912, PLR0913, PLR0915 @staticmethod def _is_metric_values_ndarray_datetime_dtype( metric_values: np.ndarray, - metric_value_vector_indices: List[tuple], + metric_value_vector_indices: list[tuple], ) -> bool: metric_value_vector: np.ndarray for metric_value_idx in metric_value_vector_indices: @@ -698,7 +695,7 @@ def _is_metric_values_ndarray_datetime_dtype( @staticmethod def _is_metric_values_ndarray_decimal_dtype( metric_values: np.ndarray, - metric_value_vector_indices: List[tuple], + metric_value_vector_indices: list[tuple], ) -> bool: metric_value_vector: np.ndarray for metric_value_idx in metric_value_vector_indices: @@ -714,10 +711,10 @@ def _get_truncate_values_using_heuristics( domain: Domain, *, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, - ) -> Dict[str, Union[Optional[int], Optional[float]]]: + parameters: Optional[dict[str, ParameterContainer]] = None, + ) -> dict[str, Union[Optional[int], Optional[float]]]: # Obtain truncate_values directive from "rule state" (i.e., variables and parameters); from instance variable otherwise. # noqa: E501 - truncate_values: Dict[str, Optional[Number]] = get_parameter_value_and_validate_return_type( + truncate_values: dict[str, Optional[Number]] = get_parameter_value_and_validate_return_type( domain=domain, parameter_reference=self.truncate_values, expected_return_type=dict, @@ -760,7 +757,7 @@ def _get_round_decimals_using_heuristics( metric_values: np.ndarray, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> int: # Obtain round_decimals directive from "rule state" (i.e., variables and parameters); from instance variable otherwise. # noqa: E501 round_decimals: Optional[int] = get_parameter_value_and_validate_return_type( diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/parameter_builder.py index 8eb5553b14dc..2d68a0e0ab36 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/parameter_builder.py @@ -12,11 +12,7 @@ Any, Callable, ClassVar, - Dict, - List, Optional, - Set, - Tuple, Union, ) @@ -92,14 +88,14 @@ class ParameterBuilder(ABC, Builder): ``` """ # noqa: E501 - exclude_field_names: ClassVar[Set[str]] = Builder.exclude_field_names | { + exclude_field_names: ClassVar[set[str]] = Builder.exclude_field_names | { "suite_parameter_builders", } def __init__( self, name: str, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -129,9 +125,9 @@ def build_parameters( # noqa: PLR0913 self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, parameter_computation_impl: Optional[Callable] = None, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, runtime_configuration: Optional[dict] = None, ) -> None: @@ -147,7 +143,7 @@ def build_parameters( # noqa: PLR0913 """ # noqa: E501 runtime_configuration = runtime_configuration or {} - fully_qualified_parameter_names: List[str] = get_fully_qualified_parameter_names( + fully_qualified_parameter_names: list[str] = get_fully_qualified_parameter_names( domain=domain, variables=variables, parameters=parameters, @@ -187,7 +183,7 @@ def build_parameters( # noqa: PLR0913 runtime_configuration=runtime_configuration, ) - parameter_values: Dict[str, Any] = { + parameter_values: dict[str, Any] = { self.raw_fully_qualified_parameter_name: parameter_computation_result, self.json_serialized_fully_qualified_parameter_name: convert_to_json_serializable( data=parameter_computation_result @@ -203,8 +199,8 @@ def resolve_evaluation_dependencies( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, - fully_qualified_parameter_names: Optional[List[str]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, + fully_qualified_parameter_names: Optional[list[str]] = None, runtime_configuration: Optional[dict] = None, ) -> None: """ @@ -212,7 +208,7 @@ def resolve_evaluation_dependencies( "ParameterBuilder" objects), whose output(s) are needed by specified "ParameterBuilder" object to operate. """ # noqa: E501 # Step-1: Check if any "suite_parameter_builders" are configured for specified "ParameterBuilder" object. # noqa: E501 - suite_parameter_builders: List[ParameterBuilder] = self.suite_parameter_builders + suite_parameter_builders: list[ParameterBuilder] = self.suite_parameter_builders if not suite_parameter_builders: return @@ -255,7 +251,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ @@ -273,13 +269,13 @@ def name(self) -> str: @property def suite_parameter_builders( self, - ) -> Optional[List[ParameterBuilder]]: + ) -> Optional[list[ParameterBuilder]]: return self._suite_parameter_builders @property def suite_parameter_builder_configs( self, - ) -> Optional[List[ParameterBuilderConfig]]: + ) -> Optional[list[ParameterBuilderConfig]]: return self._suite_parameter_builder_configs @property @@ -300,7 +296,7 @@ def get_validator( self, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> Optional[Validator]: return get_validator_using_batch_list_or_batch_request( purpose="parameter_builder", @@ -317,8 +313,8 @@ def get_batch_ids( limit: Optional[int] = None, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, - ) -> Optional[List[str]]: + parameters: Optional[dict[str, ParameterContainer]] = None, + ) -> Optional[list[str]]: return get_batch_ids_from_batch_list_or_batch_request( data_context=self.data_context, batch_list=self.batch_list, @@ -332,15 +328,15 @@ def get_batch_ids( def get_metrics( # noqa: C901, PLR0913 self, metric_name: str, - metric_domain_kwargs: Optional[Union[Union[str, dict], List[Union[str, dict]]]] = None, - metric_value_kwargs: Optional[Union[Union[str, dict], List[Union[str, dict]]]] = None, + metric_domain_kwargs: Optional[Union[Union[str, dict], list[Union[str, dict]]]] = None, + metric_value_kwargs: Optional[Union[Union[str, dict], list[Union[str, dict]]]] = None, limit: Optional[int] = None, enforce_numeric_metric: Union[str, bool] = False, replace_nan_with_zero: Union[str, bool] = False, runtime_configuration: Optional[dict] = None, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> MetricComputationResult: """ General multi-batch metric computation facility. @@ -367,7 +363,7 @@ def get_metrics( # noqa: C901, PLR0913 specified (empty "metric_name" value detected).""" # noqa: E501 ) - batch_ids: Optional[List[str]] = self.get_batch_ids( + batch_ids: Optional[list[str]] = self.get_batch_ids( limit=limit, domain=domain, variables=variables, @@ -435,15 +431,15 @@ def get_metrics( # noqa: C901, PLR0913 # Step-3: Generate "MetricConfiguration" directives for all "metric_domain_kwargs"/"metric_value_kwargs" pairs. # noqa: E501 domain_kwargs_cursor: dict - kwargs_combinations: List[List[dict]] = [ + kwargs_combinations: list[list[dict]] = [ [domain_kwargs_cursor, value_kwargs_cursor] for value_kwargs_cursor in metric_value_kwargs for domain_kwargs_cursor in metric_domain_kwargs ] - metrics_to_resolve: List[MetricConfiguration] + metrics_to_resolve: list[MetricConfiguration] - kwargs_pair_cursor: List[dict, dict] + kwargs_pair_cursor: list[dict, dict] metrics_to_resolve = [ MetricConfiguration( metric_name=metric_name, @@ -467,10 +463,10 @@ def get_metrics( # noqa: C901, PLR0913 runtime_configuration=runtime_configuration, ) - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] - aborted_metrics_info: Dict[ - Tuple[str, str, str], - Dict[str, Union[MetricConfiguration, Set[ExceptionInfo], int]], + resolved_metrics: dict[tuple[str, str, str], MetricValue] + aborted_metrics_info: dict[ + tuple[str, str, str], + dict[str, Union[MetricConfiguration, set[ExceptionInfo], int]], ] ( resolved_metrics, @@ -483,7 +479,7 @@ def get_metrics( # noqa: C901, PLR0913 # Step-5: Map resolved metrics to their attributes for identification and recovery by receiver. # noqa: E501 - attributed_resolved_metrics_map: Dict[str, AttributedResolvedMetrics] = {} + attributed_resolved_metrics_map: dict[str, AttributedResolvedMetrics] = {} resolved_metric_value: MetricValue attributed_resolved_metrics: AttributedResolvedMetrics @@ -577,7 +573,7 @@ def _sanitize_metric_computation( # noqa: PLR0913 replace_nan_with_zero: Union[str, bool] = False, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> None: """ This method conditions (or "sanitizes") data samples in the format "N x R^m", where "N" (most significant @@ -608,10 +604,10 @@ def _sanitize_metric_computation( # noqa: PLR0913 if not (enforce_numeric_metric or replace_nan_with_zero): return - metric_values_by_batch_id: Dict[str, MetricValue] = {} + metric_values_by_batch_id: dict[str, MetricValue] = {} # noinspection PyTypeChecker - conditioned_attributed_metric_values: Dict[str, MetricValues] = dict( + conditioned_attributed_metric_values: dict[str, MetricValues] = dict( filter( lambda element: element[1] is not None, attributed_resolved_metrics.conditioned_attributed_metric_values.items(), @@ -629,11 +625,11 @@ def _sanitize_metric_computation( # noqa: PLR0913 # Generate all permutations of indexes for accessing every element of the multi-dimensional metric. # noqa: E501 metric_value_shape_idx: int - axes: List[np.ndarray] = [ + axes: list[np.ndarray] = [ np.indices(dimensions=(metric_value_shape_idx,))[0] for metric_value_shape_idx in metric_value_shape ] - metric_value_indices: List[tuple] = list(itertools.product(*tuple(axes))) + metric_value_indices: list[tuple] = list(itertools.product(*tuple(axes))) metric_value_idx: tuple for metric_value_idx in metric_value_indices: @@ -670,9 +666,9 @@ def _sanitize_metric_computation( # noqa: PLR0913 @staticmethod def _get_best_candidate_above_threshold( - candidate_ratio_dict: Dict[str, float], + candidate_ratio_dict: dict[str, float], threshold: float, - ) -> Tuple[Optional[str], float]: + ) -> tuple[Optional[str], float]: """ Helper method to calculate which candidate strings or patterns are the best match (ie. highest ratio), provided they are also above the threshold. @@ -691,8 +687,8 @@ def _get_best_candidate_above_threshold( @staticmethod def _get_sorted_candidates_and_ratios( - candidate_ratio_dict: Dict[str, float], - ) -> Dict[str, float]: + candidate_ratio_dict: dict[str, float], + ) -> dict[str, float]: """ Helper method to sort all candidate strings or patterns by success ratio (how well they matched the domain). @@ -709,9 +705,9 @@ def _get_sorted_candidates_and_ratios( def init_rule_parameter_builders( - parameter_builder_configs: Optional[List[dict]] = None, + parameter_builder_configs: Optional[list[dict]] = None, data_context: Optional[AbstractDataContext] = None, -) -> Optional[List[ParameterBuilder]]: +) -> Optional[list[ParameterBuilder]]: if parameter_builder_configs is None: return None diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/regex_pattern_string_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/regex_pattern_string_parameter_builder.py index 6ff2d7c7a793..157c49e9a908 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/regex_pattern_string_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/regex_pattern_string_parameter_builder.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Union +from typing import TYPE_CHECKING, Iterable, Optional, Union import pandas as pd @@ -51,7 +51,7 @@ class RegexPatternStringParameterBuilder(ParameterBuilder): # list of candidate strings that are most commonly used # source: https://regexland.com/most-common-regular-expressions/ # source for UUID: https://stackoverflow.com/questions/7905929/how-to-test-valid-uuid-guid/13653180#13653180 - CANDIDATE_REGEX: Set[str] = { + CANDIDATE_REGEX: set[str] = { r"\d+", # whole number with 1 or more digits r"-?\d+", # negative whole numbers r"-?\d+(?:\.\d*)?", # decimal numbers with . (period) separator @@ -71,7 +71,7 @@ def __init__( # noqa: PLR0913 metric_value_kwargs: Optional[Union[str, dict]] = None, threshold: Union[str, float] = 1.0, candidate_regexes: Optional[Union[str, Iterable[str]]] = None, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -119,14 +119,14 @@ def threshold(self) -> Union[str, float]: @property def candidate_regexes( self, - ) -> Union[str, Union[List[str], Set[str]]]: + ) -> Union[str, Union[list[str], set[str]]]: return self._candidate_regexes def _build_parameters( # noqa: C901 self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ @@ -182,8 +182,8 @@ def _build_parameters( # noqa: C901 # Obtain candidate_regexes from "rule state" (i.e, variables and parameters); from instance variable otherwise. # noqa: E501 candidate_regexes: Union[ - List[str], - Set[str], + list[str], + set[str], ] = get_parameter_value_and_validate_return_type( domain=domain, parameter_reference=self.candidate_regexes, @@ -198,7 +198,7 @@ def _build_parameters( # noqa: C901 # Gather "metric_value_kwargs" for all candidate "regex" strings. regex_string: str - match_regex_metric_value_kwargs_list: List[dict] = [] + match_regex_metric_value_kwargs_list: list[dict] = [] match_regex_metric_value_kwargs: dict for regex_string in candidate_regexes: if self.metric_value_kwargs: diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/simple_date_format_string_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/simple_date_format_string_parameter_builder.py index 9b3f52be5764..bb578d9c3758 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/simple_date_format_string_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/simple_date_format_string_parameter_builder.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Union +from typing import TYPE_CHECKING, Iterable, Optional, Union from great_expectations.core.domain import Domain # noqa: TCH001 from great_expectations.core.metric_function_types import ( @@ -38,7 +38,7 @@ logger = logging.getLogger(__name__) -DEFAULT_CANDIDATE_STRINGS: Set[str] = { +DEFAULT_CANDIDATE_STRINGS: set[str] = { "%H:%M:%S", "%H:%M:%S,%f", "%H:%M:%S.%f", @@ -115,7 +115,7 @@ def __init__( # noqa: PLR0913 metric_value_kwargs: Optional[Union[str, dict]] = None, threshold: Union[str, float] = 1.0, candidate_strings: Optional[Union[Iterable[str], str]] = None, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -168,14 +168,14 @@ def threshold(self) -> Union[str, float]: @property def candidate_strings( self, - ) -> Union[str, Union[List[str], Set[str]]]: + ) -> Union[str, Union[list[str], set[str]]]: return self._candidate_strings def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ @@ -228,8 +228,8 @@ def _build_parameters( # Obtain candidate_strings from "rule state" (i.e., variables and parameters); from instance variable otherwise. # noqa: E501 candidate_strings: Union[ - List[str], - Set[str], + list[str], + set[str], ] = get_parameter_value_and_validate_return_type( domain=domain, parameter_reference=self.candidate_strings, @@ -240,7 +240,7 @@ def _build_parameters( # Gather "metric_value_kwargs" for all candidate "strftime_format" strings. format_string: str - match_strftime_metric_value_kwargs_list: List[dict] = [] + match_strftime_metric_value_kwargs_list: list[dict] = [] match_strftime_metric_value_kwargs: dict for format_string in candidate_strings: if self.metric_value_kwargs: diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/unexpected_count_statistics_multi_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/unexpected_count_statistics_multi_batch_parameter_builder.py index 3bc4993448d6..9c2572e5dca3 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/unexpected_count_statistics_multi_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/unexpected_count_statistics_multi_batch_parameter_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union import numpy as np import scipy @@ -55,7 +55,7 @@ def __init__( # noqa: PLR0913 mode: str, max_error_rate: Optional[Union[str, float]] = None, expectation_type: Optional[str] = None, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -115,7 +115,7 @@ def _build_parameters( # PLR0915, PLR0915 self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ @@ -204,7 +204,7 @@ def _build_parameters( # PLR0915, PLR0915 """ # noqa: E501 ) - result: Union[np.float64, Dict[str, Union[np.float64, np.ndarray]]] + result: Union[np.float64, dict[str, Union[np.float64, np.ndarray]]] if mode == "unexpected_count_fraction_values": result = unexpected_count_fraction_values diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/value_counts_single_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/value_counts_single_batch_parameter_builder.py index 6ea01713a3d2..00bc1faeb727 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/value_counts_single_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/value_counts_single_batch_parameter_builder.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Set +from typing import TYPE_CHECKING, ClassVar, Optional import numpy as np @@ -35,7 +35,7 @@ class ValueCountsSingleBatchParameterBuilder(MetricSingleBatchParameterBuilder): Compute value counts using specified metric for one Batch of data. """ - exclude_field_names: ClassVar[Set[str]] = ( + exclude_field_names: ClassVar[set[str]] = ( MetricSingleBatchParameterBuilder.exclude_field_names | { "column_value_counts_metric_single_batch_parameter_builder_config", @@ -52,7 +52,7 @@ class ValueCountsSingleBatchParameterBuilder(MetricSingleBatchParameterBuilder): def __init__( self, name: str, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -119,7 +119,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ diff --git a/great_expectations/experimental/rule_based_profiler/parameter_builder/value_set_multi_batch_parameter_builder.py b/great_expectations/experimental/rule_based_profiler/parameter_builder/value_set_multi_batch_parameter_builder.py index dac38c220f79..658d9b5d8a62 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_builder/value_set_multi_batch_parameter_builder.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_builder/value_set_multi_batch_parameter_builder.py @@ -5,10 +5,7 @@ TYPE_CHECKING, ClassVar, Collection, - Dict, - List, Optional, - Set, TypeVar, Union, ) @@ -68,7 +65,7 @@ class ValueSetMultiBatchParameterBuilder(MetricMultiBatchParameterBuilder): 2. This ParameterBuilder filters null values out from the unique value_set. """ - exclude_field_names: ClassVar[Set[str]] = ( + exclude_field_names: ClassVar[set[str]] = ( MetricMultiBatchParameterBuilder.exclude_field_names | { "metric_name", @@ -84,7 +81,7 @@ def __init__( name: str, metric_domain_kwargs: Optional[Union[str, dict]] = None, metric_value_kwargs: Optional[Union[str, dict]] = None, - suite_parameter_builder_configs: Optional[List[ParameterBuilderConfig]] = None, + suite_parameter_builder_configs: Optional[list[ParameterBuilderConfig]] = None, data_context: Optional[AbstractDataContext] = None, ) -> None: """ @@ -116,7 +113,7 @@ def _build_parameters( self, domain: Domain, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, runtime_configuration: Optional[dict] = None, ) -> Attributes: """ @@ -179,8 +176,8 @@ def _build_parameters( def _get_unique_values_from_nested_collection_of_sets( - collection: Collection[Collection[Set[V]]], -) -> Set[V]: + collection: Collection[Collection[set[V]]], +) -> set[V]: """Get unique values from a collection of sets e.g. a list of sets. Args: @@ -191,7 +188,7 @@ def _get_unique_values_from_nested_collection_of_sets( Single flattened set containing unique values. """ - flattened: Union[List[Set[V]], Set[V]] = list(itertools.chain.from_iterable(collection)) + flattened: Union[list[set[V]], set[V]] = list(itertools.chain.from_iterable(collection)) element: V if all(isinstance(element, set) for element in flattened): flattened = set().union(*flattened) @@ -201,7 +198,7 @@ def _get_unique_values_from_nested_collection_of_sets( reliance on "np.ndarray", "None" gets converted to "numpy.Inf", whereas "numpy.Inf == numpy.Inf" returns False, resulting in numerous "None" elements in final set. For this reason, all "None" elements must be filtered out. """ # noqa: E501 - unique_values: Set[V] = set( + unique_values: set[V] = set( sorted( # type: ignore[type-var,arg-type] # lambda destroys type info? filter( lambda element: not ( diff --git a/great_expectations/experimental/rule_based_profiler/parameter_container.py b/great_expectations/experimental/rule_based_profiler/parameter_container.py index 3aa8b280ef9d..1d3ebe2114eb 100644 --- a/great_expectations/experimental/rule_based_profiler/parameter_container.py +++ b/great_expectations/experimental/rule_based_profiler/parameter_container.py @@ -2,7 +2,7 @@ import copy from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Set, TypeVar, Union +from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar, Union from pyparsing import ( Literal, @@ -59,7 +59,7 @@ FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY: Final[str] = "details" FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY: Final[str] = "value" -RESERVED_TERMINAL_LITERALS: Final[Set[str]] = { +RESERVED_TERMINAL_LITERALS: Final[set[str]] = { FULLY_QUALIFIED_PARAMETER_NAME_ATTRIBUTED_VALUE_KEY, FULLY_QUALIFIED_PARAMETER_NAME_METADATA_KEY, FULLY_QUALIFIED_PARAMETER_NAME_VALUE_KEY, @@ -223,7 +223,7 @@ class ParameterContainer(SerializableDictDot): within the same "name space" serves as the dictionary key, and the root-level ParameterNode objects are the values). """ # noqa: E501 - parameter_nodes: Optional[Dict[str, ParameterNode]] = None + parameter_nodes: Optional[dict[str, ParameterNode]] = None def set_parameter_node(self, parameter_name_root: str, parameter_node: ParameterNode) -> None: if self.parameter_nodes is None: @@ -305,7 +305,7 @@ def convert_parameter_node_to_dictionary( def build_parameter_container_for_variables( - variables_configs: Dict[str, Any], + variables_configs: dict[str, Any], ) -> ParameterContainer: """ Build a ParameterContainer for all of the profiler config variables passed as key value pairs @@ -319,7 +319,7 @@ def build_parameter_container_for_variables( key: str value: Any for key, value in variables_configs.items(): - key_parts: List[str] = [PARAMETER_NAME_ROOT_FOR_VARIABLES] + key.split( + key_parts: list[str] = [PARAMETER_NAME_ROOT_FOR_VARIABLES] + key.split( FULLY_QUALIFIED_PARAMETER_NAME_SEPARATOR_CHARACTER ) _build_parameter_node_tree_for_one_parameter(parameter_node, key_parts, value) @@ -335,7 +335,7 @@ def build_parameter_container_for_variables( def build_parameter_container( parameter_container: ParameterContainer, - parameter_values: Dict[str, Any], + parameter_values: dict[str, Any], ) -> None: """ Builds the ParameterNode trees, corresponding to the fully_qualified_parameter_name first-level keys. @@ -368,7 +368,7 @@ def build_parameter_container( parameter_node: Optional[ParameterNode] fully_qualified_parameter_name: str parameter_value: Any - fully_qualified_parameter_name_as_list: List[str] + fully_qualified_parameter_name_as_list: list[str] parameter_name_root: str for ( fully_qualified_parameter_name, @@ -399,7 +399,7 @@ def build_parameter_container( def _build_parameter_node_tree_for_one_parameter( parameter_node: ParameterNode, - parameter_name_as_list: List[str], + parameter_name_as_list: list[str], parameter_value: Any, ) -> None: """ @@ -437,7 +437,7 @@ def get_parameter_value_by_fully_qualified_parameter_name( fully_qualified_parameter_name: str, domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, ) -> Optional[Union[Any, ParameterNode]]: """ Get the parameter value from the current "rule state" using the fully-qualified parameter name. @@ -488,7 +488,7 @@ def get_parameter_value_by_fully_qualified_parameter_name( fully_qualified_parameter_name = fully_qualified_parameter_name[1:] - fully_qualified_parameter_name_as_list: List[str] = fully_qualified_parameter_name.split( + fully_qualified_parameter_name_as_list: list[str] = fully_qualified_parameter_name.split( FULLY_QUALIFIED_PARAMETER_NAME_SEPARATOR_CHARACTER ) @@ -504,7 +504,7 @@ def get_parameter_value_by_fully_qualified_parameter_name( def _get_parameter_value_from_parameter_container( fully_qualified_parameter_name: str, - fully_qualified_parameter_name_as_list: List[str], + fully_qualified_parameter_name_as_list: list[str], parameter_container: ParameterContainer, ) -> Optional[Union[Any, ParameterNode]]: if parameter_container is None: @@ -562,8 +562,8 @@ def _get_parameter_value_from_parameter_container( def get_parameter_values_for_fully_qualified_parameter_names( domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, -) -> Dict[str, Any]: + parameters: Optional[dict[str, ParameterContainer]] = None, +) -> dict[str, Any]: fully_qualified_parameter_name: str return { fully_qualified_parameter_name: get_parameter_value_by_fully_qualified_parameter_name( @@ -583,9 +583,9 @@ def get_parameter_values_for_fully_qualified_parameter_names( def get_fully_qualified_parameter_names( domain: Optional[Domain] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, -) -> List[str]: - fully_qualified_parameter_names: List[str] = [] + parameters: Optional[dict[str, ParameterContainer]] = None, +) -> list[str]: + fully_qualified_parameter_names: list[str] = [] if not (variables is None or variables.parameter_nodes is None): fully_qualified_parameter_names.extend( _get_parameter_node_attribute_names( @@ -617,10 +617,10 @@ def get_fully_qualified_parameter_names( def _get_parameter_node_attribute_names( parameter_name_root: Optional[str] = None, parameter_node: Optional[ParameterNode] = None, -) -> List[str]: - attribute_names_as_lists: List[List[str]] = [] +) -> list[str]: + attribute_names_as_lists: list[list[str]] = [] - parameter_name_root_as_list: Optional[List[str]] = None + parameter_name_root_as_list: Optional[list[str]] = None if parameter_name_root: parameter_name_root_as_list = [parameter_name_root] @@ -630,7 +630,7 @@ def _get_parameter_node_attribute_names( parameter_node=parameter_node, ) - attribute_names: Set[str] = set() + attribute_names: set[str] = set() attribute_name: str for attribute_name_as_list in attribute_names_as_lists: @@ -646,14 +646,14 @@ def _get_parameter_node_attribute_names( def _get_parameter_node_attribute_names_as_lists( - attribute_names_as_lists: List[List[str]], - parameter_name_root_as_list: Optional[List[str]] = None, + attribute_names_as_lists: list[list[str]], + parameter_name_root_as_list: Optional[list[str]] = None, parameter_node: Optional[ParameterNode] = None, ) -> None: if parameter_node is None or parameter_name_root_as_list is None: return - partial_parameter_name_root_as_list: List[str] + partial_parameter_name_root_as_list: list[str] attribute_name_part: str attribute_value_part: Any @@ -671,8 +671,8 @@ def _get_parameter_node_attribute_names_as_lists( def _get_parameter_name_parts_up_to_including_reserved_literal( - attribute_name_as_list: List[str], -) -> List[str]: + attribute_name_as_list: list[str], +) -> list[str]: if attribute_name_as_list[0] == PARAMETER_NAME_ROOT_FOR_VARIABLES: return [ PARAMETER_NAME_ROOT_FOR_VARIABLES, @@ -683,7 +683,7 @@ def _get_parameter_name_parts_up_to_including_reserved_literal( return attribute_name_as_list # TODO: 12/29/2022: Lexicographical order avoids collisions between regular keys and reserved literals. # noqa: E501 - reserved_terminal_literals: List[str] = list(sorted(RESERVED_TERMINAL_LITERALS)) + reserved_terminal_literals: list[str] = list(sorted(RESERVED_TERMINAL_LITERALS)) idx: Optional[int] = None key: str diff --git a/great_expectations/experimental/rule_based_profiler/rule/rule.py b/great_expectations/experimental/rule_based_profiler/rule/rule.py index dcb8434228c7..69dc18c4fa3e 100644 --- a/great_expectations/experimental/rule_based_profiler/rule/rule.py +++ b/great_expectations/experimental/rule_based_profiler/rule/rule.py @@ -2,7 +2,7 @@ import copy import json -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.util import ( @@ -51,10 +51,10 @@ class Rule(SerializableDictDot): def __init__( self, name: str, - variables: Optional[Union[ParameterContainer, Dict[str, Any]]] = None, + variables: Optional[Union[ParameterContainer, dict[str, Any]]] = None, domain_builder: Optional[DomainBuilder] = None, - parameter_builders: Optional[List[ParameterBuilder]] = None, - expectation_configuration_builders: Optional[List[ExpectationConfigurationBuilder]] = None, + parameter_builders: Optional[list[ParameterBuilder]] = None, + expectation_configuration_builders: Optional[list[ExpectationConfigurationBuilder]] = None, ) -> None: """ Sets Rule name, variables, domain builder, parameters builders, configuration builders, and other instance data. @@ -92,7 +92,7 @@ def __init__( def run( # noqa: PLR0913 self, variables: Optional[ParameterContainer] = None, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, runtime_configuration: Optional[dict] = None, reconciliation_directives: Optional[ReconciliationDirectives] = None, @@ -127,7 +127,7 @@ def run( # noqa: PLR0913 if rule_state is None: rule_state = RuleState() - domains: List[Domain] = self._get_rule_domains( + domains: list[Domain] = self._get_rule_domains( variables=variables, batch_list=batch_list, batch_request=batch_request, @@ -153,7 +153,7 @@ def run( # noqa: PLR0913 ): rule_state.initialize_parameter_container_for_domain(domain=domain) - parameter_builders: List[ParameterBuilder] = self.parameter_builders or [] + parameter_builders: list[ParameterBuilder] = self.parameter_builders or [] parameter_builder: ParameterBuilder for parameter_builder in parameter_builders: parameter_builder.build_parameters( @@ -166,7 +166,7 @@ def run( # noqa: PLR0913 runtime_configuration=runtime_configuration, ) - expectation_configuration_builders: List[ExpectationConfigurationBuilder] = ( + expectation_configuration_builders: list[ExpectationConfigurationBuilder] = ( self.expectation_configuration_builders or [] ) @@ -206,19 +206,19 @@ def domain_builder(self) -> Optional[DomainBuilder]: return self._domain_builder @property - def parameter_builders(self) -> Optional[List[ParameterBuilder]]: + def parameter_builders(self) -> Optional[list[ParameterBuilder]]: return self._parameter_builders @property def expectation_configuration_builders( self, - ) -> Optional[List[ExpectationConfigurationBuilder]]: + ) -> Optional[list[ExpectationConfigurationBuilder]]: return self._expectation_configuration_builders @override def to_dict(self) -> dict: - parameter_builder_configs: Optional[List[dict]] = None - parameter_builders: Optional[Dict[str, ParameterBuilder]] = ( + parameter_builder_configs: Optional[list[dict]] = None + parameter_builders: Optional[dict[str, ParameterBuilder]] = ( self._get_parameter_builders_as_dict() ) parameter_builder: ParameterBuilder @@ -229,8 +229,8 @@ def to_dict(self) -> dict: for parameter_builder in parameter_builders.values() ] - expectation_configuration_builder_configs: Optional[List[dict]] = None - expectation_configuration_builders: Optional[Dict[str, ExpectationConfigurationBuilder]] = ( + expectation_configuration_builder_configs: Optional[list[dict]] = None + expectation_configuration_builders: Optional[dict[str, ExpectationConfigurationBuilder]] = ( self._get_expectation_configuration_builders_as_dict() ) expectation_configuration_builder: ExpectationConfigurationBuilder @@ -262,7 +262,7 @@ def to_json_dict(self) -> dict: make this refactoring infeasible at the present time. """ # noqa: E501 dict_obj: dict = self.to_dict() - variables_dict: Optional[Dict[str, Any]] = convert_variables_to_dict( + variables_dict: Optional[dict[str, Any]] = convert_variables_to_dict( variables=self.variables ) dict_obj["variables"] = variables_dict @@ -296,8 +296,8 @@ def __str__(self) -> str: """ # noqa: E501 return self.__repr__() - def _get_parameter_builders_as_dict(self) -> Dict[str, ParameterBuilder]: - parameter_builders: List[ParameterBuilder] = self.parameter_builders or [] + def _get_parameter_builders_as_dict(self) -> dict[str, ParameterBuilder]: + parameter_builders: list[ParameterBuilder] = self.parameter_builders or [] parameter_builder: ParameterBuilder return { @@ -306,8 +306,8 @@ def _get_parameter_builders_as_dict(self) -> Dict[str, ParameterBuilder]: def _get_expectation_configuration_builders_as_dict( self, - ) -> Dict[str, ExpectationConfigurationBuilder]: - expectation_configuration_builders: List[ExpectationConfigurationBuilder] = ( + ) -> dict[str, ExpectationConfigurationBuilder]: + expectation_configuration_builders: list[ExpectationConfigurationBuilder] = ( self.expectation_configuration_builders or [] ) @@ -326,12 +326,12 @@ def _get_expectation_configuration_builders_as_dict( def _get_rule_domains( self, variables: Optional[ParameterContainer] = None, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, rule_state: Optional[RuleState] = None, runtime_configuration: Optional[dict] = None, - ) -> List[Domain]: - domains: List[Domain] = ( + ) -> list[Domain]: + domains: list[Domain] = ( [] if self.domain_builder is None else self.domain_builder.get_domains( diff --git a/great_expectations/experimental/rule_based_profiler/rule/rule_output.py b/great_expectations/experimental/rule_based_profiler/rule/rule_output.py index 439e66f36664..227f837ffa0a 100644 --- a/great_expectations/experimental/rule_based_profiler/rule/rule_output.py +++ b/great_expectations/experimental/rule_based_profiler/rule/rule_output.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional from great_expectations.experimental.rule_based_profiler.expectation_configuration_builder import ( ExpectationConfigurationBuilder, # noqa: TCH001 @@ -40,15 +40,15 @@ def __init__( def rule_state(self) -> RuleState: return self._rule_state - def get_expectation_configurations(self) -> List[ExpectationConfiguration]: - expectation_configurations: List[ExpectationConfiguration] = [] + def get_expectation_configurations(self) -> list[ExpectationConfiguration]: + expectation_configurations: list[ExpectationConfiguration] = [] - domains: List[Domain] = self.rule_state.domains + domains: list[Domain] = self.rule_state.domains domain: Domain expectation_configuration_builder: ExpectationConfigurationBuilder for domain in domains: - expectation_configuration_builders: List[ExpectationConfigurationBuilder] + expectation_configuration_builders: list[ExpectationConfigurationBuilder] if self.rule_state.rule and self.rule_state.rule.expectation_configuration_builders: expectation_configuration_builders = ( self.rule_state.rule.expectation_configuration_builders @@ -67,17 +67,17 @@ def get_expectation_configurations(self) -> List[ExpectationConfiguration]: return expectation_configurations - def get_fully_qualified_parameter_names_by_domain(self) -> Dict[Domain, List[str]]: + def get_fully_qualified_parameter_names_by_domain(self) -> dict[Domain, list[str]]: domain: Domain return { domain: self.get_fully_qualified_parameter_names_for_domain(domain=domain) for domain in self.rule_state.domains } - def get_fully_qualified_parameter_names_for_domain_id(self, domain_id: str) -> List[str]: - domains_dict: Dict[str, Domain] = self.rule_state.get_domains_as_dict() + def get_fully_qualified_parameter_names_for_domain_id(self, domain_id: str) -> list[str]: + domains_dict: dict[str, Domain] = self.rule_state.get_domains_as_dict() domain: Domain = domains_dict[domain_id] - fully_qualified_parameter_names: List[str] = ( + fully_qualified_parameter_names: list[str] = ( self.get_fully_qualified_parameter_names_for_domain(domain=domain) ) return fully_qualified_parameter_names @@ -85,8 +85,8 @@ def get_fully_qualified_parameter_names_for_domain_id(self, domain_id: str) -> L def get_fully_qualified_parameter_names_for_domain( self, domain: Optional[Domain] = None, - ) -> List[str]: - fully_qualified_parameter_names: List[str] = get_fully_qualified_parameter_names( + ) -> list[str]: + fully_qualified_parameter_names: list[str] = get_fully_qualified_parameter_names( domain=domain, variables=self.rule_state.variables, parameters=self.rule_state.parameters, @@ -95,7 +95,7 @@ def get_fully_qualified_parameter_names_for_domain( def get_parameter_values_for_fully_qualified_parameter_names_by_domain( self, - ) -> Dict[Domain, Dict[str, ParameterNode]]: + ) -> dict[Domain, dict[str, ParameterNode]]: domain: Domain return { domain: self.get_parameter_values_for_fully_qualified_parameter_names_for_domain( @@ -106,10 +106,10 @@ def get_parameter_values_for_fully_qualified_parameter_names_by_domain( def get_parameter_values_for_fully_qualified_parameter_names_for_domain_id( self, domain_id: str - ) -> Dict[str, ParameterNode]: - domains_dict: Dict[str, Domain] = self.rule_state.get_domains_as_dict() + ) -> dict[str, ParameterNode]: + domains_dict: dict[str, Domain] = self.rule_state.get_domains_as_dict() domain: Domain = domains_dict[domain_id] - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] = ( + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] = ( self.get_parameter_values_for_fully_qualified_parameter_names_for_domain(domain=domain) ) return parameter_values_for_fully_qualified_parameter_names @@ -117,8 +117,8 @@ def get_parameter_values_for_fully_qualified_parameter_names_for_domain_id( def get_parameter_values_for_fully_qualified_parameter_names_for_domain( self, domain: Optional[Domain] = None, - ) -> Dict[str, ParameterNode]: - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] = ( + ) -> dict[str, ParameterNode]: + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] = ( get_parameter_values_for_fully_qualified_parameter_names( domain=domain, variables=self.rule_state.variables, diff --git a/great_expectations/experimental/rule_based_profiler/rule/rule_state.py b/great_expectations/experimental/rule_based_profiler/rule/rule_state.py index e7a12800463b..1120f5e2ea4f 100644 --- a/great_expectations/experimental/rule_based_profiler/rule/rule_state.py +++ b/great_expectations/experimental/rule_based_profiler/rule/rule_state.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional from great_expectations.core.domain import Domain # noqa: TCH001 from great_expectations.experimental.rule_based_profiler.exceptions import ( @@ -26,9 +26,9 @@ class RuleState: def __init__( self, rule: Optional[Rule] = None, - domains: Optional[List[Domain]] = None, + domains: Optional[list[Domain]] = None, variables: Optional[ParameterContainer] = None, - parameters: Optional[Dict[str, ParameterContainer]] = None, + parameters: Optional[dict[str, ParameterContainer]] = None, catch_exceptions: bool = False, ) -> None: """ @@ -67,11 +67,11 @@ def rule(self, value: Rule) -> None: self._rule = value @property - def domains(self) -> List[Domain]: + def domains(self) -> list[Domain]: return self._domains @domains.setter - def domains(self, value: List[Domain]) -> None: + def domains(self, value: list[Domain]) -> None: self._domains = value @property @@ -83,11 +83,11 @@ def variables(self, value: Optional[ParameterContainer]) -> None: self._variables = value @property - def parameters(self) -> Dict[str, ParameterContainer]: + def parameters(self) -> dict[str, ParameterContainer]: return self._parameters @parameters.setter - def parameters(self, value: Dict[str, ParameterContainer]) -> None: + def parameters(self, value: dict[str, ParameterContainer]) -> None: self._parameters = value @property @@ -159,7 +159,7 @@ def remove_domain_if_exists(self, domain: Domain) -> None: self.domains.remove(domain) self.remove_domain_if_exists(domain=domain) - def get_domains_as_dict(self) -> Dict[str, Domain]: + def get_domains_as_dict(self) -> dict[str, Domain]: domain: Domain return {domain.id: domain for domain in self.domains} diff --git a/great_expectations/experimental/rule_based_profiler/rule_based_profiler.py b/great_expectations/experimental/rule_based_profiler/rule_based_profiler.py index 6441f81b7abc..c6151a314786 100644 --- a/great_expectations/experimental/rule_based_profiler/rule_based_profiler.py +++ b/great_expectations/experimental/rule_based_profiler/rule_based_profiler.py @@ -6,7 +6,7 @@ import logging import sys import traceback -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union import great_expectations.exceptions as gx_exceptions from great_expectations.core.batch import ( @@ -98,7 +98,7 @@ class BaseRuleBasedProfiler(ConfigPeer): in the form of interface methods (which can be overwritten by subclasses) and their reference implementation. """ # noqa: E501 - EXPECTATION_SUCCESS_KEYS: Set[str] = { + EXPECTATION_SUCCESS_KEYS: set[str] = { "_auto", "_profiler_config", } @@ -129,8 +129,8 @@ def __init__( if hasattr(profiler_config, "id"): id = profiler_config.id config_version: float = profiler_config.config_version - variables: Optional[Dict[str, Any]] = profiler_config.variables - rules: Optional[Dict[str, Dict[str, Any]]] = profiler_config.rules + variables: Optional[dict[str, Any]] = profiler_config.variables + rules: Optional[dict[str, dict[str, Any]]] = profiler_config.rules self._name = name self._id = id @@ -164,15 +164,15 @@ def id(self, id: str) -> None: def _init_profiler_rules( self, - rules: Dict[str, Dict[str, Any]], - ) -> List[Rule]: + rules: dict[str, dict[str, Any]], + ) -> list[Rule]: if rules is None: rules = {} - rule_object_list: List[Rule] = [] + rule_object_list: list[Rule] = [] rule_name: str - rule_config: Dict[str, Any] + rule_config: dict[str, Any] for rule_name, rule_config in rules.items(): rule_object_list.append(self._init_rule(rule_name=rule_name, rule_config=rule_config)) @@ -181,7 +181,7 @@ def _init_profiler_rules( def _init_rule( self, rule_name: str, - rule_config: Dict[str, Any], + rule_config: dict[str, Any], ) -> Rule: # Config is validated through schema but do a sanity check attr: str @@ -195,16 +195,16 @@ def _init_rule( ) # Instantiate variables and builder attributes - variables: Dict[str, Any] = rule_config.get("variables", {}) + variables: dict[str, Any] = rule_config.get("variables", {}) domain_builder: DomainBuilder = RuleBasedProfiler._init_rule_domain_builder( domain_builder_config=rule_config.get("domain_builder"), data_context=self._data_context, ) - parameter_builders: Optional[List[ParameterBuilder]] = init_rule_parameter_builders( + parameter_builders: Optional[list[ParameterBuilder]] = init_rule_parameter_builders( parameter_builder_configs=rule_config.get("parameter_builders"), data_context=self._data_context, ) - expectation_configuration_builders: List[ExpectationConfigurationBuilder] = ( + expectation_configuration_builders: list[ExpectationConfigurationBuilder] = ( init_rule_expectation_configuration_builders( expectation_configuration_builder_configs=rule_config.get( "expectation_configuration_builders" @@ -239,14 +239,14 @@ def _init_rule_domain_builder( def run( # noqa: PLR0913 self, - variables: Optional[Dict[str, Any]] = None, - rules: Optional[Dict[str, Dict[str, Any]]] = None, - batch_list: Optional[List[Batch]] = None, + variables: Optional[dict[str, Any]] = None, + rules: Optional[dict[str, dict[str, Any]]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, runtime_configuration: Optional[dict] = None, reconciliation_directives: ReconciliationDirectives = DEFAULT_RECONCILATION_DIRECTIVES, - variables_directives_list: Optional[List[RuntimeEnvironmentVariablesDirectives]] = None, - domain_type_directives_list: Optional[List[RuntimeEnvironmentDomainTypeDirectives]] = None, + variables_directives_list: Optional[list[RuntimeEnvironmentVariablesDirectives]] = None, + domain_type_directives_list: Optional[list[RuntimeEnvironmentDomainTypeDirectives]] = None, comment: Optional[str] = None, ) -> RuleBasedProfilerResult: """Run the Rule-Based Profiler. @@ -282,7 +282,7 @@ def run( # noqa: PLR0913 reconciliation_strategy=reconciliation_directives.variables, ) - effective_rules: List[Rule] = self.reconcile_profiler_rules( + effective_rules: list[Rule] = self.reconcile_profiler_rules( rules=rules, reconciliation_directives=reconciliation_directives, ) @@ -295,7 +295,7 @@ def run( # noqa: PLR0913 ) rule: Rule - effective_rules_configs: Optional[Dict[str, Dict[str, Any]]] = { + effective_rules_configs: Optional[dict[str, dict[str, Any]]] = { rule.name: rule.to_json_dict() for rule in effective_rules } @@ -371,12 +371,12 @@ def run( # noqa: PLR0913 }, ) - def get_expectation_configurations(self) -> List[ExpectationConfiguration]: + def get_expectation_configurations(self) -> list[ExpectationConfiguration]: """ Returns: List of ExpectationConfiguration objects, accumulated from RuleState of every Rule executed. """ # noqa: E501 - expectation_configurations: List[ExpectationConfiguration] = [] + expectation_configurations: list[ExpectationConfiguration] = [] rule_state: RuleState rule_output: RuleOutput @@ -386,12 +386,12 @@ def get_expectation_configurations(self) -> List[ExpectationConfiguration]: return expectation_configurations - def get_fully_qualified_parameter_names_by_domain(self) -> Dict[Domain, List[str]]: + def get_fully_qualified_parameter_names_by_domain(self) -> dict[Domain, list[str]]: """ Returns: Dictionary of fully-qualified parameter names by Domain, accumulated from RuleState of every Rule executed. """ # noqa: E501 - fully_qualified_parameter_names_by_domain: Dict[Domain, List[str]] = {} + fully_qualified_parameter_names_by_domain: dict[Domain, list[str]] = {} rule_state: RuleState rule_output: RuleOutput @@ -403,7 +403,7 @@ def get_fully_qualified_parameter_names_by_domain(self) -> Dict[Domain, List[str return fully_qualified_parameter_names_by_domain - def get_fully_qualified_parameter_names_for_domain_id(self, domain_id: str) -> List[str]: + def get_fully_qualified_parameter_names_for_domain_id(self, domain_id: str) -> list[str]: """ Args: domain_id: ID of desired Domain object. @@ -422,13 +422,13 @@ def get_fully_qualified_parameter_names_for_domain_id(self, domain_id: str) -> L def get_parameter_values_for_fully_qualified_parameter_names_by_domain( self, - ) -> Dict[Domain, Dict[str, ParameterNode]]: + ) -> dict[Domain, dict[str, ParameterNode]]: """ Returns: Dictionaries of values for fully-qualified parameter names by Domain, accumulated from RuleState of every Rule executed. """ # noqa: E501 - values_for_fully_qualified_parameter_names_by_domain: Dict[ - Domain, Dict[str, ParameterNode] + values_for_fully_qualified_parameter_names_by_domain: dict[ + Domain, dict[str, ParameterNode] ] = {} rule_state: RuleState @@ -443,7 +443,7 @@ def get_parameter_values_for_fully_qualified_parameter_names_by_domain( def get_parameter_values_for_fully_qualified_parameter_names_for_domain_id( self, domain_id: str - ) -> Dict[str, ParameterNode]: + ) -> dict[str, ParameterNode]: """ Args: domain_id: ID of desired Domain object. @@ -464,10 +464,10 @@ def add_rule(self, rule: Rule) -> None: """ Add Rule object to existing profiler object by reconciling profiler rules and updating _profiler_config. """ # noqa: E501 - rules_dict: Dict[str, Dict[str, Any]] = { + rules_dict: dict[str, dict[str, Any]] = { rule.name: rule.to_json_dict(), } - effective_rules: List[Rule] = self.reconcile_profiler_rules( + effective_rules: list[Rule] = self.reconcile_profiler_rules( rules=rules_dict, reconciliation_directives=ReconciliationDirectives( domain_builder=ReconciliationStrategy.UPDATE, @@ -476,14 +476,14 @@ def add_rule(self, rule: Rule) -> None: ), ) self.rules = effective_rules - updated_rules: Optional[Dict[str, Dict[str, Any]]] = { + updated_rules: Optional[dict[str, dict[str, Any]]] = { rule.name: rule.to_json_dict() for rule in effective_rules } self._profiler_config.rules = updated_rules def reconcile_profiler_variables( self, - variables: Optional[Dict[str, Any]] = None, + variables: Optional[dict[str, Any]] = None, reconciliation_strategy: ReconciliationStrategy = DEFAULT_RECONCILATION_DIRECTIVES.variables, # noqa: E501 ) -> Optional[ParameterContainer]: """ @@ -512,13 +512,13 @@ def reconcile_profiler_variables( def _reconcile_profiler_variables_as_dict( self, - variables: Optional[Dict[str, Any]], + variables: Optional[dict[str, Any]], reconciliation_strategy: ReconciliationStrategy = DEFAULT_RECONCILATION_DIRECTIVES.variables, # noqa: E501 ) -> dict: if variables is None: variables = {} - variables_configs: Optional[Dict[str, Any]] = convert_variables_to_dict( + variables_configs: Optional[dict[str, Any]] = convert_variables_to_dict( variables=self.variables ) @@ -536,9 +536,9 @@ def _reconcile_profiler_variables_as_dict( def reconcile_profiler_rules( self, - rules: Optional[Dict[str, Dict[str, Any]]] = None, + rules: Optional[dict[str, dict[str, Any]]] = None, reconciliation_directives: ReconciliationDirectives = DEFAULT_RECONCILATION_DIRECTIVES, - ) -> List[Rule]: + ) -> list[Rule]: """ Profiler "rules" reconciliation involves combining the rules, instantiated from Profiler configuration (e.g., stored in a YAML file managed by the Profiler store), with the rules overrides, provided at run time. @@ -551,7 +551,7 @@ def reconcile_profiler_rules( :param reconciliation_directives directives for how each rule component should be overwritten :return: reconciled rules in their canonical List[Rule] object form """ # noqa: E501 - effective_rules: Dict[str, Rule] = self._reconcile_profiler_rules_as_dict( + effective_rules: dict[str, Rule] = self._reconcile_profiler_rules_as_dict( rules=rules, reconciliation_directives=reconciliation_directives, ) @@ -559,18 +559,18 @@ def reconcile_profiler_rules( def _reconcile_profiler_rules_as_dict( self, - rules: Optional[Dict[str, Dict[str, Any]]] = None, + rules: Optional[dict[str, dict[str, Any]]] = None, reconciliation_directives: ReconciliationDirectives = DEFAULT_RECONCILATION_DIRECTIVES, - ) -> Dict[str, Rule]: + ) -> dict[str, Rule]: if rules is None: rules = {} - effective_rules: Dict[str, Rule] = self._get_rules_as_dict() + effective_rules: dict[str, Rule] = self._get_rules_as_dict() rule_name: str rule_config: dict - override_rule_configs: Dict[str, Dict[str, Any]] = { + override_rule_configs: dict[str, dict[str, Any]] = { rule_name: RuleBasedProfiler._reconcile_rule_config( existing_rules=effective_rules, rule_name=rule_name, @@ -579,7 +579,7 @@ def _reconcile_profiler_rules_as_dict( ) for rule_name, rule_config in rules.items() } - override_rules: Dict[str, Rule] = { + override_rules: dict[str, Rule] = { rule_name: self._init_rule(rule_name=rule_name, rule_config=rule_config) for rule_name, rule_config in override_rule_configs.items() } @@ -589,11 +589,11 @@ def _reconcile_profiler_rules_as_dict( @classmethod def _reconcile_rule_config( cls, - existing_rules: Dict[str, Rule], + existing_rules: dict[str, Rule], rule_name: str, rule_config: dict, reconciliation_directives: ReconciliationDirectives = DEFAULT_RECONCILATION_DIRECTIVES, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ A "rule configuration" reconciliation is the process of combining the configuration of a single candidate override rule with at most one configuration corresponding to the list of rules instantiated from Profiler @@ -622,7 +622,7 @@ def _reconcile_rule_config( :param reconciliation_directives directives for how each rule component should be overwritten :return: reconciled rule configuration, returned in dictionary (configuration) form """ # noqa: E501 - effective_rule_config: Dict[str, Any] + effective_rule_config: dict[str, Any] if rule_name in existing_rules: rule: Rule = existing_rules[rule_name] @@ -640,8 +640,8 @@ def _reconcile_rule_config( reconciliation_strategy=reconciliation_directives.domain_builder, ) - parameter_builder_configs: List[dict] = rule_config.get("parameter_builders", []) - effective_parameter_builder_configs: Optional[List[dict]] = ( + parameter_builder_configs: list[dict] = rule_config.get("parameter_builders", []) + effective_parameter_builder_configs: Optional[list[dict]] = ( cls._reconcile_rule_parameter_builder_configs( rule=rule, parameter_builder_configs=parameter_builder_configs, @@ -649,10 +649,10 @@ def _reconcile_rule_config( ) ) - expectation_configuration_builder_configs: List[dict] = rule_config.get( + expectation_configuration_builder_configs: list[dict] = rule_config.get( "expectation_configuration_builders", [] ) - effective_expectation_configuration_builder_configs: List[dict] = ( + effective_expectation_configuration_builder_configs: list[dict] = ( cls._reconcile_rule_expectation_configuration_builder_configs( rule=rule, expectation_configuration_builder_configs=expectation_configuration_builder_configs, @@ -719,9 +719,9 @@ def _reconcile_rule_domain_builder_config( @staticmethod def _reconcile_rule_parameter_builder_configs( rule: Rule, - parameter_builder_configs: List[dict], + parameter_builder_configs: list[dict], reconciliation_strategy: ReconciliationStrategy = DEFAULT_RECONCILATION_DIRECTIVES.parameter_builder, # noqa: E501 - ) -> Optional[List[dict]]: + ) -> Optional[list[dict]]: """ Rule "parameter builders" reconciliation involves combining the parameter builders, instantiated from Rule configuration (e.g., stored in a YAML file managed by the Profiler store), with the parameter builders @@ -741,9 +741,9 @@ def _reconcile_rule_parameter_builder_configs( for parameter_builder_config in parameter_builder_configs: _validate_builder_override_config(builder_config=parameter_builder_config) - effective_parameter_builder_configs: Dict[str, dict] = {} + effective_parameter_builder_configs: dict[str, dict] = {} - current_parameter_builders: Dict[str, ParameterBuilder] = ( + current_parameter_builders: dict[str, ParameterBuilder] = ( rule._get_parameter_builders_as_dict() ) @@ -766,7 +766,7 @@ def _reconcile_rule_parameter_builder_configs( effective_parameter_builder_configs[parameter_builder_name] = serialized_config - parameter_builder_configs_override: Dict[str, dict] = { + parameter_builder_configs_override: dict[str, dict] = { parameter_builder_config["name"]: parameter_builder_config for parameter_builder_config in parameter_builder_configs } @@ -789,9 +789,9 @@ def _reconcile_rule_parameter_builder_configs( @staticmethod def _reconcile_rule_expectation_configuration_builder_configs( rule: Rule, - expectation_configuration_builder_configs: List[dict], + expectation_configuration_builder_configs: list[dict], reconciliation_strategy: ReconciliationStrategy = DEFAULT_RECONCILATION_DIRECTIVES.expectation_configuration_builder, # noqa: E501 - ) -> List[dict]: + ) -> list[dict]: """ Rule "expectation configuration builders" reconciliation involves combining the expectation configuration builders, instantiated from Rule configuration (e.g., stored in a YAML file managed by the Profiler store), with the expectation configuration builders @@ -813,9 +813,9 @@ def _reconcile_rule_expectation_configuration_builder_configs( builder_config=expectation_configuration_builder_config ) - effective_expectation_configuration_builder_configs: Dict[str, dict] = {} + effective_expectation_configuration_builder_configs: dict[str, dict] = {} - current_expectation_configuration_builders: Dict[str, ExpectationConfigurationBuilder] = ( + current_expectation_configuration_builders: dict[str, ExpectationConfigurationBuilder] = ( rule._get_expectation_configuration_builders_as_dict() ) @@ -848,7 +848,7 @@ def _reconcile_rule_expectation_configuration_builder_configs( expectation_configuration_builder_name ] = serialized_config - expectation_configuration_builder_configs_override: Dict[str, dict] = { + expectation_configuration_builder_configs_override: dict[str, dict] = { expectation_configuration_builder_config[ "expectation_type" ]: expectation_configuration_builder_config @@ -874,7 +874,7 @@ def _reconcile_rule_expectation_configuration_builder_configs( return list(effective_expectation_configuration_builder_configs.values()) - def _get_rules_as_dict(self) -> Dict[str, Rule]: + def _get_rules_as_dict(self) -> dict[str, Rule]: rule: Rule return {rule.name: rule for rule in self.rules} @@ -882,9 +882,9 @@ def _get_rules_as_dict(self) -> Dict[str, Rule]: def _apply_runtime_environment( self, variables: Optional[ParameterContainer] = None, - rules: Optional[List[Rule]] = None, - variables_directives_list: Optional[List[RuntimeEnvironmentVariablesDirectives]] = None, - domain_type_directives_list: Optional[List[RuntimeEnvironmentDomainTypeDirectives]] = None, + rules: Optional[list[Rule]] = None, + variables_directives_list: Optional[list[RuntimeEnvironmentVariablesDirectives]] = None, + domain_type_directives_list: Optional[list[RuntimeEnvironmentDomainTypeDirectives]] = None, ) -> None: """ variables: attribute name/value pairs, commonly-used in Builder objects, to modify using "runtime_environment" @@ -906,8 +906,8 @@ def _apply_runtime_environment( @staticmethod def _apply_variables_directives_runtime_environment( - rules: Optional[List[Rule]] = None, - variables_directives_list: Optional[List[RuntimeEnvironmentVariablesDirectives]] = None, + rules: Optional[list[Rule]] = None, + variables_directives_list: Optional[list[RuntimeEnvironmentVariablesDirectives]] = None, ) -> None: """ rules: name/(configuration-dictionary) to modify using "runtime_environment" @@ -918,7 +918,7 @@ def _apply_variables_directives_runtime_environment( rule: Rule - rule_names: List[str] = [rule.name for rule in rules] + rule_names: list[str] = [rule.name for rule in rules] if variables_directives_list is None: variables_directives_list = [] @@ -946,11 +946,11 @@ def _apply_variables_directives_runtime_environment( ) ) - rules_as_dict: Dict[str, Rule] = {rule.name: rule for rule in rules} + rules_as_dict: dict[str, Rule] = {rule.name: rule for rule in rules} # 4. Update "variables" of pertinent "Rule" objects, according to corresponding additional/override directives. # noqa: E501 - variables: Optional[Dict[str, Any]] - rule_variables_configs: Optional[Dict[str, Any]] + variables: Optional[dict[str, Any]] + rule_variables_configs: Optional[dict[str, Any]] for variables_directives in variables_directives_list: variables = variables_directives.variables or {} rule = rules_as_dict[variables_directives.rule_name] @@ -972,8 +972,8 @@ def _apply_variables_directives_runtime_environment( @staticmethod def _apply_domain_type_directives_runtime_environment( - rules: Optional[List[Rule]] = None, - domain_type_directives_list: Optional[List[RuntimeEnvironmentDomainTypeDirectives]] = None, + rules: Optional[list[Rule]] = None, + domain_type_directives_list: Optional[list[RuntimeEnvironmentDomainTypeDirectives]] = None, ) -> None: """ rules: name/(configuration-dictionary) to modify using "runtime_environment" @@ -986,7 +986,7 @@ def _apply_domain_type_directives_runtime_environment( domain_type_directives_list = [] domain_type_directives: RuntimeEnvironmentDomainTypeDirectives - domain_rules: List[Rule] + domain_rules: list[Rule] rule: Rule for domain_type_directives in domain_type_directives_list: # 1. Ensure that Domain directives pertain to "Rule" objects with "DomainBuilder" of correct "Domain" type. # noqa: E501 @@ -1047,7 +1047,7 @@ def run_profiler( # noqa: PLR0913 cls, data_context: AbstractDataContext, profiler_store: ProfilerStore, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, name: Optional[str] = None, id: Optional[str] = None, @@ -1078,7 +1078,7 @@ def run_profiler_on_data( # noqa: PLR0913 cls, data_context: AbstractDataContext, profiler_store: ProfilerStore, - batch_list: Optional[List[Batch]] = None, + batch_list: Optional[list[Batch]] = None, batch_request: Optional[Union[BatchRequestBase, dict]] = None, name: Optional[str] = None, id: Optional[str] = None, @@ -1091,7 +1091,7 @@ def run_profiler_on_data( # noqa: PLR0913 ) rule: Rule - rules: Dict[str, Dict[str, Any]] = { + rules: dict[str, dict[str, Any]] = { rule.name: rule.to_json_dict() for rule in profiler.rules } @@ -1281,14 +1281,14 @@ def _check_validity_of_batch_requests_in_config( config: RuleBasedProfilerConfig, ) -> bool: # Evaluate nested types in RuleConfig to parse out BatchRequests - batch_requests: List[Union[BatchRequestBase, dict]] = [] + batch_requests: list[Union[BatchRequestBase, dict]] = [] rule: dict for rule in config.rules.values(): domain_builder: dict = rule["domain_builder"] if "batch_request" in domain_builder: batch_requests.append(domain_builder["batch_request"]) - parameter_builders: List[dict] = rule.get("parameter_builders", []) + parameter_builders: list[dict] = rule.get("parameter_builders", []) parameter_builder: dict for parameter_builder in parameter_builders: if "batch_request" in parameter_builder: @@ -1361,7 +1361,7 @@ def delete_profiler( def list_profilers( profiler_store: ProfilerStore, ge_cloud_mode: bool = False, - ) -> List[str]: + ) -> list[str]: if ge_cloud_mode: return profiler_store.list_keys() return [x.configuration_key for x in profiler_store.list_keys()] @@ -1402,15 +1402,15 @@ def variables(self, value: Optional[ParameterContainer]) -> None: self.config.variables = convert_variables_to_dict(variables=value) @property - def rules(self) -> List[Rule]: + def rules(self) -> list[Rule]: return self._rules @rules.setter - def rules(self, value: List[Rule]) -> None: + def rules(self, value: list[Rule]) -> None: self._rules = value @property - def rule_states(self) -> List[RuleState]: + def rule_states(self) -> list[RuleState]: return self._rule_states def to_json_dict(self) -> dict: @@ -1419,7 +1419,7 @@ def to_json_dict(self) -> dict: Returns: A JSON-serializable dict representation of this RuleBasedProfiler. """ - variables_dict: Optional[Dict[str, Any]] = convert_variables_to_dict( + variables_dict: Optional[dict[str, Any]] = convert_variables_to_dict( variables=self.variables ) @@ -1532,8 +1532,8 @@ def __init__( # noqa: PLR0913 self, name: str, config_version: float, - variables: Optional[Dict[str, Any]] = None, - rules: Optional[Dict[str, Dict[str, Any]]] = None, + variables: Optional[dict[str, Any]] = None, + rules: Optional[dict[str, dict[str, Any]]] = None, data_context: Optional[AbstractDataContext] = None, id: Optional[str] = None, catch_exceptions: bool = False, diff --git a/great_expectations/experimental/rule_based_profiler/rule_based_profiler_result.py b/great_expectations/experimental/rule_based_profiler/rule_based_profiler_result.py index 65d1b6f300ac..9eba08c7c5da 100644 --- a/great_expectations/experimental/rule_based_profiler/rule_based_profiler_result.py +++ b/great_expectations/experimental/rule_based_profiler/rule_based_profiler_result.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.core import ( @@ -47,15 +47,15 @@ class RuleBasedProfilerResult(SerializableDictDot): """ # noqa: E501 - fully_qualified_parameter_names_by_domain: Dict[Domain, List[str]] + fully_qualified_parameter_names_by_domain: dict[Domain, list[str]] parameter_values_for_fully_qualified_parameter_names_by_domain: Optional[ - Dict[Domain, Dict[str, ParameterNode]] + dict[Domain, dict[str, ParameterNode]] ] - expectation_configurations: List[ExpectationConfiguration] + expectation_configurations: list[ExpectationConfiguration] citation: dict - rule_domain_builder_execution_time: Dict[str, float] - rule_execution_time: Dict[str, float] - rule_exception_tracebacks: Dict[str, Optional[str]] + rule_domain_builder_execution_time: dict[str, float] + rule_execution_time: dict[str, float] + rule_exception_tracebacks: dict[str, Optional[str]] catch_exceptions: bool = field(default=False) @override @@ -65,11 +65,11 @@ def to_dict(self) -> dict: This `RuleBasedProfilerResult` as dictionary (JSON-serializable for `RuleBasedProfilerResult` objects). """ # noqa: E501 domain: Domain - fully_qualified_parameter_names: List[str] - parameter_values_for_fully_qualified_parameter_names: Dict[str, ParameterNode] + fully_qualified_parameter_names: list[str] + parameter_values_for_fully_qualified_parameter_names: dict[str, ParameterNode] expectation_configuration: ExpectationConfiguration - parameter_values_for_fully_qualified_parameter_names_by_domain: Dict[ - Domain, Dict[str, ParameterNode] + parameter_values_for_fully_qualified_parameter_names_by_domain: dict[ + Domain, dict[str, ParameterNode] ] = self.parameter_values_for_fully_qualified_parameter_names_by_domain or {} return { "fully_qualified_parameter_names_by_domain": [ diff --git a/great_expectations/experimental/rule_based_profiler/semantic_type_filter.py b/great_expectations/experimental/rule_based_profiler/semantic_type_filter.py index 3b8e4d07d6ba..2fc2cea709f8 100644 --- a/great_expectations/experimental/rule_based_profiler/semantic_type_filter.py +++ b/great_expectations/experimental/rule_based_profiler/semantic_type_filter.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union if TYPE_CHECKING: from great_expectations.core.domain import SemanticDomainTypes @@ -12,14 +12,14 @@ class SemanticTypeFilter(ABC): def parse_semantic_domain_type_argument( self, semantic_types: Optional[ - Union[str, SemanticDomainTypes, List[Union[str, SemanticDomainTypes]]] + Union[str, SemanticDomainTypes, list[Union[str, SemanticDomainTypes]]] ] = None, - ) -> List[SemanticDomainTypes]: + ) -> list[SemanticDomainTypes]: pass @property @abstractmethod def table_column_name_to_inferred_semantic_domain_type_map( self, - ) -> Dict[str, SemanticDomainTypes]: + ) -> dict[str, SemanticDomainTypes]: pass diff --git a/great_expectations/render/components.py b/great_expectations/render/components.py index b000d58fb333..b2a4fd4beeb4 100644 --- a/great_expectations/render/components.py +++ b/great_expectations/render/components.py @@ -4,7 +4,7 @@ from copy import deepcopy from enum import Enum from string import Template as pTemplate -from typing import TYPE_CHECKING, Any, Final, List, Optional, Union +from typing import TYPE_CHECKING, Any, Final, Optional, Union from marshmallow import Schema, fields, post_dump, post_load @@ -750,8 +750,8 @@ def __init__( # noqa: PLR0913 template: Optional[str] = None, params: Optional[dict] = None, code_block: Optional[CodeBlock] = None, - header_row: Optional[List[RendererTableValue]] = None, - table: Optional[List[List[RendererTableValue]]] = None, + header_row: Optional[list[RendererTableValue]] = None, + table: Optional[list[list[RendererTableValue]]] = None, graph: Optional[dict] = None, meta_notes: Optional[MetaNotes] = None, ) -> None: @@ -764,8 +764,8 @@ def __init__( # noqa: PLR0913 self.code_block: Optional[CodeBlock] = code_block # TableType - self.header_row: Optional[List[RendererTableValue]] = header_row - self.table: Optional[List[List[RendererTableValue]]] = table + self.header_row: Optional[list[RendererTableValue]] = header_row + self.table: Optional[list[list[RendererTableValue]]] = table # GraphType self.graph = RenderedAtomicValueGraph(graph=graph) diff --git a/great_expectations/render/renderer/content_block/content_block.py b/great_expectations/render/renderer/content_block/content_block.py index 023efb8cb826..4264ed81ed1e 100644 --- a/great_expectations/render/renderer/content_block/content_block.py +++ b/great_expectations/render/renderer/content_block/content_block.py @@ -2,7 +2,7 @@ import logging import traceback -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union from great_expectations.core import ( ExpectationValidationResult, @@ -31,10 +31,10 @@ class ContentBlockRenderer(Renderer): - _rendered_component_type: Type[RenderedComponentContent] = TextContent + _rendered_component_type: type[RenderedComponentContent] = TextContent _default_header = "" - _default_content_block_styling: Dict[str, JSONValues] = {"classes": ["col-12"]} + _default_content_block_styling: dict[str, JSONValues] = {"classes": ["col-12"]} _default_element_styling = {} diff --git a/great_expectations/render/renderer/content_block/expectation_string.py b/great_expectations/render/renderer/content_block/expectation_string.py index 30118c68244e..8e619921e3a1 100644 --- a/great_expectations/render/renderer/content_block/expectation_string.py +++ b/great_expectations/render/renderer/content_block/expectation_string.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from great_expectations.compatibility.typing_extensions import override from great_expectations.render import RenderedStringTemplateContent @@ -27,7 +27,7 @@ def _missing_content_block_fn( result: Optional[ExpectationValidationResult] = None, runtime_configuration: Optional[dict] = None, **kwargs, - ) -> List[RenderedStringTemplateContent]: + ) -> list[RenderedStringTemplateContent]: renderer_configuration: RendererConfiguration = RendererConfiguration( configuration=configuration, result=result, diff --git a/great_expectations/render/renderer/inline_renderer.py b/great_expectations/render/renderer/inline_renderer.py index cb115e1dfa46..68fcb6e1492d 100644 --- a/great_expectations/render/renderer/inline_renderer.py +++ b/great_expectations/render/renderer/inline_renderer.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Callable, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Optional, Union from typing_extensions import TypedDict @@ -53,7 +53,7 @@ def __init__( def _get_atomic_rendered_content_for_object( self, render_object: Union[ExpectationConfiguration, ExpectationValidationResult], - ) -> List[RenderedAtomicContent]: + ) -> list[RenderedAtomicContent]: """Gets RenderedAtomicContent for a given ExpectationConfiguration or ExpectationValidationResult. Args: @@ -63,7 +63,7 @@ def _get_atomic_rendered_content_for_object( A list of RenderedAtomicContent objects for a given ExpectationConfiguration or ExpectationValidationResult. """ # noqa: E501 expectation_type: str - renderer_types: List[AtomicRendererType] + renderer_types: list[AtomicRendererType] if isinstance(render_object, ExpectationConfiguration): expectation_type = render_object.type renderer_types = [AtomicRendererType.PRESCRIPTIVE] @@ -83,14 +83,14 @@ def _get_atomic_rendered_content_for_object( f"InlineRenderer._get_atomic_rendered_content_for_object can only be used with an ExpectationConfiguration or ExpectationValidationResult, but {type(render_object)} was used." # noqa: E501 ) - renderer_names: List[ + renderer_names: list[ Union[str, AtomicDiagnosticRendererType, AtomicPrescriptiveRendererType] ] = get_renderer_names_with_renderer_types( expectation_or_metric_type=expectation_type, renderer_types=renderer_types, ) - rendered_content: List[RenderedAtomicContent] = ( + rendered_content: list[RenderedAtomicContent] = ( self._get_atomic_rendered_content_from_renderer_names( render_object=render_object, renderer_names=renderer_names, @@ -103,12 +103,12 @@ def _get_atomic_rendered_content_for_object( def _get_atomic_rendered_content_from_renderer_names( self, render_object: Union[ExpectationConfiguration, ExpectationValidationResult], - renderer_names: List[ + renderer_names: list[ Union[str, AtomicDiagnosticRendererType, AtomicPrescriptiveRendererType] ], expectation_type: str, - ) -> List[RenderedAtomicContent]: - try_renderer_names: List[ + ) -> list[RenderedAtomicContent]: + try_renderer_names: list[ Union[str, AtomicDiagnosticRendererType, AtomicPrescriptiveRendererType] ] = [ renderer_name @@ -121,7 +121,7 @@ def _get_atomic_rendered_content_from_renderer_names( ] renderer_rendered_content: RenderedAtomicContent - rendered_content: List[RenderedAtomicContent] = [] + rendered_content: list[RenderedAtomicContent] = [] for renderer_name in try_renderer_names: renderer_rendered_content = self._get_renderer_atomic_rendered_content( render_object=render_object, @@ -204,7 +204,7 @@ def _get_rendered_content_from_renderer_impl( def get_rendered_content( self, - ) -> List[RenderedAtomicContent]: + ) -> list[RenderedAtomicContent]: """Gets RenderedAtomicContent for a given object. Returns: diff --git a/great_expectations/render/renderer/page_renderer.py b/great_expectations/render/renderer/page_renderer.py index 91005d8f694e..d004430eac5f 100644 --- a/great_expectations/render/renderer/page_renderer.py +++ b/great_expectations/render/renderer/page_renderer.py @@ -3,7 +3,7 @@ import logging import os from collections import OrderedDict, defaultdict -from typing import TYPE_CHECKING, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Union from dateutil.parser import parse @@ -132,7 +132,7 @@ def render( def _parse_run_values( self, validation_results: ExpectationSuiteValidationResult - ) -> Tuple[str, str]: + ) -> tuple[str, str]: run_id: Union[str, dict, RunIdentifier] = validation_results.meta["run_id"] if isinstance(run_id, str): try: @@ -157,7 +157,7 @@ def _group_evrs_by_column( self, validation_results: ExpectationSuiteValidationResult, expectation_suite_name: str, - ) -> Dict[str, list]: + ) -> dict[str, list]: columns = defaultdict(list) try: suite_meta = ( @@ -184,7 +184,7 @@ def _group_evrs_by_column( def _generate_collapse_content_block( self, - collapse_content_blocks: List[RenderedTableContent], + collapse_content_blocks: list[RenderedTableContent], validation_results: ExpectationSuiteValidationResult, ) -> CollapseContent: attrs = [ @@ -219,10 +219,10 @@ def _generate_collapse_content_block( def _collect_rendered_document_content_sections( self, validation_results: ExpectationSuiteValidationResult, - overview_content_blocks: List[RenderedComponentContent], - collapse_content_blocks: List[RenderedTableContent], - columns: Dict[str, list], - ) -> List[RenderedSectionContent]: + overview_content_blocks: list[RenderedComponentContent], + collapse_content_blocks: list[RenderedTableContent], + columns: dict[str, list], + ) -> list[RenderedSectionContent]: ordered_columns = Renderer._get_column_list_from_evrs(validation_results) sections = [ RenderedSectionContent( @@ -639,9 +639,9 @@ def __init__(self, column_section_renderer=None) -> None: @staticmethod def _get_grouped_and_ordered_expectations_by_column( expectation_suite: ExpectationSuite, - ) -> Tuple[Dict[str, List[ExpectationConfiguration]], List[str]]: - expectations_by_column: Dict[str, List[ExpectationConfiguration]] = {} - ordered_columns: List[str] = [] + ) -> tuple[dict[str, list[ExpectationConfiguration]], list[str]]: + expectations_by_column: dict[str, list[ExpectationConfiguration]] = {} + ordered_columns: list[str] = [] column: str expectation: ExpectationConfiguration @@ -662,7 +662,7 @@ def _get_grouped_and_ordered_expectations_by_column( expectation.type == "expect_table_columns_to_match_ordered_list" and expectation.kwargs.get("column_list") ): - exp_column_list: List[str] = expectation.kwargs["column_list"] + exp_column_list: list[str] = expectation.kwargs["column_list"] if exp_column_list and len(exp_column_list) > 0: ordered_columns = exp_column_list diff --git a/great_expectations/render/renderer/site_builder.py b/great_expectations/render/renderer/site_builder.py index 244520d33c01..3e1616482e2d 100644 --- a/great_expectations/render/renderer/site_builder.py +++ b/great_expectations/render/renderer/site_builder.py @@ -6,7 +6,7 @@ import traceback import urllib from collections import OrderedDict -from typing import TYPE_CHECKING, Any, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Optional from great_expectations import exceptions from great_expectations.core import ExpectationSuite @@ -709,7 +709,7 @@ def _get_call_to_action_buttons(self, usage_statistics): # TODO: deprecate dual batch api support def build( self, skip_and_clean_missing=True, build_index: bool = True - ) -> Tuple[Any, Optional[OrderedDict]]: + ) -> tuple[Any, Optional[OrderedDict]]: """ :param skip_and_clean_missing: if True, target html store keys without corresponding source store keys will be skipped and removed from the target store @@ -796,7 +796,7 @@ def _add_expectations_to_index_links( def _build_validation_and_profiling_result_site_keys( self, skip_and_clean_missing: bool - ) -> List[ValidationResultIdentifier]: + ) -> list[ValidationResultIdentifier]: validation_and_profiling_result_site_keys = [] validations = self.site_section_builders_config.get("validations", "None") profiling = self.site_section_builders_config.get("profiling", "None") @@ -838,7 +838,7 @@ def _build_validation_and_profiling_result_site_keys( def _add_profiling_to_index_links( self, index_links_dict: OrderedDict, - validation_and_profiling_result_site_keys: List[ValidationResultIdentifier], + validation_and_profiling_result_site_keys: list[ValidationResultIdentifier], ) -> None: profiling = self.site_section_builders_config.get("profiling", "None") if profiling and profiling not in FALSEY_YAML_STRINGS: @@ -883,7 +883,7 @@ def _add_profiling_to_index_links( def _add_validations_to_index_links( self, index_links_dict: OrderedDict, - validation_and_profiling_result_site_keys: List[ValidationResultIdentifier], + validation_and_profiling_result_site_keys: list[ValidationResultIdentifier], ) -> None: validations = self.site_section_builders_config.get("validations", "None") if validations and validations not in FALSEY_YAML_STRINGS: diff --git a/great_expectations/render/renderer_configuration.py b/great_expectations/render/renderer_configuration.py index 015c2cf3f60d..ff166797ab8a 100644 --- a/great_expectations/render/renderer_configuration.py +++ b/great_expectations/render/renderer_configuration.py @@ -7,13 +7,9 @@ from typing import ( TYPE_CHECKING, Any, - Dict, Generic, Iterable, - List, Optional, - Tuple, - Type, TypeVar, Union, ) @@ -107,9 +103,9 @@ def dict( # noqa: PLR0913 RendererParams = TypeVar("RendererParams", bound=_RendererValueBase) -RendererValueTypes: TypeAlias = Union[RendererValueType, List[RendererValueType]] +RendererValueTypes: TypeAlias = Union[RendererValueType, list[RendererValueType]] -AddParamArgs: TypeAlias = Tuple[Tuple[str, RendererValueTypes], ...] +AddParamArgs: TypeAlias = tuple[tuple[str, RendererValueTypes], ...] class RendererTableValue(_RendererValueBase): @@ -130,7 +126,7 @@ class MetaNotes(TypedDict): """Notes that can be added to the meta field of an Expectation.""" format: MetaNotesFormat - content: List[str] + content: list[str] class CodeBlockLanguage(str, Enum): @@ -171,8 +167,8 @@ class RendererConfiguration(pydantic_generics.GenericModel, Generic[RendererPara ) template_str: Optional[str] = Field(None, allow_mutation=True) code_block: CodeBlock = Field({}, allow_mutation=True) - header_row: List[RendererTableValue] = Field([], allow_mutation=True) - table: List[List[RendererTableValue]] = Field([], allow_mutation=True) + header_row: list[RendererTableValue] = Field([], allow_mutation=True) + table: list[list[RendererTableValue]] = Field([], allow_mutation=True) graph: dict = Field({}, allow_mutation=True) include_column_name: bool = Field(True, allow_mutation=False) _raw_kwargs: dict = Field({}, allow_mutation=False) @@ -207,7 +203,7 @@ class _RequiredRendererParamArgs(TypedDict): class _RendererParamArgs(_RequiredRendererParamArgs, total=False): """Used for building up a dictionary that is unpacked into RendererParams upon initialization.""" # noqa: E501 - suite_parameter: Dict[str, Any] + suite_parameter: dict[str, Any] class _RendererParamBase(_RendererValueBase): """ @@ -217,7 +213,7 @@ class _RendererParamBase(_RendererValueBase): renderer_schema: RendererSchema = Field(alias="schema") value: Any - suite_parameter: Optional[Dict[str, Any]] + suite_parameter: Optional[dict[str, Any]] class Config: validate_assignment = True @@ -275,7 +271,7 @@ def __eq__(self, other: object) -> bool: @staticmethod def _get_renderer_value_base_model_type( name: str, - ) -> Type[BaseModel]: + ) -> type[BaseModel]: return create_model( name, renderer_schema=( @@ -288,8 +284,8 @@ def _get_renderer_value_base_model_type( @staticmethod def _get_suite_parameter_params_from_raw_kwargs( - raw_kwargs: Dict[str, Any], - ) -> Dict[str, RendererConfiguration._RendererParamArgs]: + raw_kwargs: dict[str, Any], + ) -> dict[str, RendererConfiguration._RendererParamArgs]: renderer_params_args = {} for kwarg_name, value in raw_kwargs.items(): renderer_params_args[kwarg_name] = RendererConfiguration._RendererParamArgs( @@ -326,7 +322,7 @@ def _validate_and_set_renderer_attrs(cls, values: dict) -> dict: for key, value in raw_configuration.kwargs.items() if (key, value) not in values["kwargs"].items() } - renderer_params_args: Dict[str, RendererConfiguration._RendererParamArgs] = ( + renderer_params_args: dict[str, RendererConfiguration._RendererParamArgs] = ( RendererConfiguration._get_suite_parameter_params_from_raw_kwargs( raw_kwargs=values["_raw_kwargs"] ) @@ -356,11 +352,11 @@ def _validate_for_include_column_name(cls, values: dict) -> dict: @staticmethod def _get_row_condition_params( row_condition_str: str, - ) -> Dict[str, RendererConfiguration._RendererParamArgs]: + ) -> dict[str, RendererConfiguration._RendererParamArgs]: row_condition_str = RendererConfiguration._parse_row_condition_str( row_condition_str=row_condition_str ) - row_conditions_list: List[str] = ( + row_conditions_list: list[str] = ( RendererConfiguration._get_row_conditions_list_from_row_condition_str( row_condition_str=row_condition_str ) @@ -376,7 +372,7 @@ def _get_row_condition_params( @root_validator() def _validate_for_row_condition(cls, values: dict) -> dict: - kwargs: Dict[str, Any] + kwargs: dict[str, Any] if ( "result" in values and values["result"] is not None @@ -388,7 +384,7 @@ def _validate_for_row_condition(cls, values: dict) -> dict: values["_row_condition"] = kwargs.get("row_condition", "") if values["_row_condition"]: - renderer_params_args: Dict[str, RendererConfiguration._RendererParamArgs] = ( + renderer_params_args: dict[str, RendererConfiguration._RendererParamArgs] = ( RendererConfiguration._get_row_condition_params( row_condition_str=values["_row_condition"], ) @@ -430,15 +426,15 @@ def _validate_for_meta_notes(cls, values: dict) -> dict: @root_validator() def _validate_for_params(cls, values: dict) -> dict: - _params: Optional[Dict[str, Dict[str, Union[str, Dict[str, RendererValueType]]]]] = ( + _params: Optional[dict[str, dict[str, Union[str, dict[str, RendererValueType]]]]] = ( values.get("_params") ) if not values["params"]: values["params"] = _RendererValueBase() if _params and _params != values["params"]: - renderer_param_definitions: Dict[str, Any] = {} + renderer_param_definitions: dict[str, Any] = {} for name in _params: - renderer_param_type: Type[BaseModel] = ( + renderer_param_type: type[BaseModel] = ( RendererConfiguration._get_renderer_value_base_model_type(name=name) ) renderer_param_definitions[name] = ( @@ -447,7 +443,7 @@ def _validate_for_params(cls, values: dict) -> dict: ) existing_params = values["params"].__dict__ combined_params = {**existing_params, **_params} - renderer_params: Type[BaseModel] = create_model( + renderer_params: type[BaseModel] = create_model( "RendererParams", **renderer_param_definitions, __base__=_RendererValueBase, @@ -459,7 +455,7 @@ def _validate_for_params(cls, values: dict) -> dict: @staticmethod def _get_row_conditions_list_from_row_condition_str( row_condition_str: str, - ) -> List[str]: + ) -> list[str]: # divide the whole condition into smaller parts row_conditions_list = re.split(r"AND|OR|NOT(?! in)|\(|\)", row_condition_str) row_conditions_list = [ @@ -496,7 +492,7 @@ def _get_row_condition_string(row_condition_str: str) -> str: row_condition_str = RendererConfiguration._parse_row_condition_str( row_condition_str=row_condition_str ) - row_conditions_list: List[str] = ( + row_conditions_list: list[str] = ( RendererConfiguration._get_row_conditions_list_from_row_condition_str( row_condition_str=row_condition_str ) @@ -524,11 +520,11 @@ def _set_template_str(cls, v: str, values: dict) -> str: @staticmethod def _choose_param_type_for_value( - param_types: List[RendererValueType], value: Any + param_types: list[RendererValueType], value: Any ) -> RendererValueType: for param_type in param_types: try: - renderer_param: Type[BaseModel] = ( + renderer_param: type[BaseModel] = ( RendererConfiguration._get_renderer_value_base_model_type(name="try_param") ) renderer_param(schema=RendererSchema(type=param_type), value=value) @@ -566,7 +562,7 @@ def add_param( Returns: None """ # noqa: E501 - renderer_param: Type[BaseModel] = RendererConfiguration._get_renderer_value_base_model_type( + renderer_param: type[BaseModel] = RendererConfiguration._get_renderer_value_base_model_type( name=name ) diff --git a/great_expectations/self_check/sqlalchemy_connection_manager.py b/great_expectations/self_check/sqlalchemy_connection_manager.py index 8f84a10a7b95..066acc0ea9c9 100644 --- a/great_expectations/self_check/sqlalchemy_connection_manager.py +++ b/great_expectations/self_check/sqlalchemy_connection_manager.py @@ -2,7 +2,6 @@ import logging import threading -from typing import Dict from great_expectations.compatibility import sqlalchemy from great_expectations.compatibility.sqlalchemy import ( @@ -18,7 +17,7 @@ class SqlAlchemyConnectionManager: def __init__(self) -> None: self.lock = threading.Lock() - self._connections: Dict[str, sqlalchemy.Connection] = {} + self._connections: dict[str, sqlalchemy.Connection] = {} def get_connection(self, connection_string): if sa is not None: diff --git a/great_expectations/self_check/util.py b/great_expectations/self_check/util.py index f4b886474495..681bc2f1a687 100644 --- a/great_expectations/self_check/util.py +++ b/great_expectations/self_check/util.py @@ -18,12 +18,8 @@ TYPE_CHECKING, Any, Callable, - Dict, Iterable, - List, Optional, - Tuple, - Type, TypeVar, Union, cast, @@ -277,7 +273,7 @@ CLICKHOUSE_TYPES = {} -TRINO_TYPES: Dict[str, Any] = ( +TRINO_TYPES: dict[str, Any] = ( { "BOOLEAN": trino.trinotypes._type_map["boolean"], "TINYINT": trino.trinotypes._type_map["tinyint"], @@ -300,7 +296,7 @@ else {} ) -REDSHIFT_TYPES: Dict[str, Any] = ( +REDSHIFT_TYPES: dict[str, Any] = ( { "BIGINT": aws.redshiftdialect.BIGINT, "BOOLEAN": aws.redshiftdialect.BOOLEAN, @@ -323,7 +319,7 @@ else {} ) -SNOWFLAKE_TYPES: Dict[str, Any] +SNOWFLAKE_TYPES: dict[str, Any] if snowflake.snowflakesqlalchemy and snowflake.snowflakedialect and snowflake.snowflaketypes: # Sometimes "snowflake-sqlalchemy" fails to self-register in certain environments, so we do it explicitly. # noqa: E501 # (see https://stackoverflow.com/questions/53284762/nosuchmoduleerror-cant-load-plugin-sqlalchemy-dialectssnowflake) @@ -352,7 +348,7 @@ else: SNOWFLAKE_TYPES = {} -ATHENA_TYPES: Dict[str, Any] = ( +ATHENA_TYPES: dict[str, Any] = ( # athenatypes is just `from sqlalchemy import types` # https://github.com/laughingman7743/PyAthena/blob/master/pyathena/sqlalchemy_athena.py#L692 # - the _get_column_type method of AthenaDialect does some mapping via conditional statements @@ -456,7 +452,7 @@ def get_test_validator_with_data( # noqa: PLR0913 # if pk_column is defined in our test, then we add a index column to our test set if pk_column: - first_column: List[Any] = list(data.values())[0] + first_column: list[Any] = list(data.values())[0] data["pk_index"] = list(range(len(first_column))) df = pd.DataFrame(data) @@ -585,7 +581,7 @@ def _get_test_validator_with_data_spark( # noqa: C901, PLR0912, PLR0915 context: AbstractDataContext | None, pk_column: bool, ) -> Validator: - spark_types: Dict[str, Callable] = { + spark_types: dict[str, Callable] = { "StringType": pyspark.types.StringType, "IntegerType": pyspark.types.IntegerType, "LongType": pyspark.types.LongType, @@ -626,7 +622,7 @@ def _get_test_validator_with_data_spark( # noqa: C901, PLR0912, PLR0915 for col in schema: type_ = schema[col] # Ints cannot be None...but None can be valid in Spark (as Null) - vals: List[Union[str, int, float, None, Decimal]] = [] + vals: list[Union[str, int, float, None, Decimal]] = [] if type_ in ["IntegerType", "LongType"]: for val in data[col]: if val is None: @@ -730,7 +726,7 @@ def build_sa_validator_with_data( # noqa: C901, PLR0912, PLR0913, PLR0915 f"(build_sa_validator_with_data) {x}" ) - dialect_classes: Dict[str, Type] = {} + dialect_classes: dict[str, type] = {} dialect_types = {} try: dialect_classes["sqlite"] = sqlalchemy.sqlite.dialect @@ -1056,7 +1052,7 @@ def build_spark_engine( if isinstance(df, pd.DataFrame): if schema is None: - data: Union[pd.DataFrame, List[tuple]] = [ + data: Union[pd.DataFrame, list[tuple]] = [ tuple( None if isinstance(x, (float, int)) and np.isnan(x) else x for x in record.tolist() @@ -1069,8 +1065,8 @@ def build_spark_engine( df = spark.createDataFrame(data=data, schema=schema) # type: ignore[type-var,arg-type] - conf: Iterable[Tuple[str, str]] = spark.sparkContext.getConf().getAll() - spark_config: Dict[str, Any] = dict(conf) + conf: Iterable[tuple[str, str]] = spark.sparkContext.getConf().getAll() + spark_config: dict[str, Any] = dict(conf) execution_engine = SparkDFExecutionEngine( spark_config=spark_config, batch_data_dict={ @@ -1307,7 +1303,7 @@ def build_test_backends_list( # noqa: C901, PLR0912, PLR0913, PLR0915 include_athena=False, include_snowflake=False, raise_exceptions_for_backends: bool = True, -) -> List[str]: +) -> list[str]: """Attempts to identify supported backends by checking which imports are available.""" test_backends = [] @@ -1532,13 +1528,13 @@ def build_test_backends_list( # noqa: C901, PLR0912, PLR0913, PLR0915 def generate_expectation_tests( # noqa: C901, PLR0912, PLR0913, PLR0915 expectation_type: str, - test_data_cases: List[ExpectationTestDataCases], + test_data_cases: list[ExpectationTestDataCases], execution_engine_diagnostics: ExpectationExecutionEngineDiagnostics, raise_exceptions_for_backends: bool = False, ignore_suppress: bool = False, ignore_only_for: bool = False, debug_logger: Optional[logging.Logger] = None, - only_consider_these_backends: Optional[List[str]] = None, + only_consider_these_backends: Optional[list[str]] = None, context: Optional[AbstractDataContext] = None, ): """Determine tests to run @@ -1916,7 +1912,7 @@ def sort_unexpected_values(test_value_list, result_value_list): def evaluate_json_test_v3_api( # noqa: C901, PLR0912, PLR0913 validator: Validator, expectation_type: str, - test: Dict[str, Any], + test: dict[str, Any], raise_exception: bool = True, debug_logger: Optional[Logger] = None, pk_column: bool = False, diff --git a/great_expectations/types/__init__.py b/great_expectations/types/__init__.py index 0540e94c69ac..8ded0afa6ac7 100644 --- a/great_expectations/types/__init__.py +++ b/great_expectations/types/__init__.py @@ -60,8 +60,8 @@ class MyClassA(DictDot): For more examples of usage, please see `test_dataclass_serializable_dot_dict_pattern.py` in the tests folder. """ # noqa: E501 - include_field_names: ClassVar[Set[str]] = set() - exclude_field_names: ClassVar[Set[str]] = set() + include_field_names: ClassVar[set[str]] = set() + exclude_field_names: ClassVar[set[str]] = set() def __getitem__(self, item): if isinstance(item, int): @@ -164,9 +164,9 @@ def to_dict(self) -> dict: # noqa: C901 def property_names( # noqa: C901 self, - include_keys: Optional[Set[str]] = None, - exclude_keys: Optional[Set[str]] = None, - ) -> Set[str]: + include_keys: Optional[set[str]] = None, + exclude_keys: Optional[set[str]] = None, + ) -> set[str]: """ Assuming that -- by convention -- names of private properties of an object are prefixed by "_" (a single underscore character), return these property names as public property names. To support this convention, the @@ -191,20 +191,20 @@ def property_names( # noqa: C901 # Gather private fields: # By Python convention, properties of non-trivial length, prefixed by underscore ("_") character, are private. # noqa: E501 - private_fields: Set[str] = set( + private_fields: set[str] = set( filter( lambda name: len(name) > 1, [key[1:] for key in self.keys() if key[0] == "_"], ) ) # Gather public fields. - public_fields: Set[str] = {key for key in self.keys() if key[0] != "_"} + public_fields: set[str] = {key for key in self.keys() if key[0] != "_"} # Combine private and public fields using the "Set Union" operation. - property_names: Set[str] = public_fields | private_fields + property_names: set[str] = public_fields | private_fields keys_for_exclusion: list = [] - def assert_valid_keys(keys: Set[str], purpose: str) -> None: + def assert_valid_keys(keys: set[str], purpose: str) -> None: name: str for name in keys: try: @@ -233,7 +233,7 @@ def assert_valid_keys(keys: Set[str], purpose: str) -> None: class SerializableDictDot(DictDot): - def to_json_dict(self) -> Dict[str, JSONValues]: + def to_json_dict(self) -> dict[str, JSONValues]: """Returns a JSON-serializable dict representation of the SerializableDictDot. Subclasses must implement this abstract method. diff --git a/great_expectations/types/base.py b/great_expectations/types/base.py index 343893a8600d..eb58915f251c 100644 --- a/great_expectations/types/base.py +++ b/great_expectations/types/base.py @@ -2,7 +2,7 @@ import copy import logging -from typing import TYPE_CHECKING, Callable, List, TypeVar +from typing import TYPE_CHECKING, Callable, TypeVar if TYPE_CHECKING: from typing_extensions import Self @@ -41,7 +41,7 @@ def __deepcopy__(self, memo): # The following are required to support yaml serialization, since we do not raise # AttributeError from __getattr__ in DotDict. We *do* raise that AttributeError when it is possible to know # noqa: E501 # a given attribute is not allowed (because it's not in _allowed_keys) - _yaml_merge: List = [] + _yaml_merge: list = [] @classmethod def yaml_anchor(cls): diff --git a/great_expectations/util.py b/great_expectations/util.py index 6c533c35e887..d89aa4d72af0 100644 --- a/great_expectations/util.py +++ b/great_expectations/util.py @@ -27,12 +27,8 @@ TYPE_CHECKING, Any, Callable, - Dict, - List, Optional, - Set, SupportsFloat, - Tuple, Union, cast, overload, @@ -92,9 +88,9 @@ class bidict(dict): Bi-directional hashmap: https://stackoverflow.com/a/21894086 """ - def __init__(self, *args: List[Any], **kwargs: Dict[str, Any]) -> None: + def __init__(self, *args: list[Any], **kwargs: dict[str, Any]) -> None: super().__init__(*args, **kwargs) - self.inverse: Dict = {} + self.inverse: dict = {} for key, value in self.items(): self.inverse.setdefault(value, []).append(key) @@ -330,8 +326,8 @@ def gen_directory_tree_str(startpath: PathStr): def filter_properties_dict( # noqa: C901, PLR0912, PLR0913 properties: Optional[dict] = None, - keep_fields: Optional[Set[str]] = None, - delete_fields: Optional[Set[str]] = None, + keep_fields: Optional[set[str]] = None, + delete_fields: Optional[set[str]] = None, clean_nulls: bool = True, clean_falsy: bool = False, keep_falsy_numerics: bool = True, @@ -444,8 +440,8 @@ def filter_properties_dict( # noqa: C901, PLR0912, PLR0913 @overload def deep_filter_properties_iterable( properties: dict, - keep_fields: Optional[Set[str]] = ..., - delete_fields: Optional[Set[str]] = ..., + keep_fields: Optional[set[str]] = ..., + delete_fields: Optional[set[str]] = ..., clean_nulls: bool = ..., clean_falsy: bool = ..., keep_falsy_numerics: bool = ..., @@ -456,8 +452,8 @@ def deep_filter_properties_iterable( @overload def deep_filter_properties_iterable( properties: list, - keep_fields: Optional[Set[str]] = ..., - delete_fields: Optional[Set[str]] = ..., + keep_fields: Optional[set[str]] = ..., + delete_fields: Optional[set[str]] = ..., clean_nulls: bool = ..., clean_falsy: bool = ..., keep_falsy_numerics: bool = ..., @@ -468,8 +464,8 @@ def deep_filter_properties_iterable( @overload def deep_filter_properties_iterable( properties: set, - keep_fields: Optional[Set[str]] = ..., - delete_fields: Optional[Set[str]] = ..., + keep_fields: Optional[set[str]] = ..., + delete_fields: Optional[set[str]] = ..., clean_nulls: bool = ..., clean_falsy: bool = ..., keep_falsy_numerics: bool = ..., @@ -480,8 +476,8 @@ def deep_filter_properties_iterable( @overload def deep_filter_properties_iterable( properties: tuple, - keep_fields: Optional[Set[str]] = ..., - delete_fields: Optional[Set[str]] = ..., + keep_fields: Optional[set[str]] = ..., + delete_fields: Optional[set[str]] = ..., clean_nulls: bool = ..., clean_falsy: bool = ..., keep_falsy_numerics: bool = ..., @@ -492,8 +488,8 @@ def deep_filter_properties_iterable( @overload def deep_filter_properties_iterable( properties: None, - keep_fields: Optional[Set[str]] = ..., - delete_fields: Optional[Set[str]] = ..., + keep_fields: Optional[set[str]] = ..., + delete_fields: Optional[set[str]] = ..., clean_nulls: bool = ..., clean_falsy: bool = ..., keep_falsy_numerics: bool = ..., @@ -503,8 +499,8 @@ def deep_filter_properties_iterable( def deep_filter_properties_iterable( # noqa: C901, PLR0913 properties: Union[dict, list, set, tuple, None] = None, - keep_fields: Optional[Set[str]] = None, - delete_fields: Optional[Set[str]] = None, + keep_fields: Optional[set[str]] = None, + delete_fields: Optional[set[str]] = None, clean_nulls: bool = True, clean_falsy: bool = False, keep_falsy_numerics: bool = True, @@ -544,7 +540,7 @@ def deep_filter_properties_iterable( # noqa: C901, PLR0913 ) # Upon unwinding the call stack, do a sanity check to ensure cleaned properties. - keys_to_delete: List[str] = list( + keys_to_delete: list[str] = list( filter( lambda k: k not in keep_fields # type: ignore[arg-type] and _is_to_be_removed_from_deep_filter_properties_iterable( @@ -597,7 +593,7 @@ def deep_filter_properties_iterable( # noqa: C901, PLR0913 def _is_to_be_removed_from_deep_filter_properties_iterable( value: Any, clean_nulls: bool, clean_falsy: bool, keep_falsy_numerics: bool ) -> bool: - conditions: Tuple[bool, ...] = ( + conditions: tuple[bool, ...] = ( clean_nulls and value is None, not keep_falsy_numerics and is_numeric(value) and value == 0, clean_falsy and not (is_numeric(value) or value), @@ -787,7 +783,7 @@ def convert_ndarray_to_datetime_dtype_best_effort( datetime_detected: bool = False, parse_strings_as_datetimes: bool = False, fuzzy: bool = False, -) -> Tuple[bool, bool, npt.NDArray]: +) -> tuple[bool, bool, npt.NDArray]: """ Attempt to parse all elements of 1-D "np.ndarray" argument into "datetime.datetime" type objects. @@ -841,7 +837,7 @@ def convert_ndarray_float_to_datetime_dtype(data: np.ndarray) -> np.ndarray: def convert_ndarray_float_to_datetime_tuple( data: np.ndarray, -) -> Tuple[datetime.datetime, ...]: +) -> tuple[datetime.datetime, ...]: """ Convert all elements of 1-D "np.ndarray" argument from "float" type to "datetime.datetime" type tuple elements. @@ -901,7 +897,7 @@ def is_sane_slack_webhook(url: str) -> bool: return url.strip().startswith("https://hooks.slack.com/") -def is_list_of_strings(_list) -> TypeGuard[List[str]]: +def is_list_of_strings(_list) -> TypeGuard[list[str]]: return isinstance(_list, list) and all(isinstance(site, str) for site in _list) @@ -1146,7 +1142,7 @@ def convert_to_json_serializable( # noqa: C901, PLR0911, PLR0912 return new_dict if isinstance(data, (list, tuple, set)): - new_list: List[JSONValues] = [] + new_list: list[JSONValues] = [] for val in data: new_list.append(convert_to_json_serializable(val)) diff --git a/great_expectations/validator/computed_metric.py b/great_expectations/validator/computed_metric.py index 7af8fa305851..7197ce66a73c 100644 --- a/great_expectations/validator/computed_metric.py +++ b/great_expectations/validator/computed_metric.py @@ -1,15 +1,15 @@ from __future__ import annotations -from typing import Any, Dict, List, Set, Tuple, Union +from typing import Any, Union import numpy as np import pandas as pd MetricValue = Union[ Any, # Encompasses deferred-query/execution plans ("SQLAlchemy" and "Spark") conditions and aggregation functions. # noqa: E501 - List[Any], - Set[Any], - Tuple[Any, ...], + list[Any], + set[Any], + tuple[Any, ...], pd.DataFrame, pd.Series, np.ndarray, @@ -17,5 +17,5 @@ str, float, bool, - Dict[str, Any], + dict[str, Any], ] diff --git a/great_expectations/validator/metric_configuration.py b/great_expectations/validator/metric_configuration.py index c6abd5a2c913..74d13deb42ba 100644 --- a/great_expectations/validator/metric_configuration.py +++ b/great_expectations/validator/metric_configuration.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from typing import Optional, Tuple, Union +from typing import Optional, Union from great_expectations.compatibility.typing_extensions import override from great_expectations.core.domain import Domain @@ -152,7 +152,7 @@ def get_domain_type(self) -> MetricDomainTypes: return MetricDomainTypes.TABLE @property - def id(self) -> Tuple[str, str, str]: + def id(self) -> tuple[str, str, str]: return ( self.metric_name, self.metric_domain_kwargs_id, diff --git a/great_expectations/validator/metrics_calculator.py b/great_expectations/validator/metrics_calculator.py index c45c7af1b04f..c4f4be682207 100644 --- a/great_expectations/validator/metrics_calculator.py +++ b/great_expectations/validator/metrics_calculator.py @@ -2,7 +2,7 @@ import logging from collections.abc import Hashable -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from great_expectations.validator.computed_metric import MetricValue from great_expectations.validator.exception_info import ExceptionInfo @@ -19,11 +19,11 @@ logging.captureWarnings(True) -_MetricKey: TypeAlias = Union[Tuple[str, Hashable, Hashable], Tuple[str, str, str]] -_MetricsDict: TypeAlias = Dict[_MetricKey, MetricValue] -_AbortedMetricsInfoDict: TypeAlias = Dict[ +_MetricKey: TypeAlias = Union[tuple[str, Hashable, Hashable], tuple[str, str, str]] +_MetricsDict: TypeAlias = dict[_MetricKey, MetricValue] +_AbortedMetricsInfoDict: TypeAlias = dict[ _MetricKey, - Dict[str, Union[MetricConfiguration, ExceptionInfo, int]], + dict[str, Union[MetricConfiguration, ExceptionInfo, int]], ] @@ -51,7 +51,7 @@ def show_progress_bars(self) -> bool: def show_progress_bars(self, enable: bool) -> None: self._show_progress_bars = enable - def columns(self, domain_kwargs: Optional[Dict[str, Any]] = None) -> List[str]: + def columns(self, domain_kwargs: Optional[dict[str, Any]] = None) -> list[str]: """ Convenience method to run "table.columns" metric. @@ -67,7 +67,7 @@ def columns(self, domain_kwargs: Optional[Dict[str, Any]] = None) -> List[str]: if domain_kwargs.get("batch_id") is None: domain_kwargs["batch_id"] = self._execution_engine.batch_manager.active_batch_id - columns: List[str] = self.get_metric( + columns: list[str] = self.get_metric( metric=MetricConfiguration( metric_name="table.columns", metric_domain_kwargs=domain_kwargs, @@ -79,7 +79,7 @@ def columns(self, domain_kwargs: Optional[Dict[str, Any]] = None) -> List[str]: def head( self, n_rows: int = 5, - domain_kwargs: Optional[Dict[str, Any]] = None, + domain_kwargs: Optional[dict[str, Any]] = None, fetch_all: bool = False, ) -> pd.DataFrame: """Convenience method to return the first several rows or records from a Batch of data. @@ -122,8 +122,8 @@ def get_metric( def get_metrics( self, - metrics: Dict[str, MetricConfiguration], - ) -> Dict[str, Any]: + metrics: dict[str, MetricConfiguration], + ) -> dict[str, Any]: """ Args: metrics: Dictionary of desired metrics to be resolved; metric_name is key and MetricConfiguration is value. @@ -144,7 +144,7 @@ def get_metrics( def compute_metrics( self, - metric_configurations: List[MetricConfiguration], + metric_configurations: list[MetricConfiguration], runtime_configuration: Optional[dict] = None, min_graph_edges_pbar_enable: int = 0, # Set to low number (e.g., 3) to suppress progress bar for small graphs. @@ -178,7 +178,7 @@ def compute_metrics( def build_metric_dependency_graph( self, - metric_configurations: List[MetricConfiguration], + metric_configurations: list[MetricConfiguration], runtime_configuration: Optional[dict] = None, ) -> ValidationGraph: """ @@ -209,7 +209,7 @@ def resolve_validation_graph_and_handle_aborted_metrics_info( runtime_configuration: Optional[dict] = None, min_graph_edges_pbar_enable: int = 0, # Set to low number (e.g., 3) to suppress progress bar for small graphs. - ) -> Tuple[_MetricsDict, _AbortedMetricsInfoDict]: + ) -> tuple[_MetricsDict, _AbortedMetricsInfoDict]: """ Args: graph: "ValidationGraph" object, containing "metric_edge" structures with "MetricConfiguration" objects. @@ -247,7 +247,7 @@ def resolve_validation_graph( runtime_configuration: Optional[dict] = None, min_graph_edges_pbar_enable: int = 0, # Set to low number (e.g., 3) to suppress progress bar for small graphs. - ) -> Tuple[_MetricsDict, _AbortedMetricsInfoDict]: + ) -> tuple[_MetricsDict, _AbortedMetricsInfoDict]: """ Calls "ValidationGraph.resolve()" method with supplied arguments. diff --git a/great_expectations/validator/validation_graph.py b/great_expectations/validator/validation_graph.py index 5ec362d5c9ad..80d71887a774 100644 --- a/great_expectations/validator/validation_graph.py +++ b/great_expectations/validator/validation_graph.py @@ -5,11 +5,7 @@ from typing import ( TYPE_CHECKING, Callable, - Dict, - List, Optional, - Set, - Tuple, Union, ) @@ -78,7 +74,7 @@ class ValidationGraph: def __init__( self, execution_engine: ExecutionEngine, - edges: Optional[List[MetricEdge]] = None, + edges: Optional[list[MetricEdge]] = None, ) -> None: self._execution_engine = execution_engine @@ -95,12 +91,12 @@ def __eq__(self, other) -> bool: return self.edge_ids == other.edge_ids @property - def edges(self) -> List[MetricEdge]: + def edges(self) -> list[MetricEdge]: """Returns "MetricEdge" objects, contained within this "ValidationGraph" object (as list).""" # noqa: E501 return self._edges @property - def edge_ids(self) -> Set[Tuple[str, str]]: + def edge_ids(self) -> set[tuple[str, str]]: """Returns "MetricEdge" objects, contained within this "ValidationGraph" object (as set of two-tuples).""" # noqa: E501 return {edge.id for edge in self._edges} @@ -167,7 +163,7 @@ def build_metric_dependency_graph( def set_metric_configuration_default_kwargs_if_absent( self, metric_configuration: MetricConfiguration - ) -> Tuple[MetricProvider, Callable]: + ) -> tuple[MetricProvider, Callable]: """ Updates "metric_domain_kwargs" and/or "metric_value_kwargs" of "MetricConfiguration" with defualts (if needed). """ # noqa: E501 @@ -195,11 +191,11 @@ def resolve( min_graph_edges_pbar_enable: int = 0, # Set to low number (e.g., 3) to suppress progress bar for small graphs. show_progress_bars: bool = True, - ) -> Tuple[ - Dict[_MetricKey, MetricValue], + ) -> tuple[ + dict[_MetricKey, MetricValue], _AbortedMetricsInfoDict, ]: - resolved_metrics: Dict[_MetricKey, MetricValue] = {} + resolved_metrics: dict[_MetricKey, MetricValue] = {} # updates graph with aborted metrics aborted_metrics_info: _AbortedMetricsInfoDict = self._resolve( @@ -213,7 +209,7 @@ def resolve( def _resolve( # noqa: C901, PLR0912, PLR0915 self, - metrics: Dict[_MetricKey, MetricValue], + metrics: dict[_MetricKey, MetricValue], runtime_configuration: Optional[dict] = None, min_graph_edges_pbar_enable: int = 0, # Set to low number (e.g., 3) to suppress progress bar for small graphs. # noqa: E501 show_progress_bars: bool = True, @@ -232,8 +228,8 @@ def _resolve( # noqa: C901, PLR0912, PLR0915 failed_metric_info: _AbortedMetricsInfoDict = {} aborted_metrics_info: _AbortedMetricsInfoDict = {} - ready_metrics: Set[MetricConfiguration] - needed_metrics: Set[MetricConfiguration] + ready_metrics: set[MetricConfiguration] + needed_metrics: set[MetricConfiguration] exception_info: ExceptionInfo @@ -323,8 +319,8 @@ def _resolve( # noqa: C901, PLR0912, PLR0915 def _parse( self, - metrics: Dict[_MetricKey, MetricValue], - ) -> Tuple[Set[MetricConfiguration], Set[MetricConfiguration]]: + metrics: dict[_MetricKey, MetricValue], + ) -> tuple[set[MetricConfiguration], set[MetricConfiguration]]: """Given validation graph, returns the ready and needed metrics necessary for validation using a traversal of validation graph (a graph structure of metric ids) edges""" # noqa: E501 unmet_dependency_ids = set() @@ -349,7 +345,7 @@ def _parse( def _set_default_metric_kwargs_if_absent( default_kwarg_values: dict, metric_kwargs: IDDict, - keys: Tuple[str, ...], + keys: tuple[str, ...], ) -> None: key: str for key in keys: @@ -400,11 +396,11 @@ def update(self, graph: ValidationGraph) -> None: def get_exception_info( self, metric_info: _AbortedMetricsInfoDict, - ) -> Dict[str, Union[MetricConfiguration, ExceptionInfo, int]]: + ) -> dict[str, Union[MetricConfiguration, ExceptionInfo, int]]: metric_info = self._filter_metric_info_in_graph(metric_info=metric_info) - metric_exception_info: Dict[str, Union[MetricConfiguration, ExceptionInfo, int]] = {} + metric_exception_info: dict[str, Union[MetricConfiguration, ExceptionInfo, int]] = {} metric_id: _MetricKey - metric_info_item: Dict[str, Union[MetricConfiguration, ExceptionInfo, int]] + metric_info_item: dict[str, Union[MetricConfiguration, ExceptionInfo, int]] for metric_id, metric_info_item in metric_info.items(): metric_exception_info[str(metric_id)] = metric_info_item["exception_info"] @@ -414,7 +410,7 @@ def _filter_metric_info_in_graph( self, metric_info: _AbortedMetricsInfoDict, ) -> _AbortedMetricsInfoDict: - graph_metric_ids: List[_MetricKey] = [] + graph_metric_ids: list[_MetricKey] = [] edge: MetricEdge vertex: MetricConfiguration for edge in self.graph.edges: @@ -423,7 +419,7 @@ def _filter_metric_info_in_graph( graph_metric_ids.append(vertex.id) metric_id: _MetricKey - metric_info_item: Dict[str, Union[MetricConfiguration, Set[ExceptionInfo], int]] + metric_info_item: dict[str, Union[MetricConfiguration, set[ExceptionInfo], int]] return { metric_id: metric_info_item for metric_id, metric_info_item in metric_info.items() diff --git a/great_expectations/validator/validator.py b/great_expectations/validator/validator.py index 91936604bf6e..873fc7a0a7de 100644 --- a/great_expectations/validator/validator.py +++ b/great_expectations/validator/validator.py @@ -14,12 +14,8 @@ TYPE_CHECKING, Any, Callable, - Dict, - List, Optional, Sequence, - Set, - Tuple, Union, ) @@ -91,8 +87,8 @@ @dataclass class ValidationDependencies: # Note: Dependent "metric_name" (key) is different from "metric_name" in dependency "MetricConfiguration" (value). # noqa: E501 - metric_configurations: Dict[str, MetricConfiguration] = field(default_factory=dict) - result_format: Dict[str, Any] = field(default_factory=dict) + metric_configurations: dict[str, MetricConfiguration] = field(default_factory=dict) + result_format: dict[str, Any] = field(default_factory=dict) def set_metric_configuration( self, metric_name: str, metric_configuration: MetricConfiguration @@ -114,13 +110,13 @@ def remove_metric_configuration(self, metric_name: str) -> None: """ # noqa: E501 del self.metric_configurations[metric_name] - def get_metric_names(self) -> List[str]: + def get_metric_names(self) -> list[str]: """ Returns "metric_name" keys, for which "MetricConfiguration" dependency objects have been specified. """ # noqa: E501 return list(self.metric_configurations.keys()) - def get_metric_configurations(self) -> List[MetricConfiguration]: + def get_metric_configurations(self) -> list[MetricConfiguration]: """ Returns "MetricConfiguration" dependency objects specified. """ @@ -155,7 +151,7 @@ def __init__( # noqa: PLR0913 expectation_suite: ExpectationSuite | None = None, expectation_suite_name: Optional[str] = None, data_context: Optional[AbstractDataContext] = None, - batches: List[Batch] | Sequence[Batch | FluentBatch] = tuple(), + batches: list[Batch] | Sequence[Batch | FluentBatch] = tuple(), **kwargs, ) -> None: self._data_context: Optional[AbstractDataContext] = data_context @@ -177,7 +173,7 @@ def __init__( # noqa: PLR0913 expectation_suite=expectation_suite, expectation_suite_name=expectation_suite_name, ) - self._default_expectation_args: Dict[str, Union[bool, str]] = copy.deepcopy( + self._default_expectation_args: dict[str, Union[bool, str]] = copy.deepcopy( Validator.DEFAULT_RUNTIME_CONFIGURATION # type: ignore[arg-type] ) @@ -215,7 +211,7 @@ def expose_dataframe_methods(self, value: bool) -> None: self._expose_dataframe_methods = value @property - def loaded_batch_ids(self) -> List[str]: + def loaded_batch_ids(self) -> list[str]: """Getter for IDs of loaded Batch objects (convenience property)""" return self._execution_engine.batch_manager.loaded_batch_ids @@ -225,12 +221,12 @@ def active_batch_data(self) -> Optional[BatchDataUnion]: return self._execution_engine.batch_manager.active_batch_data @property - def batch_cache(self) -> Dict[str, AnyBatch]: + def batch_cache(self) -> dict[str, AnyBatch]: """Getter for dictionary of Batch objects (convenience property)""" return self._execution_engine.batch_manager.batch_cache @property - def batches(self) -> Dict[str, AnyBatch]: + def batches(self) -> dict[str, AnyBatch]: """Getter for dictionary of Batch objects (alias convenience property, to be deprecated)""" return self.batch_cache @@ -299,8 +295,8 @@ def get_metric( def get_metrics( self, - metrics: Dict[str, MetricConfiguration], - ) -> Dict[str, Any]: + metrics: dict[str, MetricConfiguration], + ) -> dict[str, Any]: """ Convenience method that resolves requested metrics (specified as dictionary, keyed by MetricConfiguration ID). @@ -314,7 +310,7 @@ def get_metrics( def compute_metrics( self, - metric_configurations: List[MetricConfiguration], + metric_configurations: list[MetricConfiguration], runtime_configuration: Optional[dict] = None, min_graph_edges_pbar_enable: int = 0, # Set to low number (e.g., 3) to suppress progress bar for small graphs. @@ -338,7 +334,7 @@ def compute_metrics( min_graph_edges_pbar_enable=min_graph_edges_pbar_enable, ) - def columns(self, domain_kwargs: Optional[Dict[str, Any]] = None) -> List[str]: + def columns(self, domain_kwargs: Optional[dict[str, Any]] = None) -> list[str]: """Convenience method to obtain Batch columns. Arguments: @@ -352,7 +348,7 @@ def columns(self, domain_kwargs: Optional[Dict[str, Any]] = None) -> List[str]: def head( self, n_rows: int = 5, - domain_kwargs: Optional[Dict[str, Any]] = None, + domain_kwargs: Optional[dict[str, Any]] = None, fetch_all: bool = False, ) -> pd.DataFrame: """Convenience method to return the first several rows or records from a Batch of data. @@ -370,7 +366,7 @@ def head( ) @override - def __dir__(self) -> List[str]: + def __dir__(self) -> list[str]: """ This custom magic method is used to enable expectation tab completion on Validator objects. It also allows users to call Pandas.DataFrame methods on Validator objects @@ -460,9 +456,9 @@ def inst_expectation(*args: dict, **kwargs): # noqa: C901, PLR0912 {k: v for k, v in kwargs.items() if k in Validator.RUNTIME_KEYS} ) - allowed_config_keys: Tuple[str, ...] = expectation_impl.get_allowed_config_keys() + allowed_config_keys: tuple[str, ...] = expectation_impl.get_allowed_config_keys() - args_keys: Tuple[str, ...] = expectation_impl.args_keys or tuple() + args_keys: tuple[str, ...] = expectation_impl.args_keys or tuple() arg_name: str @@ -557,16 +553,16 @@ def inst_expectation(*args: dict, **kwargs): # noqa: C901, PLR0912 return inst_expectation - def list_available_expectation_types(self) -> List[str]: + def list_available_expectation_types(self) -> list[str]: """Returns a list of all expectations available to the validator""" keys = dir(self) return [expectation for expectation in keys if expectation.startswith("expect_")] def graph_validate( self, - configurations: List[ExpectationConfiguration], + configurations: list[ExpectationConfiguration], runtime_configuration: Optional[dict] = None, - ) -> List[ExpectationValidationResult]: + ) -> list[ExpectationValidationResult]: """Obtains validation dependencies for each metric using the implementation of their associated expectation, then proceeds to add these dependencies to the validation graph, supply readily available metric implementations to fulfill current metric requirements, @@ -589,11 +585,11 @@ def graph_validate( else: catch_exceptions = False - expectation_validation_graphs: List[ExpectationValidationGraph] + expectation_validation_graphs: list[ExpectationValidationGraph] - evrs: List[ExpectationValidationResult] + evrs: list[ExpectationValidationResult] - processed_configurations: List[ExpectationConfiguration] = [] + processed_configurations: list[ExpectationConfiguration] = [] ( expectation_validation_graphs, @@ -669,19 +665,19 @@ def graph_validate( def _generate_metric_dependency_subgraphs_for_each_expectation_configuration( self, - expectation_configurations: List[ExpectationConfiguration], - processed_configurations: List[ExpectationConfiguration], + expectation_configurations: list[ExpectationConfiguration], + processed_configurations: list[ExpectationConfiguration], catch_exceptions: bool, runtime_configuration: Optional[dict] = None, - ) -> Tuple[ - List[ExpectationValidationGraph], - List[ExpectationValidationResult], - List[ExpectationConfiguration], + ) -> tuple[ + list[ExpectationValidationGraph], + list[ExpectationValidationResult], + list[ExpectationConfiguration], ]: # While evaluating expectation configurations, create sub-graph for every metric dependency and incorporate # noqa: E501 # these sub-graphs under corresponding expectation-level sub-graph (state of ExpectationValidationGraph object). # noqa: E501 - expectation_validation_graphs: List[ExpectationValidationGraph] = [] - evrs: List[ExpectationValidationResult] = [] + expectation_validation_graphs: list[ExpectationValidationGraph] = [] + evrs: list[ExpectationValidationResult] = [] configuration: ExpectationConfiguration evaluated_config: ExpectationConfiguration metric_configuration: MetricConfiguration @@ -739,11 +735,11 @@ def _generate_metric_dependency_subgraphs_for_each_expectation_configuration( def _generate_suite_level_graph_from_expectation_level_sub_graphs( self, - expectation_validation_graphs: List[ExpectationValidationGraph], + expectation_validation_graphs: list[ExpectationValidationGraph], ) -> ValidationGraph: # Collect edges from all expectation-level sub-graphs and incorporate them under common suite-level graph. # noqa: E501 expectation_validation_graph: ExpectationValidationGraph - edges: List[MetricEdge] = list( + edges: list[MetricEdge] = list( itertools.chain.from_iterable( [ expectation_validation_graph.graph.edges @@ -758,14 +754,14 @@ def _resolve_suite_level_graph_and_process_metric_evaluation_errors( # noqa: PL self, graph: ValidationGraph, runtime_configuration: dict, - expectation_validation_graphs: List[ExpectationValidationGraph], - evrs: List[ExpectationValidationResult], - processed_configurations: List[ExpectationConfiguration], + expectation_validation_graphs: list[ExpectationValidationGraph], + evrs: list[ExpectationValidationResult], + processed_configurations: list[ExpectationConfiguration], show_progress_bars: bool, - ) -> Tuple[ + ) -> tuple[ _MetricsDict, - List[ExpectationValidationResult], - List[ExpectationConfiguration], + list[ExpectationValidationResult], + list[ExpectationConfiguration], ]: # Resolve overall suite-level graph and process any MetricResolutionError type exceptions that might occur. # noqa: E501 resolved_metrics: _MetricsDict @@ -780,9 +776,9 @@ def _resolve_suite_level_graph_and_process_metric_evaluation_errors( # noqa: PL ) # Trace MetricResolutionError occurrences to expectations relying on corresponding malfunctioning metrics. # noqa: E501 - rejected_configurations: List[ExpectationConfiguration] = [] + rejected_configurations: list[ExpectationConfiguration] = [] for expectation_validation_graph in expectation_validation_graphs: - metric_exception_info: Dict[str, Union[MetricConfiguration, ExceptionInfo, int]] = ( + metric_exception_info: dict[str, Union[MetricConfiguration, ExceptionInfo, int]] = ( expectation_validation_graph.get_exception_info(metric_info=aborted_metrics_info) ) # Report all MetricResolutionError occurrences impacting expectation and append it to rejected list. # noqa: E501 @@ -808,9 +804,9 @@ def _resolve_suite_level_graph_and_process_metric_evaluation_errors( # noqa: PL def _catch_exceptions_in_failing_expectation_validations( exception_traceback: str, exception: Exception, - failing_expectation_configurations: List[ExpectationConfiguration], - evrs: List[ExpectationValidationResult], - ) -> List[ExpectationValidationResult]: + failing_expectation_configurations: list[ExpectationConfiguration], + evrs: list[ExpectationValidationResult], + ) -> list[ExpectationValidationResult]: """ Catch exceptions in failing Expectation validations and convert to unsuccessful ExpectationValidationResult Args: @@ -846,7 +842,7 @@ def remove_expectation( match_type: str = "domain", remove_multiple_matches: bool = False, id: Optional[str] = None, - ) -> List[ExpectationConfiguration]: + ) -> list[ExpectationConfiguration]: """Remove an ExpectationConfiguration from the ExpectationSuite associated with the Validator. Args: @@ -1091,7 +1087,7 @@ def save_expectation_suite( # noqa: PLR0913 def validate( # noqa: C901, PLR0912, PLR0913 self, expectation_suite: str | ExpectationSuite | None = None, - run_id: str | RunIdentifier | Dict[str, str] | None = None, + run_id: str | RunIdentifier | dict[str, str] | None = None, data_context: Optional[Any] = None, # Cannot type DataContext due to circular import suite_parameters: Optional[dict] = None, catch_exceptions: bool = True, @@ -1342,7 +1338,7 @@ def test_expectation_function(self, function: Callable, *args, **kwargs) -> Call def _parse_validation_graph( validation_graph: ValidationGraph, metrics: _MetricsDict, - ) -> Tuple[Set[MetricConfiguration], Set[MetricConfiguration]]: + ) -> tuple[set[MetricConfiguration], set[MetricConfiguration]]: """Given validation graph, returns the ready and needed metrics necessary for validation using a traversal of validation graph (a graph structure of metric ids) edges""" # noqa: E501 unmet_dependency_ids = set() diff --git a/pyproject.toml b/pyproject.toml index c13b5e57509e..000a048faa16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -302,7 +302,6 @@ lint.ignore = [ "C416", # TODO enable "C417", # TODO enable # https://beta.ruff.rs/docs/rules/#pyupgrade-up - "UP006", # use-pep585-annotation "UP007", # use-pep604-annotation # https://beta.ruff.rs/docs/rules/#flake8-type-checking-tch # minimal cost for standard lib imports; keep this disabled diff --git a/scripts/build_api_docs.py b/scripts/build_api_docs.py index 0ee849bf86e4..9395e88eb1bb 100644 --- a/scripts/build_api_docs.py +++ b/scripts/build_api_docs.py @@ -16,7 +16,7 @@ import re import shutil from pathlib import Path -from typing import Any, Dict, Iterable, List, Set, Tuple +from typing import Any, Iterable from typing import Generator as typeGenerator WHITELISTED_TAG = "--Public API--" @@ -137,7 +137,7 @@ def _get_indentation(value: str) -> int: def _get_dictionary_from_block_in_docstring( docstring: str, block_heading_text: str -) -> Dict[str, str]: +) -> dict[str, str]: """Builds a dictionary of key: description pairs from a block of text in a docstring. Note: The description portion of the key: description pair will have all non-alphanumerics that are used in Markdown @@ -256,7 +256,7 @@ def get_paragraph_block_contents(block_heading_text: str, docstring: str) -> str return block_contents.strip() -def get_yield_or_return_block(docstring: str) -> Tuple[str, str]: +def get_yield_or_return_block(docstring: str) -> tuple[str, str]: """Retrieves the content of the `Yields:` or the `Returns:` block in a docstring, if one of those blocks exists. The retrieved block's contents will be reformatted as a string without line breaks or extra whitespace. @@ -321,7 +321,7 @@ def condense_whitespace(docstring: str) -> str: def build_relevant_api_reference_files( docstring: str, api_doc_id: str, api_doc_path: str -) -> Set[str]: +) -> set[str]: """Builds importable link snippets according to the contents of a docstring's `# Documentation` block. This method will create files if they do not exist, and will append links to the files that already do exist. @@ -350,7 +350,7 @@ def build_relevant_api_reference_files( def get_whitelisted_methods( imported_class: object, -) -> typeGenerator[Tuple[str, object], None, None]: +) -> typeGenerator[tuple[str, object], None, None]: """Provides the method_name and a reference to the method itself for every Public API method in a class. Args: @@ -367,8 +367,8 @@ def get_whitelisted_methods( def parse_method_signature( - signature: inspect.Signature, param_dict: Dict[str, str] -) -> List[List[str]]: + signature: inspect.Signature, param_dict: dict[str, str] +) -> list[list[str]]: """Combines Signature with the contents of a param_dict to create rows for a parameter table. Args: @@ -425,7 +425,7 @@ def get_title(file_path: Path) -> str: return title -def build_relevant_documentation_block(docstring: str) -> List[str]: +def build_relevant_documentation_block(docstring: str) -> list[str]: """Builds a list of links to documentation listed in the '--Documentation--' block of a docstring. Args: @@ -447,7 +447,7 @@ def build_relevant_documentation_block(docstring: str) -> List[str]: def build_method_document( # noqa: C901 method_name: str, method: Any, qualified_path: str, github_path: str -) -> Tuple[str, Set[str], str]: +) -> tuple[str, set[str], str]: """Create API documentation for a given method. This method both creates the content for the API documentation and also writes it to the corresponding file. It @@ -560,7 +560,7 @@ def build_method_document( # noqa: C901 def build_class_document( class_name: str, imported_class: object, import_path: str, github_path: str -) -> Tuple[Set[str], str]: +) -> tuple[set[str], str]: """Create API documentation for a given class and its methods that are part of the Public API. This method both creates the content for the API documentation and also writes it to the corresponding file. @@ -620,7 +620,7 @@ class and its Public API methods # Build API documentation for the class's Public API methods and populate the sidebar update. whitelisted_methods = get_whitelisted_methods(imported_class) in_progress_methods = [] - all_edited_cross_link_files: Set[str] = set() + all_edited_cross_link_files: set[str] = set() sidebar_method_items = [] for method_name, whitelisted_method in whitelisted_methods: abbreviated_method, cross_links, sidebar_id = build_method_document( @@ -683,7 +683,7 @@ def prettify_docstring(docstring: str) -> str: return new_content -def remove_existing_api_reference_files(directory_path: Path) -> List[Path]: +def remove_existing_api_reference_files(directory_path: Path) -> list[Path]: """Remove any files in the file tree for `directory_path` that end in the value of API_CROSS_LINK_SUFFIX. Args: @@ -712,7 +712,7 @@ def _gather_source_files(directory_path: Path) -> Iterable[Path]: return directory_path.rglob("*.py") -def _filter_source_files(file_paths: Iterable[Path]) -> List[Path]: +def _filter_source_files(file_paths: Iterable[Path]) -> list[Path]: """Filters out paths to files starting with an underscore from a list of paths. Args: @@ -725,7 +725,7 @@ def _filter_source_files(file_paths: Iterable[Path]) -> List[Path]: return [_ for _ in file_paths if not _.name.startswith("_")] -def get_relevant_source_files(directory_path: Path) -> List[Path]: +def get_relevant_source_files(directory_path: Path) -> list[Path]: """Retrieves filepaths to all public python files in a directory tree. Args: @@ -769,7 +769,7 @@ def convert_to_import_path(file_path: Path) -> str: def gather_classes_to_document( import_path: str, -) -> typeGenerator[Tuple[str, object], None, None]: +) -> typeGenerator[tuple[str, object], None, None]: """Get all the class names and object references for Public API classes in a given module. Args: diff --git a/scripts/gen_stub.py b/scripts/gen_stub.py index 4a2506abbab6..5e9178216d68 100644 --- a/scripts/gen_stub.py +++ b/scripts/gen_stub.py @@ -1,7 +1,7 @@ from __future__ import annotations from inspect import Parameter, Signature -from typing import TYPE_CHECKING, Callable, ForwardRef, Protocol, Type +from typing import TYPE_CHECKING, Callable, ForwardRef, Protocol from great_expectations.datasource.fluent import Datasource, PandasFilesystemDatasource from great_expectations.datasource.fluent.sources import DataSourceManager @@ -64,7 +64,7 @@ def _print_method( # noqa: C901, PLR0912 def print_add_asset_method_signatures( - datasource_class: Type[Datasource], + datasource_class: type[Datasource], method_name_template_str: str = "add_{0}_asset", default_override: str = "...", ): diff --git a/tests/actions/test_core_actions.py b/tests/actions/test_core_actions.py index c08b3aa01a9e..f0b48e127542 100644 --- a/tests/actions/test_core_actions.py +++ b/tests/actions/test_core_actions.py @@ -5,7 +5,7 @@ from contextlib import contextmanager from datetime import datetime, timezone from types import ModuleType -from typing import TYPE_CHECKING, Iterator, Type +from typing import TYPE_CHECKING, Iterator from unittest import mock import pytest @@ -223,7 +223,7 @@ class TestActionSerialization: def test_action_serialization_and_deserialization( self, mock_context, - action_class: Type[ValidationAction], + action_class: type[ValidationAction], init_params: dict, ): expected = self.SERIALIZED_ACTIONS[action_class] @@ -241,7 +241,7 @@ def test_action_serialization_and_deserialization( ) @pytest.mark.unit def test_action_deserialization( - self, action_class: Type[ValidationAction], serialized_action: dict + self, action_class: type[ValidationAction], serialized_action: dict ): actual = action_class.parse_obj(serialized_action) assert isinstance(actual, action_class) diff --git a/tests/checkpoint/cloud_config.py b/tests/checkpoint/cloud_config.py index cf54087880f7..2a9257e356d4 100644 --- a/tests/checkpoint/cloud_config.py +++ b/tests/checkpoint/cloud_config.py @@ -1,5 +1,4 @@ import os -from typing import Tuple from great_expectations.data_context.cloud_constants import GXCloudRESTResource from great_expectations.data_context.types.base import DataContextConfig @@ -57,7 +56,7 @@ def store_set(self, key, value, **kwargs): ) -def list_keys(self, prefix: Tuple = ()): +def list_keys(self, prefix: tuple = ()): type_ = self._ge_cloud_resource_type if type_ == GXCloudRESTResource.EXPECTATION_SUITE: return [ diff --git a/tests/checkpoint/conftest.py b/tests/checkpoint/conftest.py index f7350f18b20a..65b0a0e0a44a 100644 --- a/tests/checkpoint/conftest.py +++ b/tests/checkpoint/conftest.py @@ -1,5 +1,3 @@ -from typing import Dict, List - import pandas as pd import pytest @@ -50,7 +48,7 @@ def slack_notification_action(webhook): def common_action_list( store_validation_result_action: dict, update_data_docs_action: dict, -) -> List[dict]: +) -> list[dict]: return [ store_validation_result_action, update_data_docs_action, @@ -58,7 +56,7 @@ def common_action_list( @pytest.fixture -def batch_request_as_dict() -> Dict[str, str]: +def batch_request_as_dict() -> dict[str, str]: return { "datasource_name": "my_pandas_filesystem_datasource", "data_asset_name": "users", @@ -66,7 +64,7 @@ def batch_request_as_dict() -> Dict[str, str]: @pytest.fixture -def fluent_batch_request(batch_request_as_dict: Dict[str, str]) -> FluentBatchRequest: +def fluent_batch_request(batch_request_as_dict: dict[str, str]) -> FluentBatchRequest: return FluentBatchRequest( datasource_name=batch_request_as_dict["datasource_name"], data_asset_name=batch_request_as_dict["data_asset_name"], diff --git a/tests/checkpoint/test_checkpoint.py b/tests/checkpoint/test_checkpoint.py index d82031cb52ba..0ad1b67ac82e 100644 --- a/tests/checkpoint/test_checkpoint.py +++ b/tests/checkpoint/test_checkpoint.py @@ -3,7 +3,7 @@ import json import pathlib import uuid -from typing import TYPE_CHECKING, List, Type +from typing import TYPE_CHECKING from unittest import mock import pandas as pd @@ -656,7 +656,7 @@ def test_checkpoint_sorts_actions(self, validation_definition: ValidationDefinit ) og_action = OpsgenieAlertAction(name="my_opsgenie_action", api_key="api_key") data_docs_action = UpdateDataDocsAction(name="my_docs_action") - actions: List[CheckpointAction] = [pd_action, og_action, data_docs_action] + actions: list[CheckpointAction] = [pd_action, og_action, data_docs_action] validation_definitions = [validation_definition] checkpoint = Checkpoint( @@ -1199,7 +1199,7 @@ def test_is_fresh( has_validation_def_id: bool, has_suite_id: bool, has_batch_def_id: bool, - error_list: list[Type[ResourceFreshnessError]], + error_list: list[type[ResourceFreshnessError]], ): context = in_memory_runtime_context diff --git a/tests/conftest.py b/tests/conftest.py index 9ee7291b4331..8f36bbbd2d1e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ import urllib.parse import warnings from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, Final, Generator, List, Optional +from typing import TYPE_CHECKING, Any, Final, Generator, Optional from unittest import mock import numpy as np @@ -282,7 +282,7 @@ def pytest_addoption(parser): def build_test_backends_list_v2_api(metafunc): - test_backend_names: List[str] = build_test_backends_list_v3_api(metafunc) + test_backend_names: list[str] = build_test_backends_list_v3_api(metafunc) return test_backend_names @@ -312,7 +312,7 @@ def build_test_backends_list_v3_api(metafunc): include_athena: bool = metafunc.config.getoption("--athena") include_snowflake: bool = metafunc.config.getoption("--snowflake") include_clickhouse: bool = metafunc.config.getoption("--clickhouse") - test_backend_names: List[str] = build_test_backends_list_v3( + test_backend_names: list[str] = build_test_backends_list_v3( include_pandas=include_pandas, include_spark=include_spark, include_sqlalchemy=include_sqlalchemy, @@ -516,8 +516,8 @@ def spark_connect_session(test_backends): def basic_spark_df_execution_engine(spark_session): from great_expectations.execution_engine import SparkDFExecutionEngine - conf: List[tuple] = spark_session.sparkContext.getConf().getAll() - spark_config: Dict[str, Any] = dict(conf) + conf: list[tuple] = spark_session.sparkContext.getConf().getAll() + spark_config: dict[str, Any] = dict(conf) execution_engine = SparkDFExecutionEngine( spark_config=spark_config, ) @@ -1665,7 +1665,7 @@ def ge_cloud_access_token() -> str: @pytest.fixture -def request_headers(ge_cloud_access_token: str) -> Dict[str, str]: +def request_headers(ge_cloud_access_token: str) -> dict[str, str]: return { "Content-Type": "application/vnd.api+json", "Authorization": f"Bearer {ge_cloud_access_token}", @@ -1933,7 +1933,7 @@ def _multibatch_generic_csv_generator( start_date: Optional[datetime.datetime] = None, num_event_batches: Optional[int] = 20, num_events_per_batch: Optional[int] = 5, - ) -> List[str]: + ) -> list[str]: data_path = pathlib.Path(data_path) if start_date is None: start_date = datetime.datetime(2000, 1, 1) # noqa: DTZ001 diff --git a/tests/core/factory/test_suite_factory.py b/tests/core/factory/test_suite_factory.py index 0b7179eac0fa..3ed5e1f2e105 100644 --- a/tests/core/factory/test_suite_factory.py +++ b/tests/core/factory/test_suite_factory.py @@ -1,5 +1,4 @@ import re -from typing import Dict from unittest import mock from unittest.mock import ANY as ANY_TEST_ARG from unittest.mock import Mock # noqa: TID251 @@ -247,7 +246,7 @@ def __init__(self, id: int): # This type intentionally mismatches. We want a bad config. self.configuration: dict = {} - def to_json_dict(self) -> Dict[str, JSONValues]: # type: ignore[explicit-override] # FIXME + def to_json_dict(self) -> dict[str, JSONValues]: # type: ignore[explicit-override] # FIXME return {"id": self.id} # Arrange diff --git a/tests/core/test_batch.py b/tests/core/test_batch.py index db4f87553340..c0b68e4fb0f9 100644 --- a/tests/core/test_batch.py +++ b/tests/core/test_batch.py @@ -1,5 +1,3 @@ -from typing import Dict - import pytest from great_expectations.core.batch import ( @@ -11,12 +9,12 @@ @pytest.fixture() -def base_fluent() -> Dict[str, str]: +def base_fluent() -> dict[str, str]: return {"datasource_name": "ds", "data_asset_name": "da"} @pytest.fixture() -def base_block(base_fluent: Dict[str, str]) -> Dict[str, str]: +def base_block(base_fluent: dict[str, str]) -> dict[str, str]: """Basic block-style batch request args""" result = base_fluent.copy() result["data_connector_name"] = "dc" @@ -24,7 +22,7 @@ def base_block(base_fluent: Dict[str, str]) -> Dict[str, str]: @pytest.fixture() -def runtime_base_block(base_block: Dict[str, str]) -> Dict[str, str]: +def runtime_base_block(base_block: dict[str, str]) -> dict[str, str]: """Basic block-style batch request args and a runtime parameter""" result = base_block.copy() result["path"] = "p" @@ -48,7 +46,7 @@ def test_get_batch_request_from_acceptable_arguments_batch_request_passthrough() @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_runtime_parameter_conflicts_raise( - base_block: Dict[str, str], + base_block: dict[str, str], ): with pytest.raises(ValueError) as ve: runtime = base_block.copy() @@ -81,7 +79,7 @@ def test_get_batch_request_from_acceptable_arguments_runtime_parameter_conflicts @pytest.mark.unit @pytest.mark.parametrize("param,value", [("batch_data", "b"), ("query", "q"), ("path", "p")]) def test_get_batch_request_from_acceptable_arguments_runtime_parameter_path( - base_block: Dict[str, str], param, value + base_block: dict[str, str], param, value ): """Setting any of the parameters should result in a runtime batch request""" base_block["batch_identifiers"] = {"a": "1"} @@ -99,7 +97,7 @@ def test_get_batch_request_from_acceptable_arguments_runtime_parameter_path( @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_runtime_batch_identifiers_kwargs( - runtime_base_block: Dict[str, str], + runtime_base_block: dict[str, str], ): """Batch identifiers can be provided by batch identifiers or kwargs""" bids = {"a": 1, "b": 2} @@ -112,7 +110,7 @@ def test_get_batch_request_from_acceptable_arguments_runtime_batch_identifiers_k @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_runtime_parameters_no_batch_identifiers( - runtime_base_block: Dict[str, str], + runtime_base_block: dict[str, str], ): """Batch identifiers can be provided by batch identifiers or kwargs""" # testing no batch identifiers @@ -122,7 +120,7 @@ def test_get_batch_request_from_acceptable_arguments_runtime_parameters_no_batch @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_runtime_batch_identifiers( - runtime_base_block: Dict[str, str], + runtime_base_block: dict[str, str], ): """Batch identifiers can be provided by batch identifiers or kwargs""" bids = {"a": 1, "b": 2} @@ -136,7 +134,7 @@ def test_get_batch_request_from_acceptable_arguments_runtime_batch_identifiers( @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_fluent( - base_fluent: Dict[str, str], + base_fluent: dict[str, str], ): actual = get_batch_request_from_acceptable_arguments(**base_fluent) assert isinstance(actual, FluentBatchRequest) @@ -144,7 +142,7 @@ def test_get_batch_request_from_acceptable_arguments_fluent( @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_fluent_with_options( - base_fluent: Dict[str, str], + base_fluent: dict[str, str], ): actual = get_batch_request_from_acceptable_arguments(**base_fluent) assert isinstance(actual, FluentBatchRequest) @@ -152,7 +150,7 @@ def test_get_batch_request_from_acceptable_arguments_fluent_with_options( @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_fluent_and_block_args_raises( - base_fluent: Dict[str, str], + base_fluent: dict[str, str], ): base_fluent["data_connector_query"] = "q" @@ -163,14 +161,14 @@ def test_get_batch_request_from_acceptable_arguments_fluent_and_block_args_raise @pytest.mark.unit -def test_get_batch_request_from_acceptable_arguments_block(base_block: Dict[str, str]): +def test_get_batch_request_from_acceptable_arguments_block(base_block: dict[str, str]): actual = get_batch_request_from_acceptable_arguments(**base_block) assert isinstance(actual, BatchRequest) @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_block_batch_filter_parameters( - base_block: Dict[str, str], + base_block: dict[str, str], ): filter_params = {"a": "1"} @@ -193,7 +191,7 @@ def test_get_batch_request_from_acceptable_arguments_block_batch_filter_paramete @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_block_data_connector_query( - base_block: Dict[str, str], + base_block: dict[str, str], ): query = {"a": "1"} # any old dict can be passed @@ -206,7 +204,7 @@ def test_get_batch_request_from_acceptable_arguments_block_data_connector_query( @pytest.mark.unit def test_get_batch_request_from_acceptable_arguments_block_partitioner_sampler_batch_spec_passthrough( # noqa: E501 - base_block: Dict[str, str], + base_block: dict[str, str], ): # partitioner and sampling as batch_spec_passthrough base_block["sampling_method"] = "sample" diff --git a/tests/core/test_batch_definition.py b/tests/core/test_batch_definition.py index d90f504313d5..2501f5003762 100644 --- a/tests/core/test_batch_definition.py +++ b/tests/core/test_batch_definition.py @@ -23,8 +23,6 @@ ) if TYPE_CHECKING: - from typing import List - import pytest_mock from great_expectations.datasource.fluent.batch_request import BatchRequest @@ -32,7 +30,7 @@ class DataAssetForTests(DataAsset): @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: raise NotImplementedError @override diff --git a/tests/core/test_config_provider.py b/tests/core/test_config_provider.py index 2ee3a5d66b41..e5b19c2dc4c6 100644 --- a/tests/core/test_config_provider.py +++ b/tests/core/test_config_provider.py @@ -1,5 +1,3 @@ -from typing import Dict - import pytest from great_expectations.core.config_provider import _CloudConfigurationProvider @@ -41,7 +39,7 @@ ], ) def test_CloudConfigurationProvider_get_values( - cloud_config: GXCloudConfig, expected_values: Dict[str, str] + cloud_config: GXCloudConfig, expected_values: dict[str, str] ): provider = _CloudConfigurationProvider(cloud_config) assert provider.get_values() == expected_values diff --git a/tests/core/test_expectation_suite.py b/tests/core/test_expectation_suite.py index e02a9bb06c37..d56fae3c0ad1 100644 --- a/tests/core/test_expectation_suite.py +++ b/tests/core/test_expectation_suite.py @@ -2,7 +2,7 @@ import uuid from copy import copy, deepcopy -from typing import Dict, Union +from typing import Union from unittest import mock from unittest.mock import MagicMock, Mock # noqa: TID251 from uuid import UUID, uuid4 @@ -774,7 +774,7 @@ def test_expectation_suite_equality_single_expectation_true( def test_expectation_suite_equality_false( self, attribute: str, - new_value: Union[str, Dict[str, str]], + new_value: Union[str, dict[str, str]], suite_with_single_expectation: ExpectationSuite, ): different_but_equivalent_suite = deepcopy(suite_with_single_expectation) diff --git a/tests/core/test_http.py b/tests/core/test_http.py index c6b5486aa805..2b44adb79a70 100644 --- a/tests/core/test_http.py +++ b/tests/core/test_http.py @@ -1,5 +1,3 @@ -from typing import Dict - import pytest from great_expectations.core.http import DEFAULT_TIMEOUT, create_session @@ -7,7 +5,7 @@ @pytest.mark.unit def test_session_factory_contains_appropriate_headers( - ge_cloud_access_token: str, request_headers: Dict[str, str] + ge_cloud_access_token: str, request_headers: dict[str, str] ) -> None: session = create_session(access_token=ge_cloud_access_token) for key, val in request_headers.items(): diff --git a/tests/core/test_validation_definition.py b/tests/core/test_validation_definition.py index 9d16462695cb..eaa08d8f8e92 100644 --- a/tests/core/test_validation_definition.py +++ b/tests/core/test_validation_definition.py @@ -2,7 +2,7 @@ import json import uuid -from typing import TYPE_CHECKING, Type +from typing import TYPE_CHECKING from unittest import mock import pandas as pd @@ -889,7 +889,7 @@ def test_is_fresh( has_id: bool, has_suite_id: bool, has_batch_def_id: bool, - error_list: list[Type[ResourceFreshnessError]], + error_list: list[type[ResourceFreshnessError]], ): context = in_memory_runtime_context diff --git a/tests/data_context/abstract_data_context/test_list_datasources.py b/tests/data_context/abstract_data_context/test_list_datasources.py index cc33e57ec28d..1a457dbd7c18 100644 --- a/tests/data_context/abstract_data_context/test_list_datasources.py +++ b/tests/data_context/abstract_data_context/test_list_datasources.py @@ -1,5 +1,3 @@ -from typing import List - import pytest # module level markers @@ -16,9 +14,9 @@ def test_list_datasources_base_data_context_no_datasources(empty_data_context) - # no datasources - observed: List[dict] = context.list_datasources() + observed: list[dict] = context.list_datasources() - expected: List[dict] = [] + expected: list[dict] = [] assert observed == expected diff --git a/tests/data_context/conftest.py b/tests/data_context/conftest.py index 0f06bdb97bd6..6e50b7558f69 100644 --- a/tests/data_context/conftest.py +++ b/tests/data_context/conftest.py @@ -5,7 +5,7 @@ import pathlib import shutil import unittest.mock -from typing import Any, Callable, Dict, Optional, Union, cast +from typing import Any, Callable, Optional, Union, cast from unittest.mock import Mock # noqa: TID251 import pytest @@ -320,7 +320,7 @@ def fake_data_connector_id() -> str: return "0c08e6ba-8ed9-4715-a179-da2f08aab13e" -JSONData = Union[Dict[str, Any]] +JSONData = Union[dict[str, Any]] RequestError = Union[requests.exceptions.HTTPError, requests.exceptions.Timeout] @@ -330,7 +330,7 @@ def __init__( self, json_data: JSONData, status_code: int, - headers: Optional[Dict[str, str]] = None, + headers: Optional[dict[str, str]] = None, exc_to_raise: Optional[RequestError] = None, ) -> None: self._json_data = json_data @@ -414,7 +414,7 @@ def mocked_response(*args, **kwargs): patch=unittest.mock.DEFAULT, delete=unittest.mock.DEFAULT, ) as mock_requests: - for name, mock in cast(Dict[str, Mock], mock_requests).items(): + for name, mock in cast(dict[str, Mock], mock_requests).items(): mock.side_effect = mocked_response print(f"Mocking `requests.{name}` with `{mocked_response.__name__}()`") diff --git a/tests/data_context/fixtures/plugins/extended_checkpoint.py b/tests/data_context/fixtures/plugins/extended_checkpoint.py index 604b9fa65128..5c867316832c 100644 --- a/tests/data_context/fixtures/plugins/extended_checkpoint.py +++ b/tests/data_context/fixtures/plugins/extended_checkpoint.py @@ -1,5 +1,5 @@ import logging -from typing import List, Optional +from typing import Optional from great_expectations.checkpoint import Checkpoint, LegacyCheckpoint @@ -12,7 +12,7 @@ def __init__( name: str, data_context, expectation_suite_name: Optional[str] = None, - action_list: Optional[List[dict]] = None, + action_list: Optional[list[dict]] = None, ): super().__init__( name=name, @@ -28,7 +28,7 @@ def __init__( name: str, data_context, expectation_suite_name: Optional[str] = None, - action_list: Optional[List[dict]] = None, + action_list: Optional[list[dict]] = None, ): super().__init__( name=name, @@ -44,7 +44,7 @@ def __init__( name: str, data_context, expectation_suite_name: Optional[str] = None, - action_list: Optional[List[dict]] = None, + action_list: Optional[list[dict]] = None, ): super().__init__( name=name, diff --git a/tests/data_context/store/test_configuration_store.py b/tests/data_context/store/test_configuration_store.py index 4c97180535f5..2b891292fd54 100644 --- a/tests/data_context/store/test_configuration_store.py +++ b/tests/data_context/store/test_configuration_store.py @@ -1,7 +1,7 @@ import logging import string from pathlib import Path -from typing import List, Optional +from typing import Optional import pytest from marshmallow import INCLUDE, Schema, fields, validates_schema @@ -70,7 +70,7 @@ def validate_schema(self, data, **kwargs): class SampleConfigurationStore(ConfigurationStore): _configuration_class = SampleConfig - def list_keys(self) -> List[DataContextKey]: + def list_keys(self) -> list[DataContextKey]: # Mock values to work with self.self_check return [ConfigurationIdentifier(f"key{char}") for char in string.ascii_uppercase] diff --git a/tests/data_context/store/test_datasource_store_cloud_backend.py b/tests/data_context/store/test_datasource_store_cloud_backend.py index 4e21ecabba84..770ee37ac43b 100644 --- a/tests/data_context/store/test_datasource_store_cloud_backend.py +++ b/tests/data_context/store/test_datasource_store_cloud_backend.py @@ -1,5 +1,4 @@ import urllib.parse -from typing import Dict from unittest import mock import pytest @@ -142,7 +141,7 @@ def test_datasource_store_delete_by_id( ) def test_datasource_http_error_handling( datasource_store_ge_cloud_backend: DatasourceStore, - mock_http_unavailable: Dict[str, mock.Mock], # noqa: TID251 + mock_http_unavailable: dict[str, mock.Mock], # noqa: TID251 http_verb: str, method: str, args: list, diff --git a/tests/data_context/store/test_gx_cloud_store_backend.py b/tests/data_context/store/test_gx_cloud_store_backend.py index 2c135d08ac90..59d1e774d8f4 100644 --- a/tests/data_context/store/test_gx_cloud_store_backend.py +++ b/tests/data_context/store/test_gx_cloud_store_backend.py @@ -10,7 +10,7 @@ in production. The same logic applies to all UUIDs in this test. """ -from typing import Callable, Optional, Set, Union +from typing import Callable, Optional, Union from unittest import mock import pytest @@ -409,7 +409,7 @@ def test_appropriate_casting_of_str_resource_type_to_GXCloudRESTResource( ) def test_allowed_set_kwargs( resource_type: GXCloudRESTResource, - expected_set_kwargs: Set[str], + expected_set_kwargs: set[str], construct_ge_cloud_store_backend: Callable[[GXCloudRESTResource], GXCloudStoreBackend], ) -> None: store_backend = construct_ge_cloud_store_backend(resource_type) diff --git a/tests/data_context/test_cloud_data_context.py b/tests/data_context/test_cloud_data_context.py index eac166b358e0..7377ede36d9b 100644 --- a/tests/data_context/test_cloud_data_context.py +++ b/tests/data_context/test_cloud_data_context.py @@ -1,5 +1,5 @@ import uuid -from typing import Any, Dict +from typing import Any import pytest import responses @@ -18,7 +18,7 @@ def _create_cloud_config_response( expectation_suite_store_name_key: str, validation_results_store_name_key: str, validation_results_store_class_name: str, -) -> Dict[str, Any]: +) -> dict[str, Any]: return { "anonymous_usage_statistics": { "data_context_id": "6a52bdfa-e182-455b-a825-e69f076e67d6", diff --git a/tests/data_context/test_data_context.py b/tests/data_context/test_data_context.py index 9748862d36c6..bd4baff0ec55 100644 --- a/tests/data_context/test_data_context.py +++ b/tests/data_context/test_data_context.py @@ -5,7 +5,7 @@ import pathlib import shutil import uuid -from typing import Dict, List, Union +from typing import Union import pytest @@ -699,7 +699,7 @@ def _prescriptive_renderer_custom( def _validate( # type: ignore[override,explicit-override] # FIXME self, **kwargs: dict, - ) -> Dict[str, Union[bool, dict]]: + ) -> dict[str, Union[bool, dict]]: return { "success": True, "result": {"observed_value": "blue"}, @@ -754,7 +754,7 @@ def test_multiple_rendered_content_blocks_one_is_busted( ) expectation_suite = empty_cloud_data_context.suites.get(name=suite_name) - expected_rendered_content: List[RenderedAtomicContent] = [ + expected_rendered_content: list[RenderedAtomicContent] = [ RenderedAtomicContent( name=AtomicPrescriptiveRendererType.FAILED, value=renderedAtomicValueSchema.load( @@ -801,7 +801,7 @@ def test_multiple_rendered_content_blocks_one_is_busted( ), ] - actual_rendered_content: List[RenderedAtomicContent] = [] + actual_rendered_content: list[RenderedAtomicContent] = [] for expectation in expectation_suite.expectations: assert expectation.rendered_content is not None actual_rendered_content.extend(expectation.rendered_content) diff --git a/tests/data_context/test_data_context_config_ui.py b/tests/data_context/test_data_context_config_ui.py index 6ce38a303eb3..ee64d6dbc56d 100644 --- a/tests/data_context/test_data_context_config_ui.py +++ b/tests/data_context/test_data_context_config_ui.py @@ -1,6 +1,6 @@ import copy import os -from typing import Any, Dict, Final, Optional +from typing import Any, Final, Optional import pytest @@ -45,10 +45,10 @@ def _construct_data_context_config( expectations_store_name: str = DataContextConfigDefaults.DEFAULT_EXPECTATIONS_STORE_NAME.value, # noqa: E501 validation_results_store_name: str = DataContextConfigDefaults.DEFAULT_VALIDATIONS_STORE_NAME.value, # noqa: E501 checkpoint_store_name: str = DataContextConfigDefaults.DEFAULT_CHECKPOINT_STORE_NAME.value, - fluent_datasources: Optional[Dict] = None, + fluent_datasources: Optional[dict] = None, plugins_directory: Optional[str] = None, - stores: Optional[Dict] = None, - data_docs_sites: Optional[Dict] = None, + stores: Optional[dict] = None, + data_docs_sites: Optional[dict] = None, ): if stores is None: stores = copy.deepcopy(DataContextConfigDefaults.DEFAULT_STORES.value) diff --git a/tests/data_context/test_data_context_in_code_config.py b/tests/data_context/test_data_context_in_code_config.py index 4bfa21fbb14c..e4edad5fd55b 100644 --- a/tests/data_context/test_data_context_in_code_config.py +++ b/tests/data_context/test_data_context_in_code_config.py @@ -1,5 +1,5 @@ import uuid -from typing import Dict, Optional, Set +from typing import Optional import boto3 import pyparsing as pp @@ -17,7 +17,7 @@ def build_in_code_data_context_project_config( checkpoint_store_prefix: str = "checkpoint_store_prefix", validation_results_store_prefix: str = "validation_results_store_prefix", data_docs_store_prefix: str = "data_docs_store_prefix", - stores: Optional[Dict] = None, + stores: Optional[dict] = None, ) -> DataContextConfig: """ Create a project config for an in-code data context. @@ -101,7 +101,7 @@ def get_store_backend_id_from_s3(bucket: str, prefix: str, key: str) -> uuid.UUI return uuid.UUID(parsed_store_backend_id[1]) -def list_s3_bucket_contents(bucket: str, prefix: str) -> Set[str]: +def list_s3_bucket_contents(bucket: str, prefix: str) -> set[str]: """ List the contents of an s3 bucket as a set of strings given bucket name and prefix Args: diff --git a/tests/datasource/data_connector/test_batch_filter.py b/tests/datasource/data_connector/test_batch_filter.py index 6566fa52ab60..a3dcb73f4faf 100644 --- a/tests/datasource/data_connector/test_batch_filter.py +++ b/tests/datasource/data_connector/test_batch_filter.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import List - import pytest from great_expectations.datasource.fluent.data_connector.batch_filter import ( @@ -139,9 +137,9 @@ def test_batch_filter_parse_batch_slice( data_connector_query_dict: dict, parsed_batch_slice: slice, - sliced_list: List[int], + sliced_list: list[int], ): - original_list: List[int] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + original_list: list[int] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] batch_filter_obj: BatchFilter = build_batch_filter( data_connector_query_dict=data_connector_query_dict # type: ignore[arg-type] diff --git a/tests/datasource/fluent/_fake_cloud_api.py b/tests/datasource/fluent/_fake_cloud_api.py index 090c62bed4a5..5ef44b59c938 100644 --- a/tests/datasource/fluent/_fake_cloud_api.py +++ b/tests/datasource/fluent/_fake_cloud_api.py @@ -9,14 +9,11 @@ from pprint import pformat as pf from typing import ( TYPE_CHECKING, - Dict, Final, Generator, - List, NamedTuple, Optional, Sequence, - Set, TypedDict, Union, ) @@ -65,7 +62,7 @@ class _DatasourceSchema(pydantic.BaseModel, extra="allow"): id: Optional[str] = None type: str name: str - assets: List[dict] = pydantic.Field(default_factory=list) + assets: list[dict] = pydantic.Field(default_factory=list) class CloudResponseSchema(pydantic.BaseModel): @@ -98,16 +95,16 @@ class CloudDetails(NamedTuple): # using alternative syntax for creating type dict because of key names with hyphens # https://peps.python.org/pep-0589/#alternative-syntax { - "me": Dict[str, str], - "data-context-configuration": Dict[str, Union[str, dict]], - "DATASOURCE_NAMES": Set[str], - "datasources": Dict[str, dict], - "EXPECTATION_SUITE_NAMES": Set[str], - "expectation_suites": Dict[str, dict], - "CHECKPOINT_NAMES": Set[str], - "checkpoints": Dict[str, dict], - "VALIDATION_DEFINITION_NAMES": Set[str], - "validation_definitions": Dict[str, dict], + "me": dict[str, str], + "data-context-configuration": dict[str, Union[str, dict]], + "DATASOURCE_NAMES": set[str], + "datasources": dict[str, dict], + "EXPECTATION_SUITE_NAMES": set[str], + "expectation_suites": dict[str, dict], + "CHECKPOINT_NAMES": set[str], + "checkpoints": dict[str, dict], + "VALIDATION_DEFINITION_NAMES": set[str], + "validation_definitions": dict[str, dict], }, ) diff --git a/tests/datasource/fluent/conftest.py b/tests/datasource/fluent/conftest.py index 929a620aead7..d1bb1aa6d60f 100644 --- a/tests/datasource/fluent/conftest.py +++ b/tests/datasource/fluent/conftest.py @@ -12,12 +12,9 @@ Any, Callable, ContextManager, - Dict, Final, Generator, - List, Optional, - Type, Union, ) @@ -81,7 +78,7 @@ def sqlachemy_execution_engine_mock_cls( validate_batch_spec: Callable[[SqlAlchemyDatasourceBatchSpec], None], dialect: str, - partitioner_query_response: Optional[Union[List[Dict[str, Any]], List[Any]]] = None, + partitioner_query_response: Optional[Union[list[dict[str, Any]], list[Any]]] = None, ): """Creates a mock gx sql alchemy engine class @@ -139,13 +136,13 @@ def get_batch_data_and_markers(self, batch_spec) -> tuple[BatchData, BatchMarker @pytest.fixture def inject_engine_lookup_double( monkeypatch: MonkeyPatch, -) -> Generator[Type[ExecutionEngineDouble], None, None]: +) -> Generator[type[ExecutionEngineDouble], None, None]: """ Inject an execution engine test double into the _SourcesFactory.engine_lookup so that all Datasources use the execution engine double. Dynamically create a new subclass so that runtime type validation does not fail. """ - original_engine_override: dict[Type[Datasource], Type[ExecutionEngine]] = {} + original_engine_override: dict[type[Datasource], type[ExecutionEngine]] = {} for key in DataSourceManager.type_lookup: if issubclass(type(key), Datasource): original_engine_override[key] = key.execution_engine_override @@ -447,7 +444,7 @@ def _source( dialect: str, connection_string: str = "postgresql+psycopg2://postgres:@localhost/test_ci", data_context: Optional[AbstractDataContext] = None, - partitioner_query_response: Optional[List[Dict[str, Any]]] = None, + partitioner_query_response: Optional[list[dict[str, Any]]] = None, create_temp_table: bool = True, ) -> Generator[PostgresDatasource, None, None]: partitioner_response = partitioner_query_response or ( diff --git a/tests/datasource/fluent/data_asset/data_connector/test_azure_blob_storage_data_connector.py b/tests/datasource/fluent/data_asset/data_connector/test_azure_blob_storage_data_connector.py index 2defadb120bc..0b6f13ffe5d1 100644 --- a/tests/datasource/fluent/data_asset/data_connector/test_azure_blob_storage_data_connector.py +++ b/tests/datasource/fluent/data_asset/data_connector/test_azure_blob_storage_data_connector.py @@ -1,6 +1,6 @@ import logging import re -from typing import TYPE_CHECKING, List, cast +from typing import TYPE_CHECKING, cast from unittest import mock import pytest @@ -151,7 +151,7 @@ def test_return_all_batch_definitions_unsorted(mock_list_keys): my_data_connector.get_batch_definition_list() # with empty options - unsorted_batch_definition_list: List[LegacyBatchDefinition] = ( + unsorted_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list( BatchRequest( datasource_name="my_file_path_datasource", @@ -160,7 +160,7 @@ def test_return_all_batch_definitions_unsorted(mock_list_keys): ) ) ) - expected: List[LegacyBatchDefinition] = [ + expected: list[LegacyBatchDefinition] = [ LegacyBatchDefinition( datasource_name="my_file_path_datasource", data_connector_name="fluent", @@ -494,7 +494,7 @@ def test_return_only_unique_batch_definitions(mock_list_keys): "B/file_2.csv", ] - expected: List[LegacyBatchDefinition] = [ + expected: list[LegacyBatchDefinition] = [ LegacyBatchDefinition( datasource_name="my_file_path_datasource", data_connector_name="fluent", @@ -520,7 +520,7 @@ def test_return_only_unique_batch_definitions(mock_list_keys): file_path_template_map_fn=AzureUrl.AZURE_BLOB_STORAGE_HTTPS_URL_TEMPLATE.format, ) - unsorted_batch_definition_list: List[LegacyBatchDefinition] = ( + unsorted_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list( BatchRequest( datasource_name="my_file_path_datasource", @@ -570,7 +570,7 @@ def test_alpha(mock_list_keys): assert my_data_connector.get_unmatched_data_references()[:3] == [] assert my_data_connector.get_unmatched_data_reference_count() == 0 - my_batch_definition_list: List[LegacyBatchDefinition] + my_batch_definition_list: list[LegacyBatchDefinition] my_batch_definition: LegacyBatchDefinition my_batch_request: BatchRequest @@ -728,7 +728,7 @@ def test_foxtrot(mock_list_keys): data_asset_name="my_azure_blob_storage_data_asset", options={}, ) - my_batch_definition_list: List[LegacyBatchDefinition] = ( + my_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list(batch_request=my_batch_request) ) assert len(my_batch_definition_list) == 3 diff --git a/tests/datasource/fluent/data_asset/data_connector/test_filesystem_data_connector.py b/tests/datasource/fluent/data_asset/data_connector/test_filesystem_data_connector.py index 4a0bcd65e3d9..a3e94129377f 100644 --- a/tests/datasource/fluent/data_asset/data_connector/test_filesystem_data_connector.py +++ b/tests/datasource/fluent/data_asset/data_connector/test_filesystem_data_connector.py @@ -1,6 +1,6 @@ import pathlib import re -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, Union import pytest @@ -98,7 +98,7 @@ def test_return_all_batch_definitions_unsorted(tmp_path_factory): my_data_connector.get_batch_definition_list() # with empty options - unsorted_batch_definition_list: List[LegacyBatchDefinition] = ( + unsorted_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list( BatchRequest( datasource_name="my_file_path_datasource", @@ -111,7 +111,7 @@ def test_return_all_batch_definitions_unsorted(tmp_path_factory): processed_batching_regex = re.compile( "(?P(?P.+)_(?P.+)_(?P.+)\\.csv)" ) - expected: List[LegacyBatchDefinition] = [ + expected: list[LegacyBatchDefinition] = [ LegacyBatchDefinition( datasource_name="my_file_path_datasource", data_connector_name="fluent", @@ -307,7 +307,7 @@ def test_return_only_unique_batch_definitions(tmp_path_factory): ] processed_batching_regex = re.compile("(?P(?P.+)/(?P.+\\.csv))") - expected: List[LegacyBatchDefinition] = [ + expected: list[LegacyBatchDefinition] = [ LegacyBatchDefinition( datasource_name="my_file_path_datasource", data_connector_name="fluent", @@ -362,7 +362,7 @@ def test_return_only_unique_batch_definitions(tmp_path_factory): ) batching_regex = re.compile(r"(?P.+)/(?P.+\.csv)") - unsorted_batch_definition_list: List[LegacyBatchDefinition] = ( + unsorted_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list( BatchRequest( datasource_name="my_file_path_datasource", @@ -549,7 +549,7 @@ def test_foxtrot(tmp_path_factory): options={}, partitioner=FileNamePartitionerPath(regex=batching_regex), ) - my_batch_definition_list: List[LegacyBatchDefinition] = ( + my_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list(batch_request=my_batch_request) ) assert len(my_batch_definition_list) == 3 @@ -618,7 +618,7 @@ def test_relative_base_directory_path(tmp_path_factory): options={}, partitioner=FileNamePartitionerPath(regex=batching_regex), ) - my_batch_definition_list: List[LegacyBatchDefinition] = ( + my_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list(batch_request=my_batch_request) ) assert len(my_batch_definition_list) == 1 diff --git a/tests/datasource/fluent/data_asset/data_connector/test_google_cloud_storage_data_connector.py b/tests/datasource/fluent/data_asset/data_connector/test_google_cloud_storage_data_connector.py index 11658db785ba..9998b34c1052 100644 --- a/tests/datasource/fluent/data_asset/data_connector/test_google_cloud_storage_data_connector.py +++ b/tests/datasource/fluent/data_asset/data_connector/test_google_cloud_storage_data_connector.py @@ -1,6 +1,6 @@ import logging import re -from typing import TYPE_CHECKING, Iterator, List, cast +from typing import TYPE_CHECKING, Iterator, cast from unittest import mock import pytest @@ -151,7 +151,7 @@ def test_return_all_batch_definitions_unsorted(mock_list_keys): my_data_connector.get_batch_definition_list() # with empty options - unsorted_batch_definition_list: List[LegacyBatchDefinition] = ( + unsorted_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list( BatchRequest( datasource_name="my_file_path_datasource", @@ -160,7 +160,7 @@ def test_return_all_batch_definitions_unsorted(mock_list_keys): ) ) ) - expected: List[LegacyBatchDefinition] = [ + expected: list[LegacyBatchDefinition] = [ LegacyBatchDefinition( datasource_name="my_file_path_datasource", data_connector_name="fluent", @@ -492,7 +492,7 @@ def test_return_only_unique_batch_definitions(mock_list_keys): "B/file_2.csv", ] - expected: List[LegacyBatchDefinition] = [ + expected: list[LegacyBatchDefinition] = [ LegacyBatchDefinition( datasource_name="my_file_path_datasource", data_connector_name="fluent", @@ -517,7 +517,7 @@ def test_return_only_unique_batch_definitions(mock_list_keys): file_path_template_map_fn=GCSUrl.OBJECT_URL_TEMPLATE.format, ) - unsorted_batch_definition_list: List[LegacyBatchDefinition] = ( + unsorted_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list( BatchRequest( datasource_name="my_file_path_datasource", @@ -566,7 +566,7 @@ def test_alpha(mock_list_keys): assert my_data_connector.get_unmatched_data_references()[:3] == [] assert my_data_connector.get_unmatched_data_reference_count() == 0 - my_batch_definition_list: List[LegacyBatchDefinition] + my_batch_definition_list: list[LegacyBatchDefinition] my_batch_definition: LegacyBatchDefinition my_batch_request: BatchRequest @@ -720,7 +720,7 @@ def test_foxtrot(mock_list_keys): data_asset_name="my_google_cloud_storage_data_asset", options={}, ) - my_batch_definition_list: List[LegacyBatchDefinition] = ( + my_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list(batch_request=my_batch_request) ) assert len(my_batch_definition_list) == 3 diff --git a/tests/datasource/fluent/data_asset/data_connector/test_s3_data_connector.py b/tests/datasource/fluent/data_asset/data_connector/test_s3_data_connector.py index 2103adafffbb..0701ae85fd9c 100644 --- a/tests/datasource/fluent/data_asset/data_connector/test_s3_data_connector.py +++ b/tests/datasource/fluent/data_asset/data_connector/test_s3_data_connector.py @@ -1,7 +1,7 @@ import logging import os import re -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING import pandas as pd import pytest @@ -48,7 +48,7 @@ def test_basic_instantiation(): test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "alpha-1.csv", "alpha-2.csv", "alpha-3.csv", @@ -91,7 +91,7 @@ def test_instantiation_batching_regex_does_not_match_paths(): test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "alpha-1.csv", "alpha-2.csv", "alpha-3.csv", @@ -130,7 +130,7 @@ def test_return_all_batch_definitions_unsorted(): test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "alex_2020-08-09_1000.csv", "eugene_2020-08-09_1500.csv", "james_2020-08-11_1009.csv", @@ -163,7 +163,7 @@ def test_return_all_batch_definitions_unsorted(): partitioner = FileNamePartitionerYearly(regex=batching_regex) # with empty options - unsorted_batch_definition_list: List[LegacyBatchDefinition] = ( + unsorted_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list( BatchRequest( datasource_name="my_file_path_datasource", @@ -176,7 +176,7 @@ def test_return_all_batch_definitions_unsorted(): processed_batching_regex = re.compile( "(?P(?P.+)_(?P\\d{4})-(?P\\d{2})-(?P\\d{2})_(?P.*)\\.csv)" ) - expected: List[LegacyBatchDefinition] = [ + expected: list[LegacyBatchDefinition] = [ LegacyBatchDefinition( datasource_name="my_file_path_datasource", data_connector_name="fluent", @@ -363,7 +363,7 @@ def test_return_only_unique_batch_definitions(): test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "A/file_1.csv", "A/file_2.csv", "A/file_3.csv", @@ -373,7 +373,7 @@ def test_return_only_unique_batch_definitions(): for key in keys: client.put_object(Bucket=bucket, Body=test_df.to_csv(index=False).encode("utf-8"), Key=key) processed_batching_regex = re.compile("(?PB/(?P.+).*\\.csv)") - expected: List[LegacyBatchDefinition] = [ + expected: list[LegacyBatchDefinition] = [ LegacyBatchDefinition( datasource_name="my_file_path_datasource", data_connector_name="fluent", @@ -400,7 +400,7 @@ def test_return_only_unique_batch_definitions(): ) batching_regex = re.compile(r"(?P.+).*\.csv") - unsorted_batch_definition_list: List[LegacyBatchDefinition] = ( + unsorted_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list( BatchRequest( datasource_name="my_file_path_datasource", @@ -424,7 +424,7 @@ def test_data_reference_count_methods(): test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "A/file_1.csv", "A/file_2.csv", "A/file_3.csv", @@ -470,7 +470,7 @@ def test_alpha(): test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "test_dir_alpha/A.csv", "test_dir_alpha/B.csv", "test_dir_alpha/C.csv", @@ -538,7 +538,7 @@ def test_foxtrot(): test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "test_dir_foxtrot/A/A-1.csv", "test_dir_foxtrot/A/A-2.csv", "test_dir_foxtrot/A/A-3.csv", @@ -652,7 +652,7 @@ def test_foxtrot(): data_asset_name="my_s3_data_asset", options={}, ) - my_batch_definition_list: List[LegacyBatchDefinition] = ( + my_batch_definition_list: list[LegacyBatchDefinition] = ( my_data_connector.get_batch_definition_list(batch_request=my_batch_request) ) assert len(my_batch_definition_list) == 3 diff --git a/tests/datasource/fluent/data_asset/test_data_asset.py b/tests/datasource/fluent/data_asset/test_data_asset.py index e56790445224..6feb407d134d 100644 --- a/tests/datasource/fluent/data_asset/test_data_asset.py +++ b/tests/datasource/fluent/data_asset/test_data_asset.py @@ -1,5 +1,3 @@ -from typing import List - import pytest from great_expectations.core.batch_definition import BatchDefinition @@ -327,7 +325,7 @@ def test_delete_batch_definition__cloud_data__does_not_clobber_other_assets( def _test_delete_batch_definition__does_not_clobber_other_assets( - context: AbstractDataContext, datasource_name: str, asset_names: List[str] + context: AbstractDataContext, datasource_name: str, asset_names: list[str] ): # each asset has one batch config; delete it for asset_name in asset_names: @@ -350,7 +348,7 @@ class _MyPartitioner(FluentBaseModel): sort_ascending: bool = True @property - def param_names(self) -> List[str]: + def param_names(self) -> list[str]: return ["a", "b"] diff --git a/tests/datasource/fluent/data_asset/test_path_asset.py b/tests/datasource/fluent/data_asset/test_path_asset.py index e04c13206dff..ac80cd5d7d0b 100644 --- a/tests/datasource/fluent/data_asset/test_path_asset.py +++ b/tests/datasource/fluent/data_asset/test_path_asset.py @@ -1,6 +1,6 @@ import pathlib import re -from typing import Final, List, Union +from typing import Final, Union import pytest @@ -84,7 +84,7 @@ def validated_pandas_filesystem_datasource( pandas_filesystem_datasource: PandasFilesystemDatasource, ): years = ["2018", "2019", "2020"] - all_files: List[str] = [ + all_files: list[str] = [ file_name.stem for file_name in list(pathlib.Path(pandas_filesystem_datasource.base_directory).iterdir()) ] diff --git a/tests/datasource/fluent/integration/integration_test_utils.py b/tests/datasource/fluent/integration/integration_test_utils.py index 00dc346a6488..83634ee37acc 100644 --- a/tests/datasource/fluent/integration/integration_test_utils.py +++ b/tests/datasource/fluent/integration/integration_test_utils.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from typing import Dict, Tuple import pytest @@ -135,10 +134,10 @@ def run_batch_head( # noqa: C901 execution_engine: ExecutionEngine = batch.data.execution_engine execution_engine.batch_manager.load_batch_list(batch_list=[batch]) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=execution_engine) metrics.update(results) diff --git a/tests/datasource/fluent/test_batch.py b/tests/datasource/fluent/test_batch.py index c1ce235d58da..5531e02b7417 100644 --- a/tests/datasource/fluent/test_batch.py +++ b/tests/datasource/fluent/test_batch.py @@ -1,7 +1,6 @@ from __future__ import annotations import pathlib -from typing import Tuple import pytest @@ -17,7 +16,7 @@ @pytest.fixture -def pandas_setup(csv_path: pathlib.Path) -> Tuple[AbstractDataContext, Batch]: +def pandas_setup(csv_path: pathlib.Path) -> tuple[AbstractDataContext, Batch]: context = gx.get_context(mode="ephemeral") source = context.data_sources.add_pandas(DATASOURCE_NAME) filepath = ( @@ -31,7 +30,7 @@ def pandas_setup(csv_path: pathlib.Path) -> Tuple[AbstractDataContext, Batch]: @pytest.mark.filesystem -def test_batch_validate_expectation(pandas_setup: Tuple[AbstractDataContext, Batch]): +def test_batch_validate_expectation(pandas_setup: tuple[AbstractDataContext, Batch]): _, batch = pandas_setup # Make Expectation @@ -47,7 +46,7 @@ def test_batch_validate_expectation(pandas_setup: Tuple[AbstractDataContext, Bat @pytest.mark.filesystem def test_batch_validate_expectation_suite( - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], ): context, batch = pandas_setup @@ -67,7 +66,7 @@ def test_batch_validate_expectation_suite( @pytest.mark.filesystem def test_batch_validate_expectation_with_expectation_params( - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], ): _, batch = pandas_setup @@ -89,7 +88,7 @@ def test_batch_validate_expectation_with_expectation_params( @pytest.mark.filesystem def test_batch_validate_expectation_suite_with_expectation_params( - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], ): context, batch = pandas_setup @@ -116,7 +115,7 @@ def test_batch_validate_expectation_suite_with_expectation_params( @pytest.mark.filesystem def test_batch_validate_with_updated_expectation( - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], ): _, batch = pandas_setup @@ -136,7 +135,7 @@ def test_batch_validate_with_updated_expectation( @pytest.mark.filesystem def test_batch_validate_expectation_suite_with_updated_expectation( - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], ): context, batch = pandas_setup @@ -170,7 +169,7 @@ def expectation(self) -> Expectation: @pytest.mark.filesystem def test_boolean_validation_result( self, - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], expectation: Expectation, ): _, batch = pandas_setup @@ -182,7 +181,7 @@ def test_boolean_validation_result( @pytest.mark.filesystem def test_summary_validation_result( self, - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], expectation: Expectation, ): _, batch = pandas_setup @@ -194,7 +193,7 @@ def test_summary_validation_result( @pytest.mark.filesystem def test_complete_validation_result( self, - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], expectation: Expectation, ): _, batch = pandas_setup @@ -215,7 +214,7 @@ def suite(self) -> ExpectationSuite: @pytest.mark.filesystem def test_boolean_validation_result( self, - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], suite: ExpectationSuite, ): _, batch = pandas_setup @@ -227,7 +226,7 @@ def test_boolean_validation_result( @pytest.mark.filesystem def test_summary_validation_result( self, - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], suite: ExpectationSuite, ): _, batch = pandas_setup @@ -239,7 +238,7 @@ def test_summary_validation_result( @pytest.mark.filesystem def test_complete_validation_result( self, - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], suite: ExpectationSuite, ): _, batch = pandas_setup @@ -251,7 +250,7 @@ def test_complete_validation_result( @pytest.mark.filesystem def test_batch_validate_expectation_does_not_persist_a_batch_definition( - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], ): context, batch = pandas_setup datasource = context.data_sources.get(DATASOURCE_NAME) @@ -270,7 +269,7 @@ def test_batch_validate_expectation_does_not_persist_a_batch_definition( @pytest.mark.filesystem def test_batch_validate_expectation_suite_does_not_persist_a_batch_definition( - pandas_setup: Tuple[AbstractDataContext, Batch], + pandas_setup: tuple[AbstractDataContext, Batch], ): context, batch = pandas_setup datasource = context.data_sources.get(DATASOURCE_NAME) diff --git a/tests/datasource/fluent/test_config.py b/tests/datasource/fluent/test_config.py index f06631e1e618..0a66a8ffa778 100644 --- a/tests/datasource/fluent/test_config.py +++ b/tests/datasource/fluent/test_config.py @@ -13,7 +13,6 @@ TYPE_CHECKING, Callable, Final, - List, cast, ) @@ -429,7 +428,7 @@ def test_batch_definitions_are_assigned_data_assets( loaded: GxConfig = load_method(input_) assert loaded - batch_definitions: List[BatchDefinition] = [] + batch_definitions: list[BatchDefinition] = [] assert loaded.datasources for datasource in loaded.datasources: for data_asset in datasource.assets: diff --git a/tests/datasource/fluent/test_config_str.py b/tests/datasource/fluent/test_config_str.py index aa0e40339e47..53d36260c29d 100644 --- a/tests/datasource/fluent/test_config_str.py +++ b/tests/datasource/fluent/test_config_str.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Callable, List, Union +from typing import Callable, Union import pytest from pytest import MonkeyPatch @@ -119,7 +119,7 @@ def test_config_nested_substitution_dict( monkeypatch.setenv("MY_ENV_VAR", "success") class MyCollection(FluentBaseModel): - my_classes: List[MyClass] = [] + my_classes: list[MyClass] = [] MyCollection.update_forward_refs(MyClass=MyClass) @@ -143,7 +143,7 @@ def test_config_nested_substitution_dict_raises_error_for_missing_config_var( monkeypatch.setenv("NOT_MY_ENV_VAR", "failure") class MyCollection(FluentBaseModel): - my_classes: List[MyClass] = [] + my_classes: list[MyClass] = [] MyCollection.update_forward_refs(MyClass=MyClass) @@ -180,7 +180,7 @@ def test_serialization_returns_original(monkeypatch: MonkeyPatch, method: str): def test_nested_serialization_returns_original(monkeypatch: MonkeyPatch, method: str): # TODO: fix this class MyCollection(FluentBaseModel): - my_classes: List[MyClass] = [] + my_classes: list[MyClass] = [] MyCollection.update_forward_refs(MyClass=MyClass) diff --git a/tests/datasource/fluent/test_metadatasource.py b/tests/datasource/fluent/test_metadatasource.py index 514288861975..83238757f333 100644 --- a/tests/datasource/fluent/test_metadatasource.py +++ b/tests/datasource/fluent/test_metadatasource.py @@ -1,11 +1,12 @@ from __future__ import annotations +import builtins import copy import inspect import logging import pathlib from pprint import pformat as pf -from typing import TYPE_CHECKING, ClassVar, Dict, Generator, List, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, ClassVar, Generator, Optional, Union import pytest @@ -57,7 +58,7 @@ class DataContext: _context: ClassVar[Optional[DataContext]] = None _config: ClassVar[Optional[GxConfig]] = None # (kilo59) should this live here? - _datasources: Dict[str, Datasource] + _datasources: dict[str, Datasource] root_directory: Union[DirectoryPath, str, None] @classmethod @@ -76,7 +77,7 @@ def get_context( def __init__(self, context_root_dir: Optional[DirectoryPath] = None) -> None: self.root_directory = context_root_dir self._data_sources: DataSourceManager = DataSourceManager(self) # type: ignore[arg-type] - self._datasources: Dict[str, Datasource] = {} + self._datasources: dict[str, Datasource] = {} self.config_provider: _ConfigurationProvider | None = None logger.info(f"Available Factories - {self._data_sources.factories}") logger.debug(f"`type_lookup` mapping ->\n{pf(self._data_sources.type_lookup)}") @@ -120,7 +121,7 @@ def build_batch_request( return BatchRequest("datasource_name", "data_asset_name", options or {}) @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: raise NotImplementedError @override @@ -169,12 +170,12 @@ def test__new__only_registers_expected_number_of_datasources_factories_and_types assert len(empty_sources.type_lookup) == 0 class MyTestDatasource(Datasource): - asset_types: ClassVar[List[Type[DataAsset]]] = [] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [] type: str = "my_test" @property @override - def execution_engine_type(self) -> Type[ExecutionEngine]: + def execution_engine_type(self) -> type[ExecutionEngine]: return DummyExecutionEngine # One for each crud method: add, update, add_or_update, delete @@ -192,12 +193,12 @@ def test__new__registers_sources_factory_method( assert ds_factory_method_initial is None, "Check test cleanup" class MyTestDatasource(Datasource): - asset_types: ClassVar[List[Type[DataAsset]]] = [] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [] type: str = "my_test" @property @override - def execution_engine_type(self) -> Type[ExecutionEngine]: + def execution_engine_type(self) -> type[ExecutionEngine]: return DummyExecutionEngine ds_factory_method_final = getattr(context_sources_cleanup, expected_method_name, None) @@ -212,13 +213,13 @@ def test_registered_sources_factory_method_has_correct_signature( expected_method_name = "add_my_test" class MyTestDatasource(Datasource): - asset_types: ClassVar[List[Type[DataAsset]]] = [] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [] type: str = "my_test" extra_field: str @property @override - def execution_engine_type(self) -> Type[ExecutionEngine]: + def execution_engine_type(self) -> type[ExecutionEngine]: return DummyExecutionEngine ds_factory_method = getattr(context_sources_cleanup, expected_method_name) @@ -249,12 +250,12 @@ class BarAsset(DummyDataAsset): type: str = "bar" class FooBarDatasource(Datasource): - asset_types: ClassVar[List[Type[DataAsset]]] = [FooAsset, BarAsset] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [FooAsset, BarAsset] type: str = "foo_bar" @property @override - def execution_engine_type(self) -> Type[ExecutionEngine]: + def execution_engine_type(self) -> type[ExecutionEngine]: return DummyExecutionEngine print(f" type_lookup ->\n{pf(FooBarDatasource._type_lookup)}\n") @@ -280,7 +281,7 @@ def test_ds_type_field_not_set(self, empty_sources: DataSourceManager): class MissingTypeDatasource(Datasource): @property @override - def execution_engine_type(self) -> Type[ExecutionEngine]: + def execution_engine_type(self) -> type[ExecutionEngine]: return DummyExecutionEngine @override @@ -307,7 +308,7 @@ def test_ds_assets_type_field_not_set(self, empty_sources: DataSourceManager): class MissingTypeAsset(DataAsset): @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: raise NotImplementedError @override @@ -316,11 +317,11 @@ def get_batch(self, batch_request: BatchRequest) -> Batch: class BadAssetDatasource(Datasource): type: str = "valid" - asset_types: ClassVar[List[Type[DataAsset]]] = [MissingTypeAsset] + asset_types: ClassVar[list[builtins.type[DataAsset]]] = [MissingTypeAsset] @property @override - def execution_engine_type(self) -> Type[ExecutionEngine]: + def execution_engine_type(self) -> type[ExecutionEngine]: return DummyExecutionEngine @override @@ -335,7 +336,7 @@ class MissingTestConnectionDatasource(Datasource): @property @override - def execution_engine_type(self) -> Type[ExecutionEngine]: + def execution_engine_type(self) -> type[ExecutionEngine]: return DummyExecutionEngine with pytest.raises(NotImplementedError): @@ -348,7 +349,7 @@ def test_minimal_ds_to_asset_flow(context_sources_cleanup): class SampleAsset(DataAsset): @override - def get_batch_identifiers_list(self, batch_request: BatchRequest) -> List[dict]: + def get_batch_identifiers_list(self, batch_request: BatchRequest) -> list[dict]: raise NotImplementedError @override @@ -372,7 +373,7 @@ class PurpleDatasource(Datasource): @property @override - def execution_engine_type(self) -> Type[ExecutionEngine]: + def execution_engine_type(self) -> type[ExecutionEngine]: return DummyExecutionEngine def test_connection(self): ... # type: ignore[explicit-override] # FIXME @@ -403,7 +404,7 @@ def add_red_asset(self, asset_name: str) -> RedAsset: @pytest.fixture def context_config_data( file_dc_config_dir_init: pathlib.Path, -) -> Tuple[AbstractDataContext, pathlib.Path, pathlib.Path]: +) -> tuple[AbstractDataContext, pathlib.Path, pathlib.Path]: config_file_path = file_dc_config_dir_init / FileDataContext.GX_YML assert config_file_path.exists() is True context = get_gx_context(context_root_dir=file_dc_config_dir_init) @@ -436,8 +437,8 @@ def _remove_ids(config: dict) -> dict: @pytest.fixture def context_with_fluent_datasource( - context_config_data: Tuple[AbstractDataContext, pathlib.Path, pathlib.Path], -) -> Tuple[AbstractDataContext, pathlib.Path, pathlib.Path]: + context_config_data: tuple[AbstractDataContext, pathlib.Path, pathlib.Path], +) -> tuple[AbstractDataContext, pathlib.Path, pathlib.Path]: context, config_file_path, data_dir = context_config_data assert len(context.data_sources.all()) == 0 context.data_sources.add_pandas_filesystem( diff --git a/tests/datasource/fluent/test_pandas_azure_blob_storage_datasource.py b/tests/datasource/fluent/test_pandas_azure_blob_storage_datasource.py index 7ef6e1020928..1993d70d882f 100644 --- a/tests/datasource/fluent/test_pandas_azure_blob_storage_datasource.py +++ b/tests/datasource/fluent/test_pandas_azure_blob_storage_datasource.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, cast +from typing import TYPE_CHECKING, Any, Iterator, cast from unittest import mock import pytest @@ -56,7 +56,7 @@ def get_container_client(self, container: str) -> azure.ContainerClient: def _build_pandas_abs_datasource( - azure_options: Dict[str, Any] | None = None, + azure_options: dict[str, Any] | None = None, ) -> PandasAzureBlobStorageDatasource: azure_client: azure.BlobServiceClient = cast(azure.BlobServiceClient, MockBlobServiceClient()) pandas_abs_datasource = PandasAzureBlobStorageDatasource( @@ -74,7 +74,7 @@ def pandas_abs_datasource() -> PandasAzureBlobStorageDatasource: @pytest.fixture -def object_keys() -> List[str]: +def object_keys() -> list[str]: return [ "yellow_tripdata_sample_2024-01.csv", "yellow_tripdata_sample_2024-02.csv", @@ -199,7 +199,7 @@ def test_construct_pandas_abs_datasource_with_multiple_auth_methods_raises_error def test_add_csv_asset_to_datasource( mock_azure_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], pandas_abs_datasource: PandasAzureBlobStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -215,7 +215,7 @@ def test_add_csv_asset_to_datasource( "great_expectations.datasource.fluent.data_connector.azure_blob_storage_data_connector.list_azure_keys" ) @mock.patch("azure.storage.blob.BlobServiceClient") -def test_construct_csv_asset_directly(mock_azure_client, mock_list_keys, object_keys: List[str]): +def test_construct_csv_asset_directly(mock_azure_client, mock_list_keys, object_keys: list[str]): mock_list_keys.return_value = object_keys asset = CSVAsset( # type: ignore[call-arg] name="csv_asset", @@ -231,7 +231,7 @@ def test_construct_csv_asset_directly(mock_azure_client, mock_list_keys, object_ def test_csv_asset_with_batching_regex_named_parameters( mock_azure_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], pandas_abs_datasource: PandasAzureBlobStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -253,7 +253,7 @@ def test_csv_asset_with_batching_regex_named_parameters( def test_csv_asset_with_non_string_batching_regex_named_parameters( mock_azure_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], pandas_abs_datasource: PandasAzureBlobStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -274,7 +274,7 @@ def test_csv_asset_with_non_string_batching_regex_named_parameters( def test_add_csv_asset_with_recursive_file_discovery_to_datasource( mock_azure_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], pandas_abs_datasource: PandasAzureBlobStorageDatasource, ): """ diff --git a/tests/datasource/fluent/test_pandas_datasource.py b/tests/datasource/fluent/test_pandas_datasource.py index acc60635a2aa..f10d95353763 100644 --- a/tests/datasource/fluent/test_pandas_datasource.py +++ b/tests/datasource/fluent/test_pandas_datasource.py @@ -6,7 +6,7 @@ import pathlib import uuid from pprint import pformat as pf -from typing import TYPE_CHECKING, Any, Callable, Type +from typing import TYPE_CHECKING, Any, Callable import pytest from pytest import MonkeyPatch, param @@ -141,7 +141,7 @@ def test_data_asset_defined_for_io_read_method(self, method_name: str): assert type_name in asset_class_names @pytest.mark.parametrize("asset_class", _DYNAMIC_ASSET_TYPES) - def test_add_asset_method_exists_and_is_functional(self, asset_class: Type[_PandasDataAsset]): + def test_add_asset_method_exists_and_is_functional(self, asset_class: type[_PandasDataAsset]): type_name: str = _get_field_details(asset_class, "type").default_value method_name: str = f"add_{type_name}_asset" @@ -171,7 +171,7 @@ def test_add_asset_method_exists_and_is_functional(self, asset_class: Type[_Pand assert exc_info.value.model == asset_class @pytest.mark.parametrize("asset_class", _DYNAMIC_ASSET_TYPES) - def test_add_asset_method_signature(self, asset_class: Type[_PandasDataAsset]): + def test_add_asset_method_signature(self, asset_class: type[_PandasDataAsset]): type_name: str = _get_field_details(asset_class, "type").default_value method_name: str = f"add_{type_name}_asset" @@ -200,7 +200,7 @@ def test_add_asset_method_signature(self, asset_class: Type[_PandasDataAsset]): print("✅") @pytest.mark.parametrize("asset_class", _DYNAMIC_ASSET_TYPES) - def test_minimal_validation(self, asset_class: Type[_PandasDataAsset]): + def test_minimal_validation(self, asset_class: type[_PandasDataAsset]): """ These parametrized tests ensures that every `PandasDatasource` asset model does some minimal validation, and doesn't accept arbitrary keyword arguments. @@ -238,7 +238,7 @@ def test_minimal_validation(self, asset_class: Type[_PandasDataAsset]): def test_data_asset_defaults( self, csv_path: pathlib.Path, - asset_model: Type[_PandasDataAsset], + asset_model: type[_PandasDataAsset], extra_kwargs: dict, ): """ diff --git a/tests/datasource/fluent/test_pandas_filesystem_datasource.py b/tests/datasource/fluent/test_pandas_filesystem_datasource.py index f885e35b6a5e..2ec3f272246b 100644 --- a/tests/datasource/fluent/test_pandas_filesystem_datasource.py +++ b/tests/datasource/fluent/test_pandas_filesystem_datasource.py @@ -7,7 +7,7 @@ import re from dataclasses import dataclass from pprint import pformat as pf -from typing import TYPE_CHECKING, Any, Optional, Type +from typing import TYPE_CHECKING, Any, Optional import pytest from pytest import MonkeyPatch, param @@ -145,7 +145,7 @@ def test_data_asset_defined_for_io_read_method(self, method_name: str): assert type_name in asset_class_names @pytest.mark.parametrize("asset_class", PandasFilesystemDatasource.asset_types) - def test_add_asset_method_exists_and_is_functional(self, asset_class: Type[PathDataAsset]): + def test_add_asset_method_exists_and_is_functional(self, asset_class: type[PathDataAsset]): type_name: str = _get_field_details(asset_class, "type").default_value method_name: str = f"add_{type_name}_asset" @@ -169,7 +169,7 @@ def test_add_asset_method_exists_and_is_functional(self, asset_class: Type[PathD assert exc_info.value.model == asset_class @pytest.mark.parametrize("asset_class", PandasFilesystemDatasource.asset_types) - def test_add_asset_method_signature(self, asset_class: Type[PathDataAsset]): + def test_add_asset_method_signature(self, asset_class: type[PathDataAsset]): type_name: str = _get_field_details(asset_class, "type").default_value method_name: str = f"add_{type_name}_asset" @@ -207,7 +207,7 @@ def test_add_asset_method_signature(self, asset_class: Type[PathDataAsset]): ) def test_data_asset_defaults( self, - asset_model: Type[PathDataAsset], + asset_model: type[PathDataAsset], extra_kwargs: dict, ): """ @@ -324,7 +324,7 @@ def test_invalid_connect_options( def test_invalid_connect_options_value( pandas_filesystem_datasource: PandasFilesystemDatasource, glob_directive, - expected_error: Type[Exception], + expected_error: type[Exception], ): with pytest.raises(expected_error) as exc_info: pandas_filesystem_datasource.add_csv_asset( diff --git a/tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py b/tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py index b783e4f4bc53..2e387e5e777f 100644 --- a/tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py +++ b/tests/datasource/fluent/test_pandas_google_cloud_storage_datasource.py @@ -2,7 +2,7 @@ import logging import os -from typing import Any, Dict, Iterator, List, cast +from typing import Any, Iterator, cast from unittest import mock import pytest @@ -47,7 +47,7 @@ def list_blobs( def _build_pandas_gcs_datasource( - gcs_options: Dict[str, Any] | None = None, + gcs_options: dict[str, Any] | None = None, ) -> PandasGoogleCloudStorageDatasource: gcs_client: google.Client = cast(google.Client, MockGCSClient()) pandas_gcs_datasource = PandasGoogleCloudStorageDatasource( @@ -66,7 +66,7 @@ def pandas_gcs_datasource() -> PandasGoogleCloudStorageDatasource: @pytest.fixture -def object_keys() -> List[str]: +def object_keys() -> list[str]: return [ "alex_20200809_1000.csv", "eugene_20200809_1500.csv", @@ -147,7 +147,7 @@ def test_construct_pandas_gcs_datasource_with_info_in_gcs_options( def test_add_csv_asset_to_datasource( mock_gcs_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], pandas_gcs_datasource: PandasGoogleCloudStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -162,7 +162,7 @@ def test_add_csv_asset_to_datasource( "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") -def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_keys: List[str]): +def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_keys: list[str]): mock_list_keys.return_value = object_keys asset = CSVAsset( # type: ignore[call-arg] name="csv_asset", @@ -178,7 +178,7 @@ def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_ke def test_csv_asset_with_batching_regex_named_parameters( mock_gcs_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], pandas_gcs_datasource: PandasGoogleCloudStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -200,7 +200,7 @@ def test_csv_asset_with_batching_regex_named_parameters( def test_csv_asset_with_non_string_batching_regex_named_parameters( mock_gcs_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], pandas_gcs_datasource: PandasGoogleCloudStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -220,7 +220,7 @@ def test_csv_asset_with_non_string_batching_regex_named_parameters( def test_add_csv_asset_with_recursive_file_discovery_to_datasource( mock_gcs_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], pandas_gcs_datasource: PandasGoogleCloudStorageDatasource, ): """ diff --git a/tests/datasource/fluent/test_pandas_s3_datasource.py b/tests/datasource/fluent/test_pandas_s3_datasource.py index 73ffd4b92f2e..15dd74401c01 100644 --- a/tests/datasource/fluent/test_pandas_s3_datasource.py +++ b/tests/datasource/fluent/test_pandas_s3_datasource.py @@ -2,7 +2,7 @@ import logging from pprint import pformat as pf -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING import pandas as pd import pytest @@ -52,7 +52,7 @@ def s3_bucket(s3_mock: BaseClient, aws_s3_bucket_name: str) -> str: def pandas_s3_datasource(empty_data_context, s3_mock, s3_bucket: str) -> PandasS3Datasource: test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "yellow_tripdata_sample_2024-01.csv", "yellow_tripdata_sample_2024-02.csv", "yellow_tripdata_sample_2024-03.csv", diff --git a/tests/datasource/fluent/test_postgres_datasource.py b/tests/datasource/fluent/test_postgres_datasource.py index 6dcb9dd852f9..4db7a4603449 100644 --- a/tests/datasource/fluent/test_postgres_datasource.py +++ b/tests/datasource/fluent/test_postgres_datasource.py @@ -8,7 +8,6 @@ Generator, Literal, Optional, - Tuple, ) from unittest.mock import ANY @@ -189,7 +188,7 @@ def create_and_add_table_asset_without_testing_connection( source: PostgresDatasource, name: str, table_name: str, -) -> Tuple[PostgresDatasource, TableAsset]: +) -> tuple[PostgresDatasource, TableAsset]: table_asset = TableAsset( name=name, table_name=table_name, diff --git a/tests/datasource/fluent/test_schemas.py b/tests/datasource/fluent/test_schemas.py index 4470a4f4ec02..fdbeae23c532 100644 --- a/tests/datasource/fluent/test_schemas.py +++ b/tests/datasource/fluent/test_schemas.py @@ -4,7 +4,7 @@ import pathlib import sys from pprint import pformat as pf -from typing import Any, Generator, Type +from typing import Any, Generator import pandas import pytest @@ -29,9 +29,9 @@ def min_supported_python() -> Version: def _models_and_schema_dirs() -> ( - Generator[tuple[Type[Datasource | DataAsset], pathlib.Path, str], None, None] + Generator[tuple[type[Datasource | DataAsset], pathlib.Path, str], None, None] ): - datasource: Type[Datasource] = Datasource + datasource: type[Datasource] = Datasource ds_type_name: str = "" yield Datasource, _SCHEMAS_DIR, Datasource.__name__ @@ -60,7 +60,7 @@ def _models_and_schema_dirs() -> ( [pytest.param(t[0], t[1], id=t[2]) for t in _models_and_schema_dirs()], ) def test_vcs_schemas_match( # noqa: C901 - fluent_ds_or_asset_model: Type[Datasource | DataAsset], schema_dir: pathlib.Path + fluent_ds_or_asset_model: type[Datasource | DataAsset], schema_dir: pathlib.Path ): """ Test that json schemas for each DataSource and DataAsset match the current schema diff --git a/tests/datasource/fluent/test_spark_azure_blob_storage_datasource.py b/tests/datasource/fluent/test_spark_azure_blob_storage_datasource.py index b43f7471c997..0162ef4b73f0 100644 --- a/tests/datasource/fluent/test_spark_azure_blob_storage_datasource.py +++ b/tests/datasource/fluent/test_spark_azure_blob_storage_datasource.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any, Dict, Iterator, List, cast +from typing import Any, Iterator, cast from unittest import mock import pytest @@ -46,7 +46,7 @@ def get_container_client(self, container: str) -> azure.ContainerClient: def _build_spark_abs_datasource( - azure_options: Dict[str, Any] | None = None, + azure_options: dict[str, Any] | None = None, ) -> SparkAzureBlobStorageDatasource: azure_client: azure.BlobServiceClient = cast(azure.BlobServiceClient, MockBlobServiceClient()) spark_abs_datasource = SparkAzureBlobStorageDatasource( @@ -64,7 +64,7 @@ def spark_abs_datasource() -> SparkAzureBlobStorageDatasource: @pytest.fixture -def object_keys() -> List[str]: +def object_keys() -> list[str]: return [ "alex_20200809_1000.csv", "eugene_20200809_1500.csv", @@ -85,7 +85,7 @@ def object_keys() -> List[str]: ) def csv_asset( mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_abs_datasource: SparkAzureBlobStorageDatasource, ) -> PathDataAsset: mock_list_keys.return_value = object_keys @@ -180,7 +180,7 @@ def test_construct_spark_abs_datasource_with_multiple_auth_methods_raises_error( def test_add_csv_asset_to_datasource( mock_azure_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_abs_datasource: SparkAzureBlobStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -199,7 +199,7 @@ def test_add_csv_asset_to_datasource( "great_expectations.datasource.fluent.data_connector.azure_blob_storage_data_connector.list_azure_keys" ) @mock.patch("azure.storage.blob.BlobServiceClient") -def test_construct_csv_asset_directly(mock_azure_client, mock_list_keys, object_keys: List[str]): +def test_construct_csv_asset_directly(mock_azure_client, mock_list_keys, object_keys: list[str]): mock_list_keys.return_value = object_keys asset = CSVAsset( # type: ignore[call-arg] # missing args name="csv_asset", @@ -215,7 +215,7 @@ def test_construct_csv_asset_directly(mock_azure_client, mock_list_keys, object_ def test_csv_asset_with_batching_regex_named_parameters( mock_azure_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_abs_datasource: SparkAzureBlobStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -237,7 +237,7 @@ def test_csv_asset_with_batching_regex_named_parameters( def test_csv_asset_with_non_string_batching_regex_named_parameters( mock_azure_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_abs_datasource: SparkAzureBlobStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -258,7 +258,7 @@ def test_csv_asset_with_non_string_batching_regex_named_parameters( def test_add_csv_asset_with_recursive_file_discovery_to_datasource( mock_azure_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_abs_datasource: SparkAzureBlobStorageDatasource, ): """ diff --git a/tests/datasource/fluent/test_spark_filesystem_datasource.py b/tests/datasource/fluent/test_spark_filesystem_datasource.py index 1f1ddb0c591b..29afbeb03876 100644 --- a/tests/datasource/fluent/test_spark_filesystem_datasource.py +++ b/tests/datasource/fluent/test_spark_filesystem_datasource.py @@ -5,7 +5,7 @@ import pathlib import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Dict, List, Tuple, cast +from typing import TYPE_CHECKING, cast import pytest @@ -85,7 +85,7 @@ def spark_filesystem_datasource(empty_data_context, test_backends) -> SparkFiles # Verify test directory has files we expect years = ["2018", "2019", "2020"] file_name: PathStr - all_files: List[str] = [ + all_files: list[str] = [ file_name.stem for file_name in list(pathlib.Path(spark_filesystem_datasource.base_directory).iterdir()) ] @@ -838,7 +838,7 @@ def test_get_batch_list_from_partially_specified_batch_request( ): # Verify test directory has files that don't match what we will query for file_name: PathStr - all_files: List[str] = [ + all_files: list[str] = [ file_name.stem for file_name in list(pathlib.Path(spark_filesystem_datasource.base_directory).iterdir()) ] @@ -1094,29 +1094,29 @@ def daily_partitioner(): @pytest.fixture -def daily_batch_parameters_and_expected_result() -> Tuple[Dict[str, int], int]: +def daily_batch_parameters_and_expected_result() -> tuple[dict[str, int], int]: batch_parameters = {"year": 2018, "month": 1, "day": 11} expected_result = 3 return batch_parameters, expected_result @pytest.fixture -def monthly_batch_parameters_and_expected_result() -> Tuple[Dict[str, int], int]: +def monthly_batch_parameters_and_expected_result() -> tuple[dict[str, int], int]: batch_parameters = {"year": 2018, "month": 1} expected_result = 10 return batch_parameters, expected_result @pytest.fixture -def yearly_batch_parameters_and_expected_result() -> Tuple[Dict[str, int], int]: +def yearly_batch_parameters_and_expected_result() -> tuple[dict[str, int], int]: batch_parameters = {"year": 2018} expected_result = 120 return batch_parameters, expected_result @pytest.fixture -def whole_directory_batch_parameters_and_expected_result() -> Tuple[Dict[str, int], int]: - batch_parameters: Dict[str, int] = {} +def whole_directory_batch_parameters_and_expected_result() -> tuple[dict[str, int], int]: + batch_parameters: dict[str, int] = {} expected_result = 360 return batch_parameters, expected_result @@ -1152,7 +1152,7 @@ def test_monthly_batch_definition_workflow( def test_yearly_batch_definition_workflow( self, directory_asset: DirectoryCSVAsset, - yearly_batch_parameters_and_expected_result: Tuple[Dict[str, int], int], + yearly_batch_parameters_and_expected_result: tuple[dict[str, int], int], ): batch_parameters = {"year": 2018} expected_result = 120 @@ -1166,9 +1166,9 @@ def test_yearly_batch_definition_workflow( def test_whole_table_batch_definition_workflow( self, directory_asset: DirectoryCSVAsset, - whole_directory_batch_parameters_and_expected_result: Tuple[Dict[str, int], int], + whole_directory_batch_parameters_and_expected_result: tuple[dict[str, int], int], ): - batch_parameters: Dict[str, int] = {} + batch_parameters: dict[str, int] = {} expected_result = 360 batch_def = directory_asset.add_batch_definition_whole_directory(name="whole directory") batch = batch_def.get_batch(batch_parameters=batch_parameters) @@ -1207,7 +1207,7 @@ def test_get_batch_parameters_keys_with_partitioner( self, directory_asset: DirectoryCSVAsset, partitioner: ColumnPartitioner, - expected_keys: Tuple[str, ...], + expected_keys: tuple[str, ...], ): assert directory_asset.get_batch_parameters_keys(partitioner=partitioner) == expected_keys diff --git a/tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py b/tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py index 1928fea2cc22..526ecb3a4240 100644 --- a/tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py +++ b/tests/datasource/fluent/test_spark_google_cloud_storage_datasource.py @@ -3,7 +3,7 @@ import logging import os import re -from typing import Any, Dict, Iterator, List, cast +from typing import Any, Iterator, cast from unittest import mock import pytest @@ -43,7 +43,7 @@ def list_blobs( def _build_spark_gcs_datasource( - gcs_options: Dict[str, Any] | None = None, + gcs_options: dict[str, Any] | None = None, ) -> SparkGoogleCloudStorageDatasource: gcs_client: google.Client = cast(google.Client, MockGCSClient()) spark_gcs_datasource = SparkGoogleCloudStorageDatasource( @@ -62,7 +62,7 @@ def spark_gcs_datasource() -> SparkGoogleCloudStorageDatasource: @pytest.fixture -def object_keys() -> List[str]: +def object_keys() -> list[str]: return [ "alex_20200809_1000.csv", "eugene_20200809_1500.csv", @@ -83,7 +83,7 @@ def object_keys() -> List[str]: ) def csv_asset( mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_gcs_datasource: SparkGoogleCloudStorageDatasource, ) -> PathDataAsset: mock_list_keys.return_value = object_keys @@ -162,7 +162,7 @@ def test_construct_spark_gcs_datasource_with_info_in_gcs_options( def test_add_csv_asset_to_datasource( mock_gcs_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_gcs_datasource: SparkGoogleCloudStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -180,7 +180,7 @@ def test_add_csv_asset_to_datasource( "great_expectations.datasource.fluent.data_asset.data_connector.google_cloud_storage_data_connector.list_gcs_keys" ) @mock.patch("google.cloud.storage.Client") -def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_keys: List[str]): +def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_keys: list[str]): mock_list_keys.return_value = object_keys asset = CSVAsset( # type: ignore[call-arg] # missing args name="csv_asset", @@ -196,7 +196,7 @@ def test_construct_csv_asset_directly(mock_gcs_client, mock_list_keys, object_ke def test_csv_asset_with_batching_regex_named_parameters( mock_gcs_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_gcs_datasource: SparkGoogleCloudStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -217,7 +217,7 @@ def test_csv_asset_with_batching_regex_named_parameters( def test_csv_asset_with_non_string_batching_regex_named_parameters( mock_gcs_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_gcs_datasource: SparkGoogleCloudStorageDatasource, ): mock_list_keys.return_value = object_keys @@ -241,7 +241,7 @@ def test_csv_asset_with_non_string_batching_regex_named_parameters( def test_add_csv_asset_with_recursive_file_discovery_to_datasource( mock_gcs_client, mock_list_keys, - object_keys: List[str], + object_keys: list[str], spark_gcs_datasource: SparkGoogleCloudStorageDatasource, ): """ diff --git a/tests/datasource/fluent/test_spark_s3_datasource.py b/tests/datasource/fluent/test_spark_s3_datasource.py index c302587d9942..3c301e475350 100644 --- a/tests/datasource/fluent/test_spark_s3_datasource.py +++ b/tests/datasource/fluent/test_spark_s3_datasource.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING import pandas as pd import pytest @@ -32,7 +32,7 @@ def s3_bucket(s3_mock: BaseClient, aws_s3_bucket_name: str) -> str: def spark_s3_datasource(s3_mock, s3_bucket: str) -> SparkS3Datasource: test_df: pd.DataFrame = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - keys: List[str] = [ + keys: list[str] = [ "yellow_tripdata_sample_2024-01.csv", "yellow_tripdata_sample_2024-02.csv", "yellow_tripdata_sample_2024-03.csv", diff --git a/tests/datasource/fluent/test_type_lookup.py b/tests/datasource/fluent/test_type_lookup.py index 4d1086029201..461851b58b93 100644 --- a/tests/datasource/fluent/test_type_lookup.py +++ b/tests/datasource/fluent/test_type_lookup.py @@ -1,5 +1,5 @@ from pprint import pprint as pp -from typing import Iterable, Mapping, Optional, Tuple +from typing import Iterable, Mapping, Optional import pytest @@ -64,7 +64,7 @@ def test_ok_to_add_identical_key_value(): (TypeLookup(my_list=list), ("my_list", dict)), ], ) -def test_no_key_or_value_overwrites(initial: TypeLookup, kv_pair: Tuple[ValidTypes, ValidTypes]): +def test_no_key_or_value_overwrites(initial: TypeLookup, kv_pair: tuple[ValidTypes, ValidTypes]): key, value = kv_pair with pytest.raises(TypeLookupError, match=r"`.*` already set"): initial[key] = value diff --git a/tests/execution_engine/conftest.py b/tests/execution_engine/conftest.py index af48d47f45c2..707d3db3c87f 100644 --- a/tests/execution_engine/conftest.py +++ b/tests/execution_engine/conftest.py @@ -2,7 +2,6 @@ import os import random from pathlib import Path -from typing import List import pandas as pd import pytest @@ -47,7 +46,7 @@ def test_df_small_csv_compressed(test_df_small, tmpdir) -> bytes: @pytest.fixture def test_s3_files_parquet(tmpdir, s3, s3_bucket, test_df_small, test_df_small_csv): - keys: List[str] = [ + keys: list[str] = [ "path/A-100.csv", "path/A-101.csv", "directory/B-1.parquet", @@ -68,7 +67,7 @@ def test_s3_files_parquet(tmpdir, s3, s3_bucket, test_df_small, test_df_small_cs @pytest.fixture def test_s3_files_compressed(s3, s3_bucket, test_df_small_csv_compressed): - keys: List[str] = [ + keys: list[str] = [ "path/A-100.csv.gz", "path/A-101.csv.gz", "directory/B-1.csv.gz", @@ -87,7 +86,7 @@ def test_s3_files_compressed(s3, s3_bucket, test_df_small_csv_compressed): @pytest.fixture def azure_batch_spec() -> AzureBatchSpec: container = "test_container" - keys: List[str] = [ + keys: list[str] = [ "path/A-100.csv", "path/A-101.csv", "directory/B-1.csv", @@ -111,7 +110,7 @@ def azure_batch_spec() -> AzureBatchSpec: @pytest.fixture def gcs_batch_spec() -> GCSBatchSpec: bucket = "test_bucket" - keys: List[str] = [ + keys: list[str] = [ "path/A-100.csv", "path/A-101.csv", "directory/B-1.csv", @@ -134,7 +133,7 @@ def gcs_batch_spec() -> GCSBatchSpec: def test_sparkdf(spark_session) -> pyspark.DataFrame: def generate_ascending_list_of_datetimes( n, start_date=datetime.date(2020, 1, 1), end_date=datetime.date(2020, 12, 31) - ) -> List[datetime.datetime]: + ) -> list[datetime.datetime]: start_time = datetime.datetime(start_date.year, start_date.month, start_date.day) # noqa: DTZ001 seconds_between_dates = (end_date - start_date).total_seconds() # noinspection PyUnusedLocal @@ -147,18 +146,18 @@ def generate_ascending_list_of_datetimes( k: int = 120 random.seed(1) - timestamp_list: List[datetime.datetime] = generate_ascending_list_of_datetimes( + timestamp_list: list[datetime.datetime] = generate_ascending_list_of_datetimes( n=k, end_date=datetime.date(2020, 1, 31) ) - date_list: List[datetime.date] = [ + date_list: list[datetime.date] = [ datetime.date(ts.year, ts.month, ts.day) for ts in timestamp_list ] # noinspection PyUnusedLocal - batch_ids: List[int] = [random.randint(0, 10) for i in range(k)] + batch_ids: list[int] = [random.randint(0, 10) for i in range(k)] batch_ids.sort() # noinspection PyUnusedLocal - session_ids: List[int] = [random.randint(2, 60) for i in range(k)] + session_ids: list[int] = [random.randint(2, 60) for i in range(k)] session_ids = [i - random.randint(0, 2) for i in session_ids] session_ids.sort() diff --git a/tests/execution_engine/partition_and_sample/partition_and_sample_test_cases.py b/tests/execution_engine/partition_and_sample/partition_and_sample_test_cases.py index eb0c48f4ce28..6bd4303cfea5 100644 --- a/tests/execution_engine/partition_and_sample/partition_and_sample_test_cases.py +++ b/tests/execution_engine/partition_and_sample/partition_and_sample_test_cases.py @@ -1,5 +1,4 @@ import datetime -from typing import List import pytest @@ -8,7 +7,7 @@ DatePart, ) -SINGLE_DATE_PART_BATCH_IDENTIFIERS: List[pytest.param] = [ +SINGLE_DATE_PART_BATCH_IDENTIFIERS: list[pytest.param] = [ pytest.param({"month": 10}, id="month_dict"), pytest.param("10-31-2018", id="dateutil parseable date string"), pytest.param( @@ -37,7 +36,7 @@ ), ] -SINGLE_DATE_PART_DATE_PARTS: List[pytest.param] = [ +SINGLE_DATE_PART_DATE_PARTS: list[pytest.param] = [ pytest.param( [DatePart.MONTH], id="month_with_DatePart", @@ -71,7 +70,7 @@ ] -MULTIPLE_DATE_PART_BATCH_IDENTIFIERS: List[pytest.param] = [ +MULTIPLE_DATE_PART_BATCH_IDENTIFIERS: list[pytest.param] = [ pytest.param({"year": 2018, "month": 10}, id="year_and_month_dict"), pytest.param("10-31-2018", id="dateutil parseable date string"), pytest.param( @@ -105,7 +104,7 @@ ), ] -MULTIPLE_DATE_PART_DATE_PARTS: List[pytest.param] = [ +MULTIPLE_DATE_PART_DATE_PARTS: list[pytest.param] = [ pytest.param( [DatePart.YEAR, DatePart.MONTH], id="year_month_with_DatePart", diff --git a/tests/execution_engine/partition_and_sample/test_pandas_execution_engine_partitioning.py b/tests/execution_engine/partition_and_sample/test_pandas_execution_engine_partitioning.py index 7e14a93851c1..f85a4e16d08e 100644 --- a/tests/execution_engine/partition_and_sample/test_pandas_execution_engine_partitioning.py +++ b/tests/execution_engine/partition_and_sample/test_pandas_execution_engine_partitioning.py @@ -1,6 +1,5 @@ import datetime import os -from typing import List from unittest import mock import pandas as pd @@ -69,7 +68,7 @@ def simple_multi_year_pandas_df(): @pytest.fixture def test_s3_files(s3, s3_bucket, test_df_small_csv): - keys: List[str] = [ + keys: list[str] = [ "path/A-100.csv", "path/A-101.csv", "directory/B-1.csv", @@ -239,7 +238,7 @@ def test_partition_on_date_parts_multiple_date_parts( def test_named_date_part_methods( mock_partition_on_date_parts: mock.MagicMock, # noqa: TID251 partitioner_method_name: str, - called_with_date_parts: List[DatePart], + called_with_date_parts: list[DatePart], simple_multi_year_pandas_df: pd.DataFrame, ): """Test that a partially pre-filled version of partition_on_date_parts() was called with the appropriate params. diff --git a/tests/execution_engine/partition_and_sample/test_sparkdf_execution_engine_partitioning.py b/tests/execution_engine/partition_and_sample/test_sparkdf_execution_engine_partitioning.py index 4227436f5c9f..ced392ff9f87 100644 --- a/tests/execution_engine/partition_and_sample/test_sparkdf_execution_engine_partitioning.py +++ b/tests/execution_engine/partition_and_sample/test_sparkdf_execution_engine_partitioning.py @@ -1,6 +1,5 @@ import datetime import os -from typing import List, Tuple from unittest import mock import numpy as np @@ -50,7 +49,7 @@ @pytest.fixture def simple_multi_year_spark_df(spark_session): - spark_df_data: List[Tuple] = [ + spark_df_data: list[tuple] = [ ("2018-01-01 12:00:00.000",), ("2018-10-02 12:00:00.000",), ("2019-01-01 12:00:00.000",), @@ -208,7 +207,7 @@ def test_partition_on_date_parts_multiple_date_parts( def test_named_date_part_methods( mock_partition_on_date_parts: mock.MagicMock, # noqa: TID251 partitioner_method_name: str, - called_with_date_parts: List[DatePart], + called_with_date_parts: list[DatePart], simple_multi_year_spark_df: pyspark.DataFrame, ): """Test that a partially pre-filled version of partition_on_date_parts() was called with the appropriate params. diff --git a/tests/execution_engine/partition_and_sample/test_sqlalchemy_execution_engine_partitioning.py b/tests/execution_engine/partition_and_sample/test_sqlalchemy_execution_engine_partitioning.py index b8c076d2918a..de7956e764b1 100644 --- a/tests/execution_engine/partition_and_sample/test_sqlalchemy_execution_engine_partitioning.py +++ b/tests/execution_engine/partition_and_sample/test_sqlalchemy_execution_engine_partitioning.py @@ -2,7 +2,6 @@ import datetime import os -from typing import List from unittest import mock import pandas as pd @@ -77,7 +76,7 @@ def test_named_date_part_methods( mock_partition_on_date_parts: mock.MagicMock, # noqa: TID251 partitioner_method_name: str, - called_with_date_parts: List[DatePart], + called_with_date_parts: list[DatePart], ): """Test that a partially pre-filled version of partition_on_date_parts() was called with the appropriate params. For example, partition_on_year. @@ -450,7 +449,7 @@ def test_sqlite_partition( """ engine: SqlAlchemyExecutionEngine = build_sa_execution_engine(taxi_test_cases.test_df, sa) - test_cases: List[TaxiPartitioningTestCase] = taxi_test_cases.test_cases() + test_cases: list[TaxiPartitioningTestCase] = taxi_test_cases.test_cases() test_case: TaxiPartitioningTestCase batch_spec: SqlAlchemyDatasourceBatchSpec for test_case in test_cases: @@ -533,7 +532,7 @@ def test_sqlite_partition_on_year(sa, in_memory_sqlite_taxi_ten_trips_per_month_ .fetchall() ) - row_dates: List[datetime.datetime] = [parse(row["pickup_datetime"]) for row in rows] + row_dates: list[datetime.datetime] = [parse(row["pickup_datetime"]) for row in rows] for row_date in row_dates: assert row_date.month >= 1 assert row_date.month <= 12 @@ -577,7 +576,7 @@ def test_sqlite_partition_and_sample_using_limit( .fetchall() ) - row_dates: List[datetime.datetime] = [parse(row["pickup_datetime"]) for row in rows] + row_dates: list[datetime.datetime] = [parse(row["pickup_datetime"]) for row in rows] for row_date in row_dates: assert row_date.month == 1 assert row_date.year == 2018 diff --git a/tests/execution_engine/partition_and_sample/test_sqlalchemy_execution_engine_sampling.py b/tests/execution_engine/partition_and_sample/test_sqlalchemy_execution_engine_sampling.py index 8867673715ba..d76d0603c13b 100644 --- a/tests/execution_engine/partition_and_sample/test_sqlalchemy_execution_engine_sampling.py +++ b/tests/execution_engine/partition_and_sample/test_sqlalchemy_execution_engine_sampling.py @@ -2,7 +2,6 @@ import datetime import os -from typing import List import pandas as pd import pytest @@ -284,7 +283,7 @@ def test_sqlite_sample_using_limit(sa): .fetchall() ) - row_dates: List[datetime.datetime] = [parse(row["pickup_datetime"]) for row in rows] + row_dates: list[datetime.datetime] = [parse(row["pickup_datetime"]) for row in rows] for row_date in row_dates: assert row_date.month == 1 assert row_date.year == 2018 @@ -300,8 +299,8 @@ def test_sample_using_random(sqlite_view_engine, test_df): batch_spec: SqlAlchemyDatasourceBatchSpec batch_data: SqlAlchemyBatchData num_rows: int - rows_0: List[tuple] - rows_1: List[tuple] + rows_0: list[tuple] + rows_1: list[tuple] # First, make sure that degenerative case never passes. @@ -322,7 +321,7 @@ def test_sample_using_random(sqlite_view_engine, test_df): ).scalar() assert num_rows == round(p * test_df_0.shape[0]) - rows_0: List[tuple] = batch_data.execution_engine.execute_query( + rows_0: list[tuple] = batch_data.execution_engine.execute_query( sqlalchemy.select(sqlalchemy.text("*")).select_from(batch_data.selectable) ).fetchall() @@ -332,7 +331,7 @@ def test_sample_using_random(sqlite_view_engine, test_df): ).scalar() assert num_rows == round(p * test_df_0.shape[0]) - rows_1: List[tuple] = batch_data.execution_engine.execute_query( + rows_1: list[tuple] = batch_data.execution_engine.execute_query( sqlalchemy.select(sqlalchemy.text("*")).select_from(batch_data.selectable) ).fetchall() diff --git a/tests/execution_engine/test_execution_engine.py b/tests/execution_engine/test_execution_engine.py index 087c850585b4..fd7879398f4d 100644 --- a/tests/execution_engine/test_execution_engine.py +++ b/tests/execution_engine/test_execution_engine.py @@ -1,7 +1,5 @@ from __future__ import annotations -from typing import Dict, Tuple - import pandas as pd import pytest @@ -33,7 +31,7 @@ def test_execution_engine(): """ class TestExecutionEngine(ExecutionEngine): - def get_batch_data_and_markers(self, batch_spec) -> Tuple[BatchData, BatchMarkers]: # type: ignore[explicit-override] # FIXME + def get_batch_data_and_markers(self, batch_spec) -> tuple[BatchData, BatchMarkers]: # type: ignore[explicit-override] # FIXME raise NotImplementedError return TestExecutionEngine() @@ -172,10 +170,10 @@ def test_resolve_metrics_with_aggregates_and_column_map(): df = pd.DataFrame({"a": [1, 2, 3, None]}) engine = PandasExecutionEngine(batch_data_dict={"my_id": df}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) @@ -251,10 +249,10 @@ def test_resolve_metrics_with_extraneous_value_key(): df = pd.DataFrame({"a": [1, 2, 3, None]}) engine = PandasExecutionEngine(batch_data_dict={"my_id": df}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) diff --git a/tests/execution_engine/test_pandas_execution_engine.py b/tests/execution_engine/test_pandas_execution_engine.py index 6ed81ddc6234..d63056d27c06 100644 --- a/tests/execution_engine/test_pandas_execution_engine.py +++ b/tests/execution_engine/test_pandas_execution_engine.py @@ -1,5 +1,4 @@ import os -from typing import Dict, Tuple from unittest import mock import pandas as pd @@ -368,10 +367,10 @@ def test_resolve_metric_bundle(): # Building engine and configurations in attempt to resolve metrics engine = PandasExecutionEngine(batch_data_dict={"made-up-id": df}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) diff --git a/tests/execution_engine/test_sparkdf_execution_engine.py b/tests/execution_engine/test_sparkdf_execution_engine.py index 9776b380acc4..1235da8efe8a 100644 --- a/tests/execution_engine/test_sparkdf_execution_engine.py +++ b/tests/execution_engine/test_sparkdf_execution_engine.py @@ -1,6 +1,5 @@ import datetime import logging -from typing import Dict, Tuple import numpy as np import pandas as pd @@ -631,10 +630,10 @@ def test_sparkdf_batch_aggregate_metrics(caplog, spark_session): batch_id="1234", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) @@ -1046,10 +1045,10 @@ def test_resolve_metric_bundle_with_compute_domain_kwargs_json_serialization( batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) diff --git a/tests/execution_engine/test_sqlalchemy_execution_engine.py b/tests/execution_engine/test_sqlalchemy_execution_engine.py index 0d5869954be3..50cb2029eb64 100644 --- a/tests/execution_engine/test_sqlalchemy_execution_engine.py +++ b/tests/execution_engine/test_sqlalchemy_execution_engine.py @@ -1,6 +1,6 @@ import logging import os -from typing import Dict, Tuple, cast +from typing import cast import pandas as pd import pytest @@ -260,10 +260,10 @@ def test_sa_batch_aggregate_metrics(caplog, sa): pd.DataFrame({"a": [1, 2, 1, 2, 3, 3], "b": [4, 4, 4, 4, 4, 4]}), sa ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=execution_engine) metrics.update(results) @@ -966,10 +966,10 @@ def test_resolve_metric_bundle_with_compute_domain_kwargs_json_serialization(sa) batch_id="1234", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=execution_engine) metrics.update(results) @@ -1049,10 +1049,10 @@ def validate_tmp_tables(execution_engine): pd.DataFrame({"a": [1, 2, 1, 2, 3, 3], "b": [4, 4, 4, 4, 4, 4]}), sa ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=execution_engine) metrics.update(results) diff --git a/tests/expectations/metrics/column_map_metrics/test_column_map_condition_auxillary_methods.py b/tests/expectations/metrics/column_map_metrics/test_column_map_condition_auxillary_methods.py index 4d897116b59c..7ca6aadcb122 100644 --- a/tests/expectations/metrics/column_map_metrics/test_column_map_condition_auxillary_methods.py +++ b/tests/expectations/metrics/column_map_metrics/test_column_map_condition_auxillary_methods.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any import numpy as np import pandas as pd @@ -72,8 +72,8 @@ def sql_execution_engine_with_mini_taxi_loaded(sa, mini_taxi_df): @pytest.fixture def spark_execution_engine_with_mini_taxi_loaded(spark_session, mini_taxi_df): - conf: List[tuple] = spark_session.sparkContext.getConf().getAll() - spark_config: Dict[str, Any] = dict(conf) + conf: list[tuple] = spark_session.sparkContext.getConf().getAll() + spark_config: dict[str, Any] = dict(conf) pandas_df = mini_taxi_df spark_df = spark_session.createDataFrame( diff --git a/tests/expectations/metrics/column_map_metrics/test_column_values_in_set.py b/tests/expectations/metrics/column_map_metrics/test_column_values_in_set.py index 80bbfff4f251..25a4e2db59b7 100644 --- a/tests/expectations/metrics/column_map_metrics/test_column_values_in_set.py +++ b/tests/expectations/metrics/column_map_metrics/test_column_values_in_set.py @@ -1,5 +1,5 @@ from types import ModuleType -from typing import List, Optional +from typing import Optional import pytest @@ -23,7 +23,7 @@ reason="sqlalchemy or sqlalchemy_bigquery is not installed", ) @pytest.mark.parametrize("value_set", [[False, True], [False, True, None], [True], [False], [None]]) -def test_sqlalchemy_impl_bigquery_bool(value_set: List[Optional[bool]]): +def test_sqlalchemy_impl_bigquery_bool(value_set: list[Optional[bool]]): column_name = "my_bool_col" column = sqlalchemy.column(column_name) kwargs = _make_sqlalchemy_kwargs(column_name, sqlalchemy_bigquery) @@ -41,7 +41,7 @@ def test_sqlalchemy_impl_bigquery_bool(value_set: List[Optional[bool]]): @pytest.mark.skipif(sqlalchemy is None, reason="sqlalchemy is not installed") @pytest.mark.parametrize("dialect", [mysql, postgresql, sqlite]) @pytest.mark.parametrize("value_set", [[False, True], [False, True, None], [True], [False], [None]]) -def test_sqlalchemy_impl_not_bigquery_bool(dialect: ModuleType, value_set: List[Optional[bool]]): +def test_sqlalchemy_impl_not_bigquery_bool(dialect: ModuleType, value_set: list[Optional[bool]]): column_name = "my_bool_col" column = sqlalchemy.column(column_name) kwargs = _make_sqlalchemy_kwargs(column_name, dialect) diff --git a/tests/expectations/metrics/column_map_metrics/test_unexpected_indices_and_query.py b/tests/expectations/metrics/column_map_metrics/test_unexpected_indices_and_query.py index 897d03d83c51..5c806180d298 100644 --- a/tests/expectations/metrics/column_map_metrics/test_unexpected_indices_and_query.py +++ b/tests/expectations/metrics/column_map_metrics/test_unexpected_indices_and_query.py @@ -1,5 +1,3 @@ -from typing import Dict, Tuple - import pandas as pd import pytest @@ -64,7 +62,7 @@ def metric_value_kwargs_complete() -> dict: def _build_table_columns_and_unexpected( engine, metric_value_kwargs -) -> Tuple[MetricConfiguration, MetricConfiguration, dict]: +) -> tuple[MetricConfiguration, MetricConfiguration, dict]: """ Helper method build the dependencies needed for unexpected_indices (ID/PK) related metrics. @@ -85,11 +83,11 @@ def _build_table_columns_and_unexpected( Tuple with MetricConfigurations corresponding to unexpected_condition and table_columns metric, as well as metrics dict. """ # noqa: E501 - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} # get table_columns_metric table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -138,7 +136,7 @@ def test_pd_unexpected_index_list_metric_without_id_pk(animal_table_df): "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(desired_metric,), metrics=metrics ) for key, val in results.items(): @@ -177,7 +175,7 @@ def test_pd_unexpected_index_list_metric_without_id_pk_without_column_values( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(desired_metric,), metrics=metrics ) for key, val in results.items(): @@ -205,7 +203,7 @@ def test_pd_unexpected_index_list_metric_with_id_pk(metric_value_kwargs_complete "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) for key, val in results.items(): @@ -249,7 +247,7 @@ def test_pd_unexpected_index_list_metric_with_id_pk_without_column_values( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) for key, val in results.items(): @@ -281,7 +279,7 @@ def test_sa_unexpected_index_list_metric_with_id_pk( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) for key, val in results.items(): @@ -325,7 +323,7 @@ def test_sa_unexpected_index_list_metric_with_id_pk_without_column_values( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) for key, val in results.items(): @@ -363,7 +361,7 @@ def test_sa_unexpected_index_list_metric_without_id_pk(sa, animal_table_df): "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) assert list(results.values())[0] is None @@ -399,7 +397,7 @@ def test_sa_unexpected_index_list_metric_without_id_pk_without_column_values(sa, "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) assert list(results.values())[0] is None @@ -427,7 +425,7 @@ def test_sa_unexpected_index_query_metric_with_id_pk( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_query,), metrics=metrics ) for key, val in results.items(): @@ -467,7 +465,7 @@ def test_sa_unexpected_index_query_metric_without_id_pk(sa, animal_table_df): "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_query,), metrics=metrics ) for key, val in results.items(): @@ -501,7 +499,7 @@ def test_spark_unexpected_index_list_metric_with_id_pk( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) for val in results.values(): @@ -547,7 +545,7 @@ def test_spark_unexpected_index_list_metric_with_id_pk_without_column_values( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) for val in results.values(): @@ -586,7 +584,7 @@ def test_spark_unexpected_index_list_metric_without_id_pk(spark_session, animal_ "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) assert list(results.values())[0] is None @@ -625,7 +623,7 @@ def test_spark_unexpected_index_list_metric_without_id_pk_without_column_values( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_list,), metrics=metrics ) assert list(results.values())[0] is None @@ -651,7 +649,7 @@ def test_pd_unexpected_index_query_metric_with_id_pk(animal_table_df, metric_val "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_query,), metrics=metrics ) for val in results.values(): @@ -688,7 +686,7 @@ def test_pd_unexpected_index_query_metric_without_id_pk( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_query,), metrics=metrics ) for val in results.values(): @@ -718,7 +716,7 @@ def test_spark_unexpected_index_query_metric_with_id_pk( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_query,), metrics=metrics ) for val in results.values(): @@ -760,7 +758,7 @@ def test_spark_unexpected_index_query_metric_without_id_pk( "unexpected_condition": unexpected_columns_metric, "table.columns": table_columns_metric, } - results: Dict[Tuple[str, str, str], MetricValue] = engine.resolve_metrics( + results: dict[tuple[str, str, str], MetricValue] = engine.resolve_metrics( metrics_to_resolve=(unexpected_index_query,), metrics=metrics ) for val in results.values(): diff --git a/tests/expectations/metrics/test_core.py b/tests/expectations/metrics/test_core.py index 479fd038aa11..c3627b444589 100644 --- a/tests/expectations/metrics/test_core.py +++ b/tests/expectations/metrics/test_core.py @@ -2,7 +2,7 @@ import datetime import logging from decimal import Decimal -from typing import Dict, Tuple, Union +from typing import Union import numpy as np import pandas as pd @@ -53,10 +53,10 @@ def test_basic_metric_pd(): batch = Batch(data=df) engine = PandasExecutionEngine(batch_data_dict={batch.id: batch.data}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -94,10 +94,10 @@ def test_basic_metric_pd(): def test_column_sum_metric_pd(build_engine, dataframe, expected_result): engine = build_engine(df=dataframe) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -132,10 +132,10 @@ def test_column_sum_metric_pd(build_engine, dataframe, expected_result): def test_column_sum_metric_spark(spark_session, dataframe, expected_result): engine = build_spark_engine(spark=spark_session, df=dataframe, batch_id="my_id") - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -179,10 +179,10 @@ def test_column_sum_metric_spark(spark_session, dataframe, expected_result): def test_column_mean_metric_pd(dataframe, expected_result): engine = build_pandas_engine(dataframe) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -214,10 +214,10 @@ def test_column_mean_metric_pd(dataframe, expected_result): def test_column_mean_metric_spark(spark_session, dataframe, expected_result): engine = build_spark_engine(spark=spark_session, df=dataframe, batch_id="my_id") - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -266,10 +266,10 @@ def test_column_mean_metric_spark(spark_session, dataframe, expected_result): def test_column_standard_deviation_metric_pd(build_engine, dataframe, expected_result): engine = build_engine(df=dataframe) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -306,10 +306,10 @@ def test_column_value_lengths_min_metric_pd(): ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -347,10 +347,10 @@ def test_column_quoted_name_type_sa(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -360,7 +360,7 @@ def test_column_quoted_name_type_sa(sa): metric_domain_kwargs={}, metric_value_kwargs=None, ) - table_columns_metric_id: Tuple[str, str, str] = table_columns_metric.id + table_columns_metric_id: tuple[str, str, str] = table_columns_metric.id batch_column_list = metrics[table_columns_metric_id] column_name: str @@ -429,10 +429,10 @@ def test_column_quoted_name_type_sa_handles_explicit_string_identifiers(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -442,7 +442,7 @@ def test_column_quoted_name_type_sa_handles_explicit_string_identifiers(sa): metric_domain_kwargs={}, metric_value_kwargs=None, ) - table_columns_metric_id: Tuple[str, str, str] = table_columns_metric.id + table_columns_metric_id: tuple[str, str, str] = table_columns_metric.id batch_column_list = metrics[table_columns_metric_id] for column_name in [ @@ -478,10 +478,10 @@ def test_column_value_lengths_min_metric_sa(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -532,10 +532,10 @@ def test_column_value_lengths_min_metric_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -584,10 +584,10 @@ def test_column_value_lengths_max_metric_pd(): ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -625,10 +625,10 @@ def test_column_value_lengths_max_metric_sa(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -679,10 +679,10 @@ def test_column_value_lengths_max_metric_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -716,10 +716,10 @@ def test_column_value_lengths_max_metric_spark(spark_session): def test_quantiles_metric_pd(): engine = build_pandas_engine(pd.DataFrame({"a": [1, 2, 3, 4]})) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -744,10 +744,10 @@ def test_quantiles_metric_pd(): def test_quantiles_metric_sa(sa): engine = build_sa_execution_engine(pd.DataFrame({"a": [1, 2, 3, 4]}), sa) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -796,10 +796,10 @@ def test_quantiles_metric_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -840,10 +840,10 @@ def test_column_histogram_metric_pd(): ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -897,10 +897,10 @@ def test_column_histogram_metric_sa(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -957,10 +957,10 @@ def test_column_histogram_metric_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1028,10 +1028,10 @@ def test_column_partition_metric_pd(): idx: int element: Union[float, pd.Timestamp] - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] # Test using standard numeric column. @@ -1190,10 +1190,10 @@ def test_column_partition_metric_sa(sa): # noqa: PLR0915 idx: int element: Union[float, pd.Timestamp] - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] # Test using standard numeric column. @@ -1413,10 +1413,10 @@ def test_column_partition_metric_spark(spark_session): # noqa: PLR0915 idx: int element: Union[float, pd.Timestamp] - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] # Test using standard numeric column. @@ -1585,10 +1585,10 @@ def test_max_metric_column_exists_pd(): batch = Batch(data=df) engine = PandasExecutionEngine(batch_data_dict={batch.id: batch.data}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1612,10 +1612,10 @@ def test_max_metric_column_does_not_exist_pd(): batch = Batch(data=df) engine = PandasExecutionEngine(batch_data_dict={batch.id: batch.data}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1640,10 +1640,10 @@ def test_max_metric_column_does_not_exist_pd(): def test_max_metric_column_exists_sa(sa): engine = build_sa_execution_engine(pd.DataFrame({"a": [1, 2, 1, None]}), sa) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1679,10 +1679,10 @@ def test_max_metric_column_exists_sa(sa): def test_max_metric_column_does_not_exist_sa(sa): engine = build_sa_execution_engine(pd.DataFrame({"a": [1, 2, 1, None]}), sa) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1711,10 +1711,10 @@ def test_max_metric_column_exists_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1754,10 +1754,10 @@ def test_max_metric_column_does_not_exist_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1782,10 +1782,10 @@ def test_max_metric_column_does_not_exist_spark(spark_session): def test_map_value_set_sa(sa): engine = build_sa_execution_engine(pd.DataFrame({"a": [1, 2, 3, 3, None]}), sa) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1855,10 +1855,10 @@ def test_map_value_set_spark(spark_session, basic_spark_df_execution_engine): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -1957,10 +1957,10 @@ def test_map_value_set_spark(spark_session, basic_spark_df_execution_engine): def test_map_column_value_lengths_between_pd(): engine = build_pandas_engine(pd.DataFrame({"a": ["a", "aaa", "bcbc", "defgh", None]})) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2003,10 +2003,10 @@ def test_map_column_values_increasing_pd(): ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2079,10 +2079,10 @@ def test_map_column_values_increasing_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2171,10 +2171,10 @@ def test_map_column_values_decreasing_pd(): ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2247,10 +2247,10 @@ def test_map_column_values_decreasing_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2320,10 +2320,10 @@ def test_map_column_values_decreasing_spark(spark_session): def test_map_unique_column_exists_pd(): engine = build_pandas_engine(pd.DataFrame({"a": [1, 2, 3, 3, 4, None]})) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2379,10 +2379,10 @@ def test_map_unique_column_exists_pd(): def test_map_unique_column_does_not_exist_pd(): engine = build_pandas_engine(pd.DataFrame({"a": [1, 2, 3, 3, None]})) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2409,10 +2409,10 @@ def test_map_unique_column_exists_sa(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2496,10 +2496,10 @@ def test_map_unique_column_does_not_exist_sa(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2570,10 +2570,10 @@ def test_map_unique_column_exists_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2663,10 +2663,10 @@ def test_map_unique_column_does_not_exist_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2691,10 +2691,10 @@ def test_z_score_under_threshold_pd(): df = pd.DataFrame({"a": [1, 2, 3, None]}) engine = PandasExecutionEngine(batch_data_dict={"my_id": df}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2774,10 +2774,10 @@ def test_z_score_under_threshold_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -2904,10 +2904,10 @@ def test_map_column_pairs_equal_metric_pd(): # noqa: PLR0915 ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -3143,10 +3143,10 @@ def test_map_column_pairs_equal_metric_sa(sa): # noqa: PLR0915 sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -3351,10 +3351,10 @@ def test_map_column_pairs_equal_metric_spark(spark_session): # noqa: PLR0915 batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -3551,10 +3551,10 @@ def test_map_column_pairs_greater_metric_pd(): df = pd.DataFrame({"a": [2, 3, 4, None, 3, None], "b": [1, 2, 3, None, 3, 5]}) engine = PandasExecutionEngine(batch_data_dict={"my_id": df}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -3629,10 +3629,10 @@ def test_map_column_pairs_greater_metric_sa(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -3702,10 +3702,10 @@ def test_map_column_pairs_greater_metric_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -3767,10 +3767,10 @@ def test_map_column_pairs_in_set_metric_pd(): df = pd.DataFrame({"a": [10, 3, 4, None, 3, None], "b": [1, 2, 3, None, 3, 5]}) engine = PandasExecutionEngine(batch_data_dict={"my_id": df}) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -3809,10 +3809,10 @@ def test_map_column_pairs_in_set_metric_sa(sa): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -3932,10 +3932,10 @@ def test_map_column_pairs_in_set_metric_spark(spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4085,10 +4085,10 @@ def test_column_median_metric_pd(): ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4126,10 +4126,10 @@ def test_column_median_metric_sa(sa, dataframe: pd.DataFrame, median: int): sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4240,10 +4240,10 @@ def test_column_median_metric_spark(spark_session): def test_value_counts_metric_pd(): engine = build_pandas_engine(pd.DataFrame({"a": [1, 2, 1, 2, 3, 3]})) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4348,10 +4348,10 @@ def test_distinct_metric_spark( batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4436,10 +4436,10 @@ def test_distinct_metric_sa( sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4515,10 +4515,10 @@ def test_distinct_metric_sa( def test_distinct_metric_pd(): engine = build_pandas_engine(pd.DataFrame({"a": [1, 2, 1, 2, 3, 3]})) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4601,10 +4601,10 @@ def test_batch_aggregate_metrics_pd(): ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4665,10 +4665,10 @@ def test_batch_aggregate_metrics_sa(caplog, sa): pd.DataFrame({"a": [1, 2, 1, 2, 3, 3], "b": [4, 4, 4, 4, 4, 4]}), sa ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4793,10 +4793,10 @@ def test_batch_aggregate_metrics_spark(caplog, spark_session): batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -4910,10 +4910,10 @@ def test_map_multicolumn_sum_equal_pd(): # noqa: PLR0915 pd.DataFrame(data={"a": [0, 1, 2], "b": [5, 4, 3], "c": [0, 0, 1], "d": [7, 8, 9]}) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -5111,10 +5111,10 @@ def test_map_multicolumn_sum_equal_sa(sa): # noqa: PLR0915 sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -5304,10 +5304,10 @@ def test_map_multicolumn_sum_equal_spark(spark_session): # noqa: PLR0915 batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -5497,10 +5497,10 @@ def test_map_compound_columns_unique_pd(): # noqa: PLR0915 pd.DataFrame(data={"a": [0, 1, 1], "b": [1, 2, 3], "c": [0, 2, 2]}) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -5694,10 +5694,10 @@ def test_map_compound_columns_unique_sa(sa): # noqa: PLR0915 sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -5922,10 +5922,10 @@ def test_map_compound_columns_unique_spark(spark_session): # noqa: PLR0915 batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -6117,10 +6117,10 @@ def test_map_select_column_values_unique_within_record_pd(): # noqa: PLR0915 ) ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -6355,10 +6355,10 @@ def test_map_select_column_values_unique_within_record_sa(sa): # noqa: PLR0915 sa, ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) @@ -6564,10 +6564,10 @@ def test_map_select_column_values_unique_within_record_spark( # noqa: PLR0915 # batch_id="my_id", ) - metrics: Dict[Tuple[str, str, str], MetricValue] = {} + metrics: dict[tuple[str, str, str], MetricValue] = {} table_columns_metric: MetricConfiguration - results: Dict[Tuple[str, str, str], MetricValue] + results: dict[tuple[str, str, str], MetricValue] table_columns_metric, results = get_table_columns_metric(execution_engine=engine) metrics.update(results) diff --git a/tests/expectations/metrics/test_metric_providers.py b/tests/expectations/metrics/test_metric_providers.py index cf7d98ab0054..e85df08e2967 100644 --- a/tests/expectations/metrics/test_metric_providers.py +++ b/tests/expectations/metrics/test_metric_providers.py @@ -1,7 +1,7 @@ from __future__ import annotations import copy -from typing import Any, Dict, List +from typing import Any from unittest import mock import pytest @@ -105,7 +105,7 @@ def _pandas( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): raise NotImplementedError @@ -116,7 +116,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): raise NotImplementedError @@ -127,7 +127,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): raise NotImplementedError @@ -155,7 +155,7 @@ def _pandas( execution_engine: PandasExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): raise NotImplementedError @@ -166,7 +166,7 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): raise NotImplementedError @@ -177,7 +177,7 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, ): raise NotImplementedError @@ -346,9 +346,9 @@ def _sqlalchemy( execution_engine: SqlAlchemyExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, - ) -> List[dict]: + ) -> list[dict]: raise NotImplementedError @metric_value(engine=SparkDFExecutionEngine) @@ -357,9 +357,9 @@ def _spark( execution_engine: SparkDFExecutionEngine, metric_domain_kwargs: dict, metric_value_kwargs: dict, - metrics: Dict[str, Any], + metrics: dict[str, Any], runtime_configuration: dict, - ) -> List[dict]: + ) -> list[dict]: raise NotImplementedError CustomQueryMetricProvider() diff --git a/tests/expectations/metrics/test_metrics_util.py b/tests/expectations/metrics/test_metrics_util.py index 66fd371da8eb..b8a3951cb729 100644 --- a/tests/expectations/metrics/test_metrics_util.py +++ b/tests/expectations/metrics/test_metrics_util.py @@ -1,7 +1,7 @@ from __future__ import annotations import random -from typing import TYPE_CHECKING, Final, List, Union +from typing import TYPE_CHECKING, Final, Union import pytest from _pytest import monkeypatch @@ -123,7 +123,7 @@ def unexpected_index_list_two_index_columns_without_column_values(): ], ) def test_sql_statement_conversion_to_string_for_backends( - backend_name: str, connection_string: str, test_backends: List[str] + backend_name: str, connection_string: str, test_backends: list[str] ): if backend_name in test_backends: engine = SqlAlchemyExecutionEngine(connection_string=connection_string) @@ -172,8 +172,8 @@ def test_get_unexpected_indices_for_single_pandas_named_index_named_unexpected_i ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = ["pk_1"] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = ["pk_1"] unexpected_index_list = get_unexpected_indices_for_single_pandas_named_index( domain_records_df=updated_dataframe, @@ -190,8 +190,8 @@ def test_get_unexpected_indices_for_single_pandas_named_index_named_unexpected_i ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = ["pk_1"] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = ["pk_1"] unexpected_index_list = get_unexpected_indices_for_single_pandas_named_index( domain_records_df=updated_dataframe, @@ -209,8 +209,8 @@ def test_get_unexpected_indices_for_single_pandas_named_index( ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = [updated_dataframe.index.name] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = [updated_dataframe.index.name] unexpected_index_list = get_unexpected_indices_for_single_pandas_named_index( domain_records_df=updated_dataframe, @@ -227,8 +227,8 @@ def test_get_unexpected_indices_for_single_pandas_named_index_without_column_val ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = [updated_dataframe.index.name] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = [updated_dataframe.index.name] unexpected_index_list = get_unexpected_indices_for_single_pandas_named_index( domain_records_df=updated_dataframe, @@ -246,8 +246,8 @@ def test_get_unexpected_indices_for_multiple_pandas_named_indices( ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1", "pk_2"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = list(updated_dataframe.index.names) + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = list(updated_dataframe.index.names) unexpected_index_list = get_unexpected_indices_for_multiple_pandas_named_indices( domain_records_df=updated_dataframe, @@ -264,8 +264,8 @@ def test_get_unexpected_indices_for_multiple_pandas_named_indices_without_column ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1", "pk_2"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = list(updated_dataframe.index.names) + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = list(updated_dataframe.index.names) unexpected_index_list = get_unexpected_indices_for_multiple_pandas_named_indices( domain_records_df=updated_dataframe, @@ -283,8 +283,8 @@ def test_get_unexpected_indices_for_multiple_pandas_named_indices_named_unexpect ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1", "pk_2"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = ["pk_1", "pk_2"] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = ["pk_1", "pk_2"] unexpected_index_list = get_unexpected_indices_for_multiple_pandas_named_indices( domain_records_df=updated_dataframe, @@ -301,8 +301,8 @@ def test_get_unexpected_indices_for_multiple_pandas_named_indices_named_unexpect ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1", "pk_2"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = ["pk_1", "pk_2"] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = ["pk_1", "pk_2"] unexpected_index_list = get_unexpected_indices_for_multiple_pandas_named_indices( domain_records_df=updated_dataframe, @@ -320,8 +320,8 @@ def test_get_unexpected_indices_for_multiple_pandas_named_indices_named_unexpect ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1", "pk_2"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = ["pk_1"] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = ["pk_1"] unexpected_index_list = get_unexpected_indices_for_multiple_pandas_named_indices( domain_records_df=updated_dataframe, @@ -338,8 +338,8 @@ def test_get_unexpected_indices_for_multiple_pandas_named_indices_named_unexpect ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1", "pk_2"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = ["pk_1"] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = ["pk_1"] unexpected_index_list = get_unexpected_indices_for_multiple_pandas_named_indices( domain_records_df=updated_dataframe, @@ -356,8 +356,8 @@ def test_get_unexpected_indices_for_multiple_pandas_named_indices_named_unexpect ): dataframe: pd.DataFrame = pandas_animals_dataframe_for_unexpected_rows_and_index updated_dataframe: pd.DataFrame = dataframe.set_index(["pk_1", "pk_2"]) - expectation_domain_column_list: List[str] = ["animals"] - unexpected_index_column_names: List[str] = ["i_dont_exist"] + expectation_domain_column_list: list[str] = ["animals"] + unexpected_index_column_names: list[str] = ["i_dont_exist"] with pytest.raises(MetricResolutionError) as e: get_unexpected_indices_for_multiple_pandas_named_indices( domain_records_df=updated_dataframe, diff --git a/tests/expectations/test_dataclass_serializable_dot_dict_pattern.py b/tests/expectations/test_dataclass_serializable_dot_dict_pattern.py index abb3e13979b6..9746c698797f 100644 --- a/tests/expectations/test_dataclass_serializable_dot_dict_pattern.py +++ b/tests/expectations/test_dataclass_serializable_dot_dict_pattern.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from enum import Enum -from typing import List, Optional, Tuple +from typing import Optional import pytest from pytest import raises @@ -22,7 +22,7 @@ class MyClassA(SerializableDictDot): @dataclass class MyClassB(MyClassA): - baz: List[str] + baz: list[str] qux: Optional[int] = None quux: int = 42 @@ -41,10 +41,10 @@ class MyEnum(Enum): class MyClassC(SerializableDictDot): alpha_var: int beta_var: MyEnum - A_list: List[MyClassA] - B_list: List[MyClassB] - enum_list: List[MyEnum] = field(default_factory=list) - some_tuple: Optional[Tuple[MyClassA, MyClassB]] = None + A_list: list[MyClassA] + B_list: list[MyClassB] + enum_list: list[MyEnum] = field(default_factory=list) + some_tuple: Optional[tuple[MyClassA, MyClassB]] = None @property def num_As(self): diff --git a/tests/expectations/test_expectation.py b/tests/expectations/test_expectation.py index 490449faa2aa..0bf84fdbf666 100644 --- a/tests/expectations/test_expectation.py +++ b/tests/expectations/test_expectation.py @@ -2,7 +2,7 @@ import itertools import logging -from typing import Any, Dict, List, Optional, Sequence, Union +from typing import Any, Optional, Sequence, Union import pytest @@ -59,8 +59,8 @@ def metrics_dict(): def fake_metrics_config_list( - metric_name: str, metric_domain_kwargs: Dict[str, Any] -) -> List[MetricConfiguration]: + metric_name: str, metric_domain_kwargs: dict[str, Any] +) -> list[MetricConfiguration]: """ Helper method to generate list of MetricConfiguration objects for tests. """ @@ -74,7 +74,7 @@ def fake_metrics_config_list( def fake_expectation_config( - expectation_type: str, config_kwargs: Dict[str, Any] + expectation_type: str, config_kwargs: dict[str, Any] ) -> ExpectationConfiguration: """ Helper method to generate of ExpectationConfiguration objects for tests. @@ -191,7 +191,7 @@ def test_multicolumn_expectation_validation_errors_with_bad_mostly(fake_expectat @pytest.mark.unit def test_validate_dependencies_against_available_metrics_success(metrics_dict): - metric_config_list: List[MetricConfiguration] = fake_metrics_config_list( + metric_config_list: list[MetricConfiguration] = fake_metrics_config_list( metric_name="column_values.nonnull.unexpected_count", metric_domain_kwargs={ "batch_id": "projects-projects", @@ -206,7 +206,7 @@ def test_validate_dependencies_against_available_metrics_success(metrics_dict): @pytest.mark.unit def test_validate_dependencies_against_available_metrics_failure(metrics_dict): - metric_config_list: List[MetricConfiguration] = fake_metrics_config_list( + metric_config_list: list[MetricConfiguration] = fake_metrics_config_list( metric_name="column_values.nonnull.unexpected_count", metric_domain_kwargs={ "batch_id": "projects-projects", diff --git a/tests/expectations/test_expectation_atomic_renderers.py b/tests/expectations/test_expectation_atomic_renderers.py index 0bdaf8c3aecd..ba9cd2747f87 100644 --- a/tests/expectations/test_expectation_atomic_renderers.py +++ b/tests/expectations/test_expectation_atomic_renderers.py @@ -1,6 +1,6 @@ import re from pprint import pprint -from typing import Callable, Dict, Union +from typing import Callable, Union import pytest @@ -25,10 +25,10 @@ def expectation_configuration_kwargs(): @pytest.fixture def get_prescriptive_rendered_content( - expectation_configuration_kwargs: Dict[str, Union[str, dict]], + expectation_configuration_kwargs: dict[str, Union[str, dict]], ) -> Callable: def _get_prescriptive_rendered_content( - update_dict: Dict[str, Union[str, dict]], + update_dict: dict[str, Union[str, dict]], ) -> RenderedAtomicContent: # Overwrite any fields passed in from test and instantiate ExpectationConfiguration expectation_configuration_kwargs.update(update_dict) @@ -60,10 +60,10 @@ def evr_kwargs(expectation_configuration_kwargs): @pytest.fixture def get_diagnostic_rendered_content( - evr_kwargs: Dict[str, Union[dict, ExpectationConfiguration]], + evr_kwargs: dict[str, Union[dict, ExpectationConfiguration]], ) -> Callable: def _get_diagnostic_rendered_content( - update_dict: Dict[str, Union[dict, ExpectationConfiguration]], + update_dict: dict[str, Union[dict, ExpectationConfiguration]], ) -> RenderedAtomicContent: # Overwrite any fields passed in from test and instantiate ExpectationValidationResult evr_kwargs.update(update_dict) diff --git a/tests/expectations/test_util.py b/tests/expectations/test_util.py index c6a3155549a7..494c3c1981fd 100644 --- a/tests/expectations/test_util.py +++ b/tests/expectations/test_util.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Dict, List, cast +from typing import cast import pandas as pd import pytest @@ -336,7 +336,7 @@ def test_table_column_reflection_fallback(test_backends, sa): if not sa.create_engine: pytest.skip("Unable to import sqlalchemy.create_engine() -- skipping.") - test_backend_names: List[str] = build_test_backends_list_v3( + test_backend_names: list[str] = build_test_backends_list_v3( include_pandas=False, include_spark=False, include_sqlalchemy=include_sqlalchemy, @@ -356,7 +356,7 @@ def test_table_column_reflection_fallback(test_backends, sa): } ) - validators_config: Dict[str, Validator] = {} + validators_config: dict[str, Validator] = {} validator: Validator backend_name: str table_name: str @@ -381,8 +381,8 @@ def test_table_column_reflection_fallback(test_backends, sa): table_columns_metric: MetricConfiguration results: dict - reflected_columns_list: List[Dict[str, str]] - reflected_column_config: Dict[str, str] + reflected_columns_list: list[dict[str, str]] + reflected_column_config: dict[str, str] column_name: str validation_result: ExpectationValidationResult diff --git a/tests/experimental/metric_repository/test_metric_list_metric_retriever.py b/tests/experimental/metric_repository/test_metric_list_metric_retriever.py index 0c9c2c0d4394..7e8356eee151 100644 --- a/tests/experimental/metric_repository/test_metric_list_metric_retriever.py +++ b/tests/experimental/metric_repository/test_metric_list_metric_retriever.py @@ -1,5 +1,3 @@ -from typing import Dict, List - import pytest from great_expectations.data_context import CloudDataContext @@ -74,7 +72,7 @@ def test_get_metrics_table_metrics_only( MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, ] - aborted_metrics: Dict[str, str] = {} + aborted_metrics: dict[str, str] = {} mock_validator.compute_metrics.return_value = ( computed_metrics, aborted_metrics, @@ -91,13 +89,13 @@ def test_get_metrics_table_metrics_only( value=2, exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["col1", "col2"], exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.column_types", value=[ @@ -140,7 +138,7 @@ def test_get_metrics_full_list( MetricTypes.COLUMN_MEDIAN, MetricTypes.COLUMN_NULL_COUNT, ] - aborted_metrics: Dict[str, str] = {} + aborted_metrics: dict[str, str] = {} mock_validator.compute_metrics.return_value = ( computed_metrics, aborted_metrics, @@ -157,13 +155,13 @@ def test_get_metrics_full_list( TableMetric[int]( batch_id="batch_id", metric_name="table.row_count", value=2, exception=None ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["col1", "col2"], exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.column_types", value=[ @@ -257,7 +255,7 @@ def test_column_metrics_not_returned_if_column_types_missing( ("table.row_count", (), ()): 2, ("table.columns", (), ()): ["timestamp_col"], } - metrics_list: List[MetricTypes] = [ + metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, # MetricTypes.TABLE_COLUMN_TYPES, @@ -284,7 +282,7 @@ def test_column_metrics_not_returned_if_column_types_missing( value=2, exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["timestamp_col"], @@ -316,7 +314,7 @@ def test_get_metrics_metrics_missing( ("column.min", "column=col2", ()): 2.7, } - metrics_list: List[MetricTypes] = [ + metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, @@ -342,13 +340,13 @@ def test_get_metrics_metrics_missing( message="Metric was not successfully computed but exception was not found.", ), ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["col1", "col2"], exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.column_types", value=[ @@ -418,7 +416,7 @@ def test_get_metrics_with_exception( computed_metrics, aborted_metrics, ) - metrics_list: List[MetricTypes] = [ + metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, @@ -435,13 +433,13 @@ def test_get_metrics_with_exception( value=None, exception=MetricException(type="Unknown", message="test exception message"), ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["col1", "col2"], exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.column_types", value=[ @@ -493,7 +491,7 @@ def test_get_metrics_with_column_type_missing( computed_metrics, aborted_metrics, ) - metrics_list: List[MetricTypes] = [ + metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, @@ -513,13 +511,13 @@ def test_get_metrics_with_column_type_missing( value=None, exception=MetricException(type="Unknown", message="test exception message"), ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["col1", "col2"], exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.column_types", value=[ @@ -560,7 +558,7 @@ def test_get_metrics_with_timestamp_columns( ("column.max", "column=timestamp_col", ()): "2023-12-31T00:00:00", ("column_values.null.count", "column=timestamp_col", ()): 1, } - metrics_list: List[MetricTypes] = [ + metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, @@ -587,13 +585,13 @@ def test_get_metrics_with_timestamp_columns( value=2, exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["timestamp_col"], exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.column_types", value=[{"name": "timestamp_col", "type": "TIMESTAMP_NTZ"}], @@ -636,7 +634,7 @@ def test_get_metrics_only_gets_a_validator_once( {"name": "col2", "type": "float"}, ], } - metrics_list: List[MetricTypes] = [ + metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, @@ -658,7 +656,7 @@ def test_get_metrics_with_no_metrics( mock_context, mock_validator, mock_batch_request, metric_retriever ): computed_metrics = {} - metrics_list: List[MetricTypes] = [] + metrics_list: list[MetricTypes] = [] aborted_metrics = {} mock_validator.compute_metrics.return_value = ( computed_metrics, @@ -738,7 +736,7 @@ def test_get_table_columns( mock_validator.compute_metrics.return_value = (computed_metrics, aborted_metrics) ret = metric_retriever._get_table_columns(mock_batch_request) - assert ret == TableMetric[List[str]]( + assert ret == TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["col1", "col2"], @@ -777,7 +775,7 @@ def test_get_metrics_with_timestamp_columns_exclude_time( ("column_values.null.count", "column=timestamp_col", ()): 1, ("column_values.null.count", "column=time_col", ()): 1, } - metrics_list: List[MetricTypes] = [ + metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, @@ -803,13 +801,13 @@ def test_get_metrics_with_timestamp_columns_exclude_time( value=2, exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.columns", value=["timestamp_col", "time_col"], exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id="batch_id", metric_name="table.column_types", value=[ diff --git a/tests/experimental/metric_repository/test_metric_list_metric_retriever_integration.py b/tests/experimental/metric_repository/test_metric_list_metric_retriever_integration.py index 2309f3e78362..22cf638be5dd 100644 --- a/tests/experimental/metric_repository/test_metric_list_metric_retriever_integration.py +++ b/tests/experimental/metric_repository/test_metric_list_metric_retriever_integration.py @@ -2,8 +2,6 @@ from __future__ import annotations -from typing import List - import pandas as pd import pytest from pandas import Timestamp @@ -52,7 +50,7 @@ def test_get_metrics_table_metrics_only( cloud_context_and_batch_request_with_simple_dataframe: tuple[CloudDataContext, BatchRequest], ): context, batch_request = cloud_context_and_batch_request_with_simple_dataframe - table_metrics_list: List[MetricTypes] = [ + table_metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, @@ -71,7 +69,7 @@ def test_get_metrics_table_metrics_only( value=3, exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id=batch_id, metric_name="table.columns", value=[ @@ -84,7 +82,7 @@ def test_get_metrics_table_metrics_only( ], exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id=batch_id, metric_name="table.column_types", value=[ @@ -110,7 +108,7 @@ def test_get_metrics_full_cdm( cloud_context_and_batch_request_with_simple_dataframe: tuple[CloudDataContext, BatchRequest], ): context, batch_request = cloud_context_and_batch_request_with_simple_dataframe - cdm_metrics_list: List[MetricTypes] = [ + cdm_metrics_list: list[MetricTypes] = [ MetricTypes.TABLE_ROW_COUNT, MetricTypes.TABLE_COLUMNS, MetricTypes.TABLE_COLUMN_TYPES, @@ -134,7 +132,7 @@ def test_get_metrics_full_cdm( value=3, exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id=batch_id, metric_name="table.columns", value=[ @@ -203,7 +201,7 @@ def test_get_metrics_full_cdm( value=3.5, exception=None, ), - TableMetric[List[str]]( + TableMetric[list[str]]( batch_id=batch_id, metric_name="table.column_types", value=[ diff --git a/tests/integration/cloud/rest_contracts/conftest.py b/tests/integration/cloud/rest_contracts/conftest.py index 08cc172ef8ea..1794e2c20d01 100644 --- a/tests/integration/cloud/rest_contracts/conftest.py +++ b/tests/integration/cloud/rest_contracts/conftest.py @@ -5,7 +5,7 @@ import pathlib import subprocess import uuid -from typing import TYPE_CHECKING, Any, Callable, Dict, Final, List, Union +from typing import TYPE_CHECKING, Any, Callable, Final, Union import pact import pytest @@ -30,10 +30,10 @@ PACT_MOCK_SERVICE_URL: Final[str] = f"http://{PACT_MOCK_HOST}:{PACT_MOCK_PORT}" -JsonData: TypeAlias = Union[None, int, str, bool, List[Any], Dict[str, Any]] +JsonData: TypeAlias = Union[None, int, str, bool, list[Any], dict[str, Any]] PactBody: TypeAlias = Union[ - Dict[str, Union[JsonData, pact.matchers.Matcher]], pact.matchers.Matcher, None + dict[str, Union[JsonData, pact.matchers.Matcher]], pact.matchers.Matcher, None ] diff --git a/tests/integration/common_workflows/test_filesystem_asset_workflows.py b/tests/integration/common_workflows/test_filesystem_asset_workflows.py index 533ced04961b..fd6388ed02eb 100644 --- a/tests/integration/common_workflows/test_filesystem_asset_workflows.py +++ b/tests/integration/common_workflows/test_filesystem_asset_workflows.py @@ -7,7 +7,7 @@ """ import re -from typing import Dict, Optional +from typing import Optional import pytest @@ -303,7 +303,7 @@ def test_get_batch_identifiers_list__no_batches( def test_batch_validate_expectation( expectation: gxe.Expectation, batch_definition_fixture_name: str, - batch_parameters: Optional[Dict], + batch_parameters: Optional[dict], request: pytest.FixtureRequest, ) -> None: """Ensure Batch::validate(Epectation) works""" @@ -324,7 +324,7 @@ def test_batch_validate_expectation( def test_batch_validate_expectation_suite( expectation: gxe.Expectation, batch_definition_fixture_name: str, - batch_parameters: Optional[Dict], + batch_parameters: Optional[dict], request: pytest.FixtureRequest, expect_10_rows: gxe.Expectation, ) -> None: @@ -345,7 +345,7 @@ def test_batch_validate_expectation_suite( def test_validation_definition_run( expectation: gxe.Expectation, batch_definition_fixture_name: str, - batch_parameters: Optional[Dict], + batch_parameters: Optional[dict], context: AbstractDataContext, request: pytest.FixtureRequest, expect_10_rows: gxe.Expectation, @@ -371,7 +371,7 @@ def test_validation_definition_run( def test_checkpoint_run( expectation: gxe.Expectation, batch_definition_fixture_name: str, - batch_parameters: Optional[Dict], + batch_parameters: Optional[dict], context: AbstractDataContext, request: pytest.FixtureRequest, expect_10_rows: gxe.Expectation, diff --git a/tests/integration/common_workflows/test_sql_asset_workflows.py b/tests/integration/common_workflows/test_sql_asset_workflows.py index 9dbd6d90529f..b2ab1d6ac743 100644 --- a/tests/integration/common_workflows/test_sql_asset_workflows.py +++ b/tests/integration/common_workflows/test_sql_asset_workflows.py @@ -4,7 +4,7 @@ based on knowledge of the data in the test set. """ -from typing import Dict, Optional +from typing import Optional import pytest @@ -165,7 +165,7 @@ def test_get_batch_identifiers_list__no_batches( def test_batch_validate_expectation( expectation: gxe.Expectation, batch_definition_fixture_name: str, - batch_parameters: Optional[Dict], + batch_parameters: Optional[dict], request: pytest.FixtureRequest, ) -> None: """Ensure Batch::validate(Epectation) works""" @@ -184,7 +184,7 @@ def test_batch_validate_expectation( def test_batch_validate_expectation_suite( expectation: gxe.Expectation, batch_definition_fixture_name: str, - batch_parameters: Optional[Dict], + batch_parameters: Optional[dict], request: pytest.FixtureRequest, ) -> None: """Ensure Batch::validate(EpectationSuite) works""" @@ -204,7 +204,7 @@ def test_batch_validate_expectation_suite( def test_validation_definition_run( expectation: gxe.Expectation, batch_definition_fixture_name: str, - batch_parameters: Optional[Dict], + batch_parameters: Optional[dict], context: AbstractDataContext, request: pytest.FixtureRequest, ) -> None: @@ -227,7 +227,7 @@ def test_validation_definition_run( def test_checkpoint_run( expectation: gxe.Expectation, batch_definition_fixture_name: str, - batch_parameters: Optional[Dict], + batch_parameters: Optional[dict], context: AbstractDataContext, request: pytest.FixtureRequest, ) -> None: diff --git a/tests/integration/db/taxi_data_utils.py b/tests/integration/db/taxi_data_utils.py index b93a833d3c7c..25f64ed4c3db 100644 --- a/tests/integration/db/taxi_data_utils.py +++ b/tests/integration/db/taxi_data_utils.py @@ -2,7 +2,7 @@ import warnings from contextlib import contextmanager -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING import sqlalchemy as sa @@ -36,7 +36,7 @@ def _load_data( table_name: str = TAXI_DATA_TABLE_NAME, random_table_suffix: bool = True, ) -> LoadedTable: - dialects_supporting_multiple_values_in_single_insert_clause: List[str] = ["redshift"] + dialects_supporting_multiple_values_in_single_insert_clause: list[str] = ["redshift"] to_sql_method: str = ( "multi" if dialect in dialects_supporting_multiple_values_in_single_insert_clause else None ) @@ -97,7 +97,7 @@ def _execute_taxi_partitioning_test_cases( connection_string: str, table_name: str, ) -> None: - test_cases: List[TaxiPartitioningTestCase] = taxi_partitioning_test_cases.test_cases() + test_cases: list[TaxiPartitioningTestCase] = taxi_partitioning_test_cases.test_cases() test_case: TaxiPartitioningTestCase for test_case in test_cases: @@ -112,7 +112,7 @@ def _execute_taxi_partitioning_test_cases( data_asset_name: str = table_name # Read from generated table name column_name: str = taxi_partitioning_test_cases.test_column_name - column_names: List[str] = taxi_partitioning_test_cases.test_column_names + column_names: list[str] = taxi_partitioning_test_cases.test_column_names # 2. Set partitioner in DataConnector config datasource = add_datasource( @@ -150,7 +150,7 @@ def _execute_taxi_partitioning_test_cases( f"but expected {test_case.num_expected_batch_definitions}" ) - expected_batch_metadata: List[dict] + expected_batch_metadata: list[dict] if test_case.table_domain_test_case: expected_batch_metadata = [{}] diff --git a/tests/integration/docusaurus/connecting_to_your_data/datasource_configuration/datasource_configuration_test_utilities.py b/tests/integration/docusaurus/connecting_to_your_data/datasource_configuration/datasource_configuration_test_utilities.py index 49ee411a7391..e532541b1abe 100644 --- a/tests/integration/docusaurus/connecting_to_your_data/datasource_configuration/datasource_configuration_test_utilities.py +++ b/tests/integration/docusaurus/connecting_to_your_data/datasource_configuration/datasource_configuration_test_utilities.py @@ -2,13 +2,12 @@ import operator from functools import reduce -from typing import List, Tuple # The following method is used to ensure that the dictionary used to verify universal configuration elements # remains the same in all the configuration tests. Users may disregard it. -def _get_items_by_path(root_dictionary: dict, keys: Tuple[str]) -> Tuple: +def _get_items_by_path(root_dictionary: dict, keys: tuple[str]) -> tuple: try: return "/".join(keys), reduce(operator.getitem, keys, root_dictionary) except Exception: @@ -16,10 +15,10 @@ def _get_items_by_path(root_dictionary: dict, keys: Tuple[str]) -> Tuple: def _gather_key_paths_from_dict( - target_dict: dict, current_path: List[str] | None = None -) -> Tuple[List[Tuple[str]], List[str]]: - key_paths: List[Tuple[str]] = [] - full_paths: List[Tuple[str]] = [] + target_dict: dict, current_path: list[str] | None = None +) -> tuple[list[tuple[str]], list[str]]: + key_paths: list[tuple[str]] = [] + full_paths: list[tuple[str]] = [] for key, value in target_dict.items(): if isinstance(value, dict): if current_path: diff --git a/tests/integration/fixtures/partition_and_sample_data/partitioner_test_cases_and_fixtures.py b/tests/integration/fixtures/partition_and_sample_data/partitioner_test_cases_and_fixtures.py index 0103040c4c56..cfaefab627df 100644 --- a/tests/integration/fixtures/partition_and_sample_data/partitioner_test_cases_and_fixtures.py +++ b/tests/integration/fixtures/partition_and_sample_data/partitioner_test_cases_and_fixtures.py @@ -2,7 +2,7 @@ import hashlib from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, List, Optional +from typing import Any, Optional import pandas as pd @@ -18,8 +18,8 @@ def __init__( self, test_df: pd.DataFrame, test_column_name: Optional[str] = None, - test_column_names: Optional[List[str]] = None, - column_names_to_convert: Optional[List[str]] = None, + test_column_names: Optional[list[str]] = None, + column_names_to_convert: Optional[list[str]] = None, ): if ( sum( @@ -54,43 +54,43 @@ def test_column_name(self) -> Optional[str]: return self._test_column_name @property - def test_column_names(self) -> Optional[List[str]]: + def test_column_names(self) -> Optional[list[str]]: return self._test_column_names @staticmethod - def years_in_taxi_data() -> List[datetime.datetime]: + def years_in_taxi_data() -> list[datetime.datetime]: return ( pd.date_range(start="2018-01-01", end="2020-12-31", freq="AS").to_pydatetime().tolist() ) - def year_batch_identifier_data(self) -> List[dict]: + def year_batch_identifier_data(self) -> list[dict]: return [{DatePart.YEAR.value: dt.year} for dt in self.years_in_taxi_data()] @staticmethod - def months_in_taxi_data() -> List[datetime.datetime]: + def months_in_taxi_data() -> list[datetime.datetime]: return ( pd.date_range(start="2018-01-01", end="2020-12-31", freq="MS").to_pydatetime().tolist() ) - def get_unique_sorted_months_in_taxi_data(self) -> List[str]: - months: List[datetime.datetime] = sorted(set(self.months_in_taxi_data())) + def get_unique_sorted_months_in_taxi_data(self) -> list[str]: + months: list[datetime.datetime] = sorted(set(self.months_in_taxi_data())) month: datetime.datetime return [month.strftime("%Y-%m-%d") for month in months] - def year_month_batch_identifier_data(self) -> List[dict]: + def year_month_batch_identifier_data(self) -> list[dict]: return [ {DatePart.YEAR.value: dt.year, DatePart.MONTH.value: dt.month} for dt in self.months_in_taxi_data() ] - def month_batch_identifier_data(self) -> List[dict]: + def month_batch_identifier_data(self) -> list[dict]: return [{DatePart.MONTH.value: dt.month} for dt in self.months_in_taxi_data()] - def year_month_day_batch_identifier_data(self) -> List[dict]: + def year_month_day_batch_identifier_data(self) -> list[dict]: # Since taxi data does not contain all days, # we need to introspect the data to build the fixture: - year_month_day_batch_identifier_list_unsorted: List[dict] = list( + year_month_day_batch_identifier_list_unsorted: list[dict] = list( {val[0]: val[1], val[2]: val[3], val[4]: val[5]} for val in { ( @@ -114,12 +114,12 @@ def year_month_day_batch_identifier_data(self) -> List[dict]: ), ) - def get_test_column_values(self) -> List[Optional[Any]]: - column_values: List[Optional[Any]] = self.test_df[self.test_column_name].tolist() + def get_test_column_values(self) -> list[Optional[Any]]: + column_values: list[Optional[Any]] = self.test_df[self.test_column_name].tolist() return column_values - def get_test_multi_column_values(self) -> List[dict]: - multi_column_values: List[dict] = self.test_df[self.test_column_names].to_dict("records") + def get_test_multi_column_values(self) -> list[dict]: + multi_column_values: list[dict] = self.test_df[self.test_column_names].to_dict("records") return multi_column_values def get_unique_sorted_test_column_values( @@ -127,8 +127,8 @@ def get_unique_sorted_test_column_values( reverse: Optional[bool] = False, move_null_to_front: Optional[bool] = False, limit: Optional[int] = None, - ) -> List[Optional[Any]]: - column_values: List[Optional[Any]] = self.get_test_column_values() + ) -> list[Optional[Any]]: + column_values: list[Optional[Any]] = self.get_test_column_values() column_values = list(set(column_values)) column_values = sorted( column_values, @@ -156,8 +156,8 @@ def get_unique_sorted_test_multi_column_values( self, reverse: Optional[bool] = False, limit: Optional[int] = None, - ) -> List[dict]: - multi_column_values: List[dict] = self.get_test_multi_column_values() + ) -> list[dict]: + multi_column_values: list[dict] = self.get_test_multi_column_values() multi_column_values = sorted( multi_column_values, key=lambda element: sum( @@ -171,9 +171,9 @@ def get_unique_sorted_test_multi_column_values( reverse=reverse, ) - unique_multi_column_values: List[dict] = [] + unique_multi_column_values: list[dict] = [] - hash_codes: List[str] = [] + hash_codes: list[str] = [] hash_code: str dictionary_element: dict for dictionary_element in multi_column_values: @@ -191,29 +191,29 @@ def get_unique_sorted_test_multi_column_values( return unique_multi_column_values[:limit] - def get_divided_integer_test_column_values(self, divisor: int) -> List[Optional[Any]]: - column_values: List[Optional[Any]] = self.get_test_column_values() + def get_divided_integer_test_column_values(self, divisor: int) -> list[Optional[Any]]: + column_values: list[Optional[Any]] = self.get_test_column_values() column_value: Any column_values = [column_value // divisor for column_value in column_values] return list(set(column_values)) - def get_mod_integer_test_column_values(self, mod: int) -> List[Optional[Any]]: - column_values: List[Optional[Any]] = self.get_test_column_values() + def get_mod_integer_test_column_values(self, mod: int) -> list[Optional[Any]]: + column_values: list[Optional[Any]] = self.get_test_column_values() column_value: Any column_values = [column_value % mod for column_value in column_values] return list(set(column_values)) - def get_hashed_test_column_values(self, hash_digits: int) -> List[Optional[Any]]: + def get_hashed_test_column_values(self, hash_digits: int) -> list[Optional[Any]]: """ hashlib.md5(string).hexdigest() hashlib.md5(str(tuple_).encode("utf-8")).hexdigest() [:num_digits] """ - column_values: List[Optional[Any]] = self.get_unique_sorted_test_column_values( + column_values: list[Optional[Any]] = self.get_unique_sorted_test_column_values( reverse=False, move_null_to_front=False, limit=None ) @@ -235,7 +235,7 @@ class TaxiPartitioningTestCase: num_expected_rows_in_first_batch_definition: int add_batch_definition_method_name: str add_batch_definition_kwargs: dict - expected_column_values: List[Any] = field(default_factory=list) + expected_column_values: list[Any] = field(default_factory=list) class TaxiPartitioningTestCasesBase(ABC): @@ -255,17 +255,17 @@ def test_column_name(self) -> str: return self._taxi_test_data.test_column_name @property - def test_column_names(self) -> List[str]: + def test_column_names(self) -> list[str]: return self._taxi_test_data.test_column_names @abstractmethod - def test_cases(self) -> List[TaxiPartitioningTestCase]: + def test_cases(self) -> list[TaxiPartitioningTestCase]: pass class TaxiPartitioningTestCasesWholeTable(TaxiPartitioningTestCasesBase): @override - def test_cases(self) -> List[TaxiPartitioningTestCase]: + def test_cases(self) -> list[TaxiPartitioningTestCase]: return [ TaxiPartitioningTestCase( table_domain_test_case=True, @@ -280,7 +280,7 @@ def test_cases(self) -> List[TaxiPartitioningTestCase]: class TaxiPartitioningTestCasesDateTime(TaxiPartitioningTestCasesBase): @override - def test_cases(self) -> List[TaxiPartitioningTestCase]: + def test_cases(self) -> list[TaxiPartitioningTestCase]: return [ TaxiPartitioningTestCase( table_domain_test_case=False, diff --git a/tests/integration/fixtures/partition_and_sample_data/sampler_test_cases_and_fixtures.py b/tests/integration/fixtures/partition_and_sample_data/sampler_test_cases_and_fixtures.py index a49017dbb7ec..095c9dfa813c 100644 --- a/tests/integration/fixtures/partition_and_sample_data/sampler_test_cases_and_fixtures.py +++ b/tests/integration/fixtures/partition_and_sample_data/sampler_test_cases_and_fixtures.py @@ -1,7 +1,6 @@ """Test cases and fixtures for sampler integration test configurations.""" from dataclasses import dataclass -from typing import List import pandas as pd @@ -55,7 +54,7 @@ def test_df(self) -> pd.DataFrame: def test_column_name(self) -> str: return self._taxi_test_data.test_column_name - def test_cases(self) -> List[TaxiSamplingTestCase]: + def test_cases(self) -> list[TaxiSamplingTestCase]: return [ TaxiSamplingTestCase( sampling_method_name="sample_using_limit", diff --git a/tests/integration/integration_test_fixture.py b/tests/integration/integration_test_fixture.py index 5fd262387adc..8b3051b71d43 100644 --- a/tests/integration/integration_test_fixture.py +++ b/tests/integration/integration_test_fixture.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import List, Optional, Tuple +from typing import Optional from tests.integration.backend_dependencies import BackendDependencies @@ -25,8 +25,8 @@ class IntegrationTestFixture: name: str user_flow_script: str - backend_dependencies: List[BackendDependencies] + backend_dependencies: list[BackendDependencies] data_context_dir: Optional[str] = None data_dir: Optional[str] = None - other_files: Optional[Tuple[Tuple[str, str]]] = None + other_files: Optional[tuple[tuple[str, str]]] = None util_script: Optional[str] = None diff --git a/tests/integration/spark/test_spark_config.py b/tests/integration/spark/test_spark_config.py index b6d7d6053dfa..d31c2733d8c5 100644 --- a/tests/integration/spark/test_spark_config.py +++ b/tests/integration/spark/test_spark_config.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict, List +from typing import Any import pytest from packaging.version import Version @@ -48,7 +48,7 @@ def test_spark_config_datasource(spark_session_v012): assert not sc_stopped # Test that our values were set - conf: List[tuple] = spark_session.sparkContext.getConf().getAll() + conf: list[tuple] = spark_session.sparkContext.getConf().getAll() assert ("spark.app.name", name) in conf assert ("spark.sql.catalogImplementation", "hive") in conf assert ("spark.executor.memory", "768m") in conf @@ -56,7 +56,7 @@ def test_spark_config_datasource(spark_session_v012): def test_spark_config_execution_engine_block_config(spark_session): - new_spark_config: Dict[str, Any] = { + new_spark_config: dict[str, Any] = { "spark.app.name": "great_expectations-ee-config", "spark.sql.catalogImplementation": "hive", "spark.executor.memory": "512m", @@ -70,7 +70,7 @@ def test_spark_config_execution_engine_block_config(spark_session): assert not sc_stopped - current_spark_config: List[tuple] = execution_engine.spark.sparkContext.getConf().getAll() + current_spark_config: list[tuple] = execution_engine.spark.sparkContext.getConf().getAll() assert ("spark.sql.catalogImplementation", "hive") in current_spark_config assert ( "spark.app.name", diff --git a/tests/integration/test_definitions/abs/integration_tests.py b/tests/integration/test_definitions/abs/integration_tests.py index 5193ae1e70a8..19624650bc2a 100644 --- a/tests/integration/test_definitions/abs/integration_tests.py +++ b/tests/integration/test_definitions/abs/integration_tests.py @@ -1,11 +1,9 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture abs_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] partition_data = [ IntegrationTestFixture( @@ -22,7 +20,7 @@ ), ] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] fluent_datasources = [ IntegrationTestFixture( diff --git a/tests/integration/test_definitions/bigquery/integration_tests.py b/tests/integration/test_definitions/bigquery/integration_tests.py index 546a7f0d7321..5d5980daa976 100644 --- a/tests/integration/test_definitions/bigquery/integration_tests.py +++ b/tests/integration/test_definitions/bigquery/integration_tests.py @@ -1,11 +1,9 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture bigquery_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] partition_data = [ IntegrationTestFixture( @@ -38,7 +36,7 @@ ), ] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] deployment_patterns = [ IntegrationTestFixture( @@ -49,7 +47,7 @@ ), ] -runtime_parameters: List[IntegrationTestFixture] = [] +runtime_parameters: list[IntegrationTestFixture] = [] bigquery_integration_tests += connecting_to_your_data bigquery_integration_tests += partition_data diff --git a/tests/integration/test_definitions/gcs/integration_tests.py b/tests/integration/test_definitions/gcs/integration_tests.py index acd4001996fc..4dd0a6cae4e1 100644 --- a/tests/integration/test_definitions/gcs/integration_tests.py +++ b/tests/integration/test_definitions/gcs/integration_tests.py @@ -1,11 +1,9 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture gcs_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] how_to_configure_metadata_store = [ # Chetan - 20231117 - These have been commented out due to their reliance on the CLI (which has been deleted). # noqa: E501 @@ -47,7 +45,7 @@ ), ] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] deployment_patterns = [ IntegrationTestFixture( diff --git a/tests/integration/test_definitions/mssql/integration_tests.py b/tests/integration/test_definitions/mssql/integration_tests.py index 3232ef8cc18c..b9068bf003d8 100644 --- a/tests/integration/test_definitions/mssql/integration_tests.py +++ b/tests/integration/test_definitions/mssql/integration_tests.py @@ -1,5 +1,3 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture @@ -36,7 +34,7 @@ ), ] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] mssql_integration_tests += partition_data mssql_integration_tests += sample_data diff --git a/tests/integration/test_definitions/mysql/integration_tests.py b/tests/integration/test_definitions/mysql/integration_tests.py index 18f2a69a76a0..60bb1eea8389 100644 --- a/tests/integration/test_definitions/mysql/integration_tests.py +++ b/tests/integration/test_definitions/mysql/integration_tests.py @@ -1,11 +1,9 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture mysql_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] partition_data = [ IntegrationTestFixture( @@ -38,7 +36,7 @@ ), ] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] mysql_integration_tests += connecting_to_your_data mysql_integration_tests += partition_data diff --git a/tests/integration/test_definitions/postgresql/integration_tests.py b/tests/integration/test_definitions/postgresql/integration_tests.py index a5e81273af44..b0b65ef631a6 100644 --- a/tests/integration/test_definitions/postgresql/integration_tests.py +++ b/tests/integration/test_definitions/postgresql/integration_tests.py @@ -1,5 +1,3 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture @@ -47,7 +45,7 @@ ), ] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] fluent_datasources = [ diff --git a/tests/integration/test_definitions/s3/integration_tests.py b/tests/integration/test_definitions/s3/integration_tests.py index f929270ad4ce..d8d01ba447ab 100644 --- a/tests/integration/test_definitions/s3/integration_tests.py +++ b/tests/integration/test_definitions/s3/integration_tests.py @@ -1,11 +1,9 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture s3_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] deployment_patterns = [ IntegrationTestFixture( @@ -31,7 +29,7 @@ ), ] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] fluent_datasources = [ IntegrationTestFixture( diff --git a/tests/integration/test_definitions/snowflake/integration_tests.py b/tests/integration/test_definitions/snowflake/integration_tests.py index 225fe32c7ed8..76f88f55d1f1 100644 --- a/tests/integration/test_definitions/snowflake/integration_tests.py +++ b/tests/integration/test_definitions/snowflake/integration_tests.py @@ -1,11 +1,9 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture snowflake_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] partition_data = [ IntegrationTestFixture( @@ -38,7 +36,7 @@ ), ] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] snowflake_integration_tests += connecting_to_your_data snowflake_integration_tests += partition_data diff --git a/tests/integration/test_definitions/spark/integration_tests.py b/tests/integration/test_definitions/spark/integration_tests.py index 2586da6a41a9..d2d962ca9476 100644 --- a/tests/integration/test_definitions/spark/integration_tests.py +++ b/tests/integration/test_definitions/spark/integration_tests.py @@ -1,13 +1,11 @@ -from typing import List - from tests.integration.backend_dependencies import BackendDependencies from tests.integration.integration_test_fixture import IntegrationTestFixture spark_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] -databricks_deployment_patterns: List[IntegrationTestFixture] = [ +databricks_deployment_patterns: list[IntegrationTestFixture] = [ # unable to mock dbfs in CI # IntegrationTestFixture( # name="databricks_deployment_patterns_file_python_configs", diff --git a/tests/integration/test_definitions/sqlite/integration_tests.py b/tests/integration/test_definitions/sqlite/integration_tests.py index 91fa3d101150..d8b1342d8497 100644 --- a/tests/integration/test_definitions/sqlite/integration_tests.py +++ b/tests/integration/test_definitions/sqlite/integration_tests.py @@ -1,14 +1,12 @@ -from typing import List - from tests.integration.integration_test_fixture import IntegrationTestFixture sqlite_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] -partition_data: List[IntegrationTestFixture] = [] +partition_data: list[IntegrationTestFixture] = [] -sample_data: List[IntegrationTestFixture] = [] +sample_data: list[IntegrationTestFixture] = [] sqlite_integration_tests += connecting_to_your_data sqlite_integration_tests += partition_data diff --git a/tests/integration/test_definitions/trino/integration_tests.py b/tests/integration/test_definitions/trino/integration_tests.py index eea61de74826..4ad7dbbf4f1a 100644 --- a/tests/integration/test_definitions/trino/integration_tests.py +++ b/tests/integration/test_definitions/trino/integration_tests.py @@ -1,10 +1,8 @@ -from typing import List - from tests.integration.integration_test_fixture import IntegrationTestFixture trino_integration_tests = [] -connecting_to_your_data: List[IntegrationTestFixture] = [] +connecting_to_your_data: list[IntegrationTestFixture] = [] trino_integration_tests += connecting_to_your_data diff --git a/tests/integration/test_script_runner.py b/tests/integration/test_script_runner.py index 65d2489dd2c7..e346837ba3c8 100644 --- a/tests/integration/test_script_runner.py +++ b/tests/integration/test_script_runner.py @@ -10,7 +10,6 @@ import os import pathlib import shutil -from typing import List import pkg_resources import pytest @@ -90,7 +89,7 @@ def delay_rerun(*args): # to be populated by the smaller lists below -docs_test_matrix: List[IntegrationTestFixture] = [] +docs_test_matrix: list[IntegrationTestFixture] = [] local_tests = [ # IntegrationTestFixture( @@ -340,10 +339,10 @@ def delay_rerun(*args): docs_test_matrix += multiple_backend docs_test_matrix += failed_rows_tests -pandas_integration_tests: List[IntegrationTestFixture] = [] +pandas_integration_tests: list[IntegrationTestFixture] = [] # populate integration_test_matrix with sub-lists -integration_test_matrix: List[IntegrationTestFixture] = [] +integration_test_matrix: list[IntegrationTestFixture] = [] integration_test_matrix += pandas_integration_tests diff --git a/tests/integration/test_utils/data_source_config/base.py b/tests/integration/test_utils/data_source_config/base.py index d6a1107de1dc..8224a8a978ab 100644 --- a/tests/integration/test_utils/data_source_config/base.py +++ b/tests/integration/test_utils/data_source_config/base.py @@ -5,7 +5,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from functools import cached_property -from typing import TYPE_CHECKING, Dict, Generic, Optional, TypeVar, Union +from typing import TYPE_CHECKING, Generic, Optional, TypeVar, Union import great_expectations as gx from great_expectations.data_context.data_context.abstract_data_context import AbstractDataContext @@ -22,7 +22,7 @@ @dataclass(frozen=True) class DataSourceTestConfig(ABC, Generic[_ColumnTypes]): name: Optional[str] = None - column_types: Union[Dict[str, _ColumnTypes], None] = None + column_types: Union[dict[str, _ColumnTypes], None] = None @property @abstractmethod diff --git a/tests/integration/test_utils/data_source_config/postgres.py b/tests/integration/test_utils/data_source_config/postgres.py index ef2f81330fcc..7d8871a5759f 100644 --- a/tests/integration/test_utils/data_source_config/postgres.py +++ b/tests/integration/test_utils/data_source_config/postgres.py @@ -1,5 +1,5 @@ from random import randint -from typing import Dict, Union +from typing import Union import pandas as pd import pytest @@ -188,7 +188,7 @@ def teardown(self) -> None: if self.table is not None: self.table.drop(self.engine) - def get_column_types(self) -> Dict[str, PostgresColumnType]: + def get_column_types(self) -> dict[str, PostgresColumnType]: if self.config.column_types is None: return {} return self.config.column_types diff --git a/tests/integration/test_utils/data_source_config/snowflake.py b/tests/integration/test_utils/data_source_config/snowflake.py index 7b492f926d63..16c85262d802 100644 --- a/tests/integration/test_utils/data_source_config/snowflake.py +++ b/tests/integration/test_utils/data_source_config/snowflake.py @@ -1,5 +1,5 @@ from random import randint -from typing import TYPE_CHECKING, Dict, List, TypeVar, Union +from typing import TYPE_CHECKING, TypeVar, Union import pandas as pd import pytest @@ -106,8 +106,8 @@ def make_batch(self) -> Batch: @override def setup(self) -> None: - column_types: Dict[str, SnowflakeType] = self.get_column_types() - columns: List[Column] = [ + column_types: dict[str, SnowflakeType] = self.get_column_types() + columns: list[Column] = [ Column(name, column_type) for name, column_type in column_types.items() ] self.table = Table( @@ -131,7 +131,7 @@ def teardown(self) -> None: if self.table is not None: self.table.drop(self.engine) - def get_column_types(self) -> Dict[str, _SnowflakeType]: + def get_column_types(self) -> dict[str, _SnowflakeType]: if self.config.column_types is None: raise NotImplementedError("Column inference not implemented") else: diff --git a/tests/profile/conftest.py b/tests/profile/conftest.py index eb8e7ca7eccd..d837c6d59239 100644 --- a/tests/profile/conftest.py +++ b/tests/profile/conftest.py @@ -1,6 +1,5 @@ import os import shutil -from typing import Set, Tuple import pytest @@ -64,7 +63,7 @@ def possible_expectations_set(): def get_set_of_columns_and_expectations_from_suite( suite: ExpectationSuite, -) -> Tuple[Set[str], Set[str]]: +) -> tuple[set[str], set[str]]: """ Args: suite: An expectation suite @@ -72,11 +71,11 @@ def get_set_of_columns_and_expectations_from_suite( Returns: A tuple containing a set of columns and a set of expectations found in a suite """ - columns: Set[str] = { + columns: set[str] = { i.kwargs.get("column") # type: ignore[misc] for i in suite.expectation_configurations if i.kwargs.get("column") } - expectations: Set[str] = {i.type for i in suite.expectation_configurations} + expectations: set[str] = {i.type for i in suite.expectation_configurations} return columns, expectations diff --git a/tests/render/test_data_documentation_site_builder.py b/tests/render/test_data_documentation_site_builder.py index b68e319c3f1a..59d72ec725de 100644 --- a/tests/render/test_data_documentation_site_builder.py +++ b/tests/render/test_data_documentation_site_builder.py @@ -1,6 +1,5 @@ import os import shutil -from typing import Dict import pytest @@ -21,7 +20,7 @@ def assert_how_to_buttons( context, index_page_locator_info: str, - index_links_dict: Dict, + index_links_dict: dict, show_how_to_buttons=True, ): """Helper function to assert presence or non-presence of how-to buttons and related content in various diff --git a/tests/render/test_inline_renderer.py b/tests/render/test_inline_renderer.py index 6aef623f319d..bb262dab9f9e 100644 --- a/tests/render/test_inline_renderer.py +++ b/tests/render/test_inline_renderer.py @@ -1,5 +1,3 @@ -from typing import List - import pytest from great_expectations.core.expectation_suite import ExpectationSuite @@ -23,8 +21,8 @@ def clean_serialized_rendered_atomic_content_graphs( - serialized_rendered_atomic_content: List[dict], -) -> List[dict]: + serialized_rendered_atomic_content: list[dict], +) -> list[dict]: for content_block in serialized_rendered_atomic_content: if content_block["value_type"] == "GraphType": content_block["value"]["graph"].pop("$schema") @@ -392,11 +390,11 @@ def test_inline_renderer_expectation_validation_result_serialization( inline_renderer: InlineRenderer = InlineRenderer(render_object=expectation_validation_result) - expectation_validation_result_rendered_atomic_content: List[RenderedAtomicContent] = ( + expectation_validation_result_rendered_atomic_content: list[RenderedAtomicContent] = ( inline_renderer.get_rendered_content() ) - actual_serialized_expectation_validation_result_rendered_atomic_content: List[dict] = ( + actual_serialized_expectation_validation_result_rendered_atomic_content: list[dict] = ( clean_serialized_rendered_atomic_content_graphs( serialized_rendered_atomic_content=[ rendered_atomic_content.to_json_dict() @@ -726,11 +724,11 @@ def test_inline_renderer_expectation_configuration_serialization( ): inline_renderer: InlineRenderer = InlineRenderer(render_object=expectation_configuration) - expectation_configuration_rendered_atomic_content: List[RenderedAtomicContent] = ( + expectation_configuration_rendered_atomic_content: list[RenderedAtomicContent] = ( inline_renderer.get_rendered_content() ) - actual_serialized_expectation_configuration_rendered_atomic_content: List[dict] = ( + actual_serialized_expectation_configuration_rendered_atomic_content: list[dict] = ( clean_serialized_rendered_atomic_content_graphs( serialized_rendered_atomic_content=[ rendered_atomic_content.to_json_dict() diff --git a/tests/render/test_util.py b/tests/render/test_util.py index e17c82c693c0..5c4fcfbba759 100644 --- a/tests/render/test_util.py +++ b/tests/render/test_util.py @@ -1,5 +1,5 @@ import sys -from typing import List, Union +from typing import Union import pytest @@ -356,14 +356,14 @@ def test_convert_unexpected_indices_to_df_actual_values(): def test_truncate_list_of_indices(): - int_indices: List[Union[int, str]] = [4, 5, 6, 7] + int_indices: list[Union[int, str]] = [4, 5, 6, 7] result: str = truncate_list_of_indices(indices=int_indices) assert result == "4, 5, 6, 7" result: str = truncate_list_of_indices(indices=int_indices, max_index=2) assert result == "4, 5, ..." - str_indices: List[Union[int, str]] = ["four", "five", "six", "seven"] + str_indices: list[Union[int, str]] = ["four", "five", "six", "seven"] result: str = truncate_list_of_indices(indices=str_indices) assert result == "four, five, six, seven" diff --git a/tests/scripts/test_public_api_report.py b/tests/scripts/test_public_api_report.py index b254e40d6da9..83ac76a9237c 100644 --- a/tests/scripts/test_public_api_report.py +++ b/tests/scripts/test_public_api_report.py @@ -1,6 +1,6 @@ import ast import pathlib -from typing import List, Union +from typing import Union import pytest @@ -399,7 +399,7 @@ def test_get_all_public_api_definitions(self, public_api_checker: PublicAPICheck def _class_and_function_definitions( self, tree: ast.AST - ) -> List[Union[ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef]]: + ) -> list[Union[ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef]]: """Helper function to find class and function definitions from ast tree for tests.""" definitions = [] for node in ast.walk(tree): @@ -884,7 +884,7 @@ def test_instantiate(self, public_api_report: PublicAPIReport): assert isinstance(public_api_report, PublicAPIReport) def test_generate_printable_definitions(self, public_api_report: PublicAPIReport): - expected: List[str] = [ + expected: list[str] = [ "File: great_expectations/sample_with_definitions_python_file_string.py Name: " "ExampleClass", "File: great_expectations/sample_with_definitions_python_file_string.py Name: " @@ -904,7 +904,7 @@ def test_generate_printable_definitions(self, public_api_report: PublicAPIReport def test_generate_printable_definitions_exclude_by_file( self, public_api_report_filter_out_file: PublicAPIReport ): - expected: List[str] = [] + expected: list[str] = [] observed = public_api_report_filter_out_file.generate_printable_definitions() assert observed == expected diff --git a/tests/sqlalchemy_test_doubles.py b/tests/sqlalchemy_test_doubles.py index c4c9d1f48f03..d92666662a1d 100644 --- a/tests/sqlalchemy_test_doubles.py +++ b/tests/sqlalchemy_test_doubles.py @@ -1,14 +1,14 @@ from __future__ import annotations from contextlib import contextmanager -from typing import Any, Dict, List +from typing import Any class MockSaInspector: - def get_columns(self) -> List[Dict[str, Any]]: # type: ignore[empty-body] + def get_columns(self) -> list[dict[str, Any]]: # type: ignore[empty-body] ... - def get_schema_names(self) -> List[str]: # type: ignore[empty-body] + def get_schema_names(self) -> list[str]: # type: ignore[empty-body] ... def get_table_names(self, schema: str | None): ... diff --git a/tests/test_deprecation.py b/tests/test_deprecation.py index 4f874981f3e2..c04f6f57cb00 100644 --- a/tests/test_deprecation.py +++ b/tests/test_deprecation.py @@ -1,6 +1,6 @@ import glob import re -from typing import List, Pattern, Tuple +from typing import Pattern import pytest from packaging import version @@ -20,8 +20,8 @@ def regex_for_deprecation_comments() -> Pattern: @pytest.fixture -def files_with_deprecation_warnings() -> List[str]: - files: List[str] = glob.glob( # noqa: PTH207 +def files_with_deprecation_warnings() -> list[str]: + files: list[str] = glob.glob( # noqa: PTH207 "great_expectations/**/*.py", recursive=True ) files_to_exclude = [ @@ -41,7 +41,7 @@ def files_with_deprecation_warnings() -> List[str]: @pytest.mark.unit def test_deprecation_warnings_are_accompanied_by_appropriate_comment( regex_for_deprecation_comments: Pattern, - files_with_deprecation_warnings: List[str], + files_with_deprecation_warnings: list[str], ): """ What does this test do and why? @@ -55,7 +55,7 @@ def test_deprecation_warnings_are_accompanied_by_appropriate_comment( with open(file) as f: contents = f.read() - matches: List[str] = regex_for_deprecation_comments.findall(contents) + matches: list[str] = regex_for_deprecation_comments.findall(contents) warning_count: int = contents.count("DeprecationWarning") assert ( len(matches) == warning_count @@ -65,7 +65,7 @@ def test_deprecation_warnings_are_accompanied_by_appropriate_comment( @pytest.mark.unit def test_deprecation_warnings_have_been_removed_after_two_minor_versions( regex_for_deprecation_comments: Pattern, - files_with_deprecation_warnings: List[str], + files_with_deprecation_warnings: list[str], ): """ What does this test do and why? @@ -84,12 +84,12 @@ def test_deprecation_warnings_have_been_removed_after_two_minor_versions( current_major_version: int = current_parsed_version.major current_minor_version: int = current_parsed_version.minor - unneeded_deprecation_warnings: List[Tuple[str, str]] = [] + unneeded_deprecation_warnings: list[tuple[str, str]] = [] for file in files_with_deprecation_warnings: with open(file) as f: contents = f.read() - matches: List[str] = regex_for_deprecation_comments.findall(contents) + matches: list[str] = regex_for_deprecation_comments.findall(contents) for match in matches: parsed_version: version.Version = version.parse(match) major_version: int = parsed_version.major @@ -97,7 +97,7 @@ def test_deprecation_warnings_have_been_removed_after_two_minor_versions( if (current_major_version - major_version > 0) and ( current_minor_version - minor_version > 2 ): - unneeded_deprecation_warning: Tuple[str, str] = (file, match) + unneeded_deprecation_warning: tuple[str, str] = (file, match) unneeded_deprecation_warnings.append(unneeded_deprecation_warning) if unneeded_deprecation_warnings: diff --git a/tests/test_packaging.py b/tests/test_packaging.py index b7bcb452c62d..3318e5d5e78d 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -3,7 +3,7 @@ import os.path import pathlib from pprint import pformat as pf -from typing import Final, List +from typing import Final import pytest import requirements as rp @@ -11,7 +11,7 @@ IGNORE_PINS: Final[set[str]] = {"mypy", "ruff", "pytest"} -def collect_requirements_files() -> List[pathlib.Path]: +def collect_requirements_files() -> list[pathlib.Path]: project_root = pathlib.Path(__file__).parents[1] assert project_root.exists() reqs_dir = project_root.joinpath("reqs") diff --git a/tests/test_utils.py b/tests/test_utils.py index 88d012aa4038..ea52b7af1226 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,7 +4,7 @@ import warnings from contextlib import contextmanager from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple, Union, cast +from typing import Optional, Union, cast import pandas as pd @@ -49,7 +49,7 @@ def safe_remove(path): def create_files_in_directory( - directory: str, file_name_list: List[str], file_content_fn=lambda: "x,y\n1,2\n2,3" + directory: str, file_name_list: list[str], file_content_fn=lambda: "x,y\n1,2\n2,3" ): subdirectories = [] for file_name in file_name_list: @@ -412,9 +412,9 @@ class LoadedTable: def load_and_concatenate_csvs( - csv_paths: List[str], + csv_paths: list[str], load_full_dataset: bool = False, - convert_column_names_to_datetime: Optional[List[str]] = None, + convert_column_names_to_datetime: Optional[list[str]] = None, ) -> pd.DataFrame: """Utility method that is used in loading test data into a pandas dataframe. @@ -434,7 +434,7 @@ def load_and_concatenate_csvs( import pandas as pd - dfs: List[pd.DataFrame] = [] + dfs: list[pd.DataFrame] = [] for csv_path in csv_paths: df = pd.read_csv(csv_path) convert_string_columns_to_datetime( @@ -452,7 +452,7 @@ def load_and_concatenate_csvs( def convert_string_columns_to_datetime( - df: pd.DataFrame, column_names_to_convert: Optional[List[str]] = None + df: pd.DataFrame, column_names_to_convert: Optional[list[str]] = None ) -> None: """ Converts specified columns (e.g., "pickup_datetime" and "dropoff_datetime") to datetime column type. @@ -471,9 +471,9 @@ def load_data_into_test_database( # noqa: C901, PLR0912, PLR0915 connection_string: str, schema_name: Optional[str] = None, csv_path: Optional[str] = None, - csv_paths: Optional[List[str]] = None, + csv_paths: Optional[list[str]] = None, load_full_dataset: bool = False, - convert_colnames_to_datetime: Optional[List[str]] = None, + convert_colnames_to_datetime: Optional[list[str]] = None, random_table_suffix: bool = False, to_sql_method: Optional[str] = None, drop_existing_table: bool = True, @@ -658,7 +658,7 @@ def load_dataframe_into_test_athena_database_as_table( ) -def clean_up_tables_with_prefix(connection_string: str, table_prefix: str) -> List[str]: +def clean_up_tables_with_prefix(connection_string: str, table_prefix: str) -> list[str]: """Drop all tables starting with the provided table_prefix. Note: Uses private method InferredAssetSqlDataConnector._introspect_db() to get the table names to not duplicate code, but should be refactored in the @@ -676,8 +676,8 @@ def clean_up_tables_with_prefix(connection_string: str, table_prefix: str) -> Li ) introspection_output = introspect_db(execution_engine=execution_engine) - tables_to_drop: List[str] = [] - tables_dropped: List[str] = [] + tables_to_drop: list[str] = [] + tables_dropped: list[str] = [] for table in introspection_output: if table["table_name"].startswith(table_prefix): @@ -688,7 +688,7 @@ def clean_up_tables_with_prefix(connection_string: str, table_prefix: str) -> Li execution_engine.execute_query_in_transaction(sa.text(f"DROP TABLE IF EXISTS {table_name}")) tables_dropped.append(table_name) - tables_skipped: List[str] = list(set(tables_to_drop) - set(tables_dropped)) + tables_skipped: list[str] = list(set(tables_to_drop) - set(tables_dropped)) if len(tables_skipped) > 0: warnings.warn(f"Warning: Tables skipped: {tables_skipped}") @@ -699,10 +699,10 @@ def introspect_db( # noqa: C901, PLR0912 execution_engine: SqlAlchemyExecutionEngine, schema_name: Union[str, None] = None, ignore_information_schemas_and_system_tables: bool = True, - information_schemas: Optional[List[str]] = None, - system_tables: Optional[List[str]] = None, + information_schemas: Optional[list[str]] = None, + system_tables: Optional[list[str]] = None, include_views=True, -) -> List[Dict[str, str]]: +) -> list[dict[str, str]]: # This code was broken out from the InferredAssetSqlDataConnector when it was removed if information_schemas is None: information_schemas = [ @@ -721,8 +721,8 @@ def introspect_db( # noqa: C901, PLR0912 selected_schema_name = schema_name - tables: List[Dict[str, str]] = [] - all_schema_names: List[str] = inspector.get_schema_names() + tables: list[dict[str, str]] = [] + all_schema_names: list[str] = inspector.get_schema_names() for schema in all_schema_names: if ignore_information_schemas_and_system_tables and schema_name in information_schemas: continue @@ -730,7 +730,7 @@ def introspect_db( # noqa: C901, PLR0912 if selected_schema_name is not None and schema_name != selected_schema_name: continue - table_names: List[str] = inspector.get_table_names(schema=schema) + table_names: list[str] = inspector.get_table_names(schema=schema) for table_name in table_names: if ignore_information_schemas_and_system_tables and (table_name in system_tables): continue @@ -915,7 +915,7 @@ def get_awsathena_connection_url(db_name_env_var: str = "ATHENA_DB_NAME") -> str def get_connection_string_and_dialect( athena_db_name_env_var: str = "ATHENA_DB_NAME", -) -> Tuple[str, str]: +) -> tuple[str, str]: with open("./connection_string.yml") as f: db_config: dict = yaml_handler.load(f) diff --git a/tests/validator/test_validation_graph.py b/tests/validator/test_validation_graph.py index 1aca3cb4610e..f97658cf54af 100644 --- a/tests/validator/test_validation_graph.py +++ b/tests/validator/test_validation_graph.py @@ -1,6 +1,6 @@ import sys import uuid -from typing import Dict, Iterable, Optional, Set, Tuple, Union, cast +from typing import Iterable, Optional, Union, cast from unittest import mock import pytest @@ -32,9 +32,9 @@ class PandasExecutionEngineFake: @staticmethod def resolve_metrics( metrics_to_resolve: Iterable[MetricConfiguration], - metrics: Optional[Dict[Tuple[str, str, str], MetricConfiguration]] = None, + metrics: Optional[dict[tuple[str, str, str], MetricConfiguration]] = None, runtime_configuration: Optional[dict] = None, - ) -> Dict[Tuple[str, str, str], MetricValue]: + ) -> dict[tuple[str, str, str], MetricValue]: """ This stub method implementation insures that specified "MetricConfiguration", designed to fail, will cause appropriate exception to be raised, while its dependencies resolve to actual values ("my_value" is used here @@ -294,7 +294,7 @@ def test_ExpectationValidationGraph_get_exception_info( def test_parse_validation_graph( expect_column_value_z_scores_to_be_less_than_expectation_validation_graph: ValidationGraph, ): - available_metrics: Dict[Tuple[str, str, str], MetricValue] + available_metrics: dict[tuple[str, str, str], MetricValue] # Parse input "ValidationGraph" object and confirm the numbers of ready and still needed metrics. # noqa: E501 available_metrics = {} @@ -382,10 +382,10 @@ def test_resolve_validation_graph_with_bad_config_catch_exceptions_true( runtime_configuration=runtime_configuration, ) - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] - aborted_metrics_info: Dict[ - Tuple[str, str, str], - Dict[str, Union[MetricConfiguration, Set[ExceptionInfo], int]], + resolved_metrics: dict[tuple[str, str, str], MetricValue] + aborted_metrics_info: dict[ + tuple[str, str, str], + dict[str, Union[MetricConfiguration, set[ExceptionInfo], int]], ] _resolved_metrics, aborted_metrics_info = graph.resolve( runtime_configuration=runtime_configuration, @@ -471,10 +471,10 @@ class DummyExecutionEngine: ) graph = ValidationGraph(execution_engine=execution_engine) - resolved_metrics: Dict[Tuple[str, str, str], MetricValue] - aborted_metrics_info: Dict[ - Tuple[str, str, str], - Dict[str, Union[MetricConfiguration, Set[ExceptionInfo], int]], + resolved_metrics: dict[tuple[str, str, str], MetricValue] + aborted_metrics_info: dict[ + tuple[str, str, str], + dict[str, Union[MetricConfiguration, set[ExceptionInfo], int]], ] # noinspection PyUnusedLocal _resolved_metrics, _aborted_metrics_info = graph.resolve(**call_args)