From fe1080c6be1ff17d0b41b39898e3ed98d843abed Mon Sep 17 00:00:00 2001 From: Santosh Chavala <76093335+chavalasantosh@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:00:36 -0700 Subject: [PATCH 1/2] Add cross-engine column outlier expectation --- great_expectations/expectations/__init__.py | 1 + .../expectations/core/__init__.py | 1 + ...expect_column_values_to_not_be_outliers.py | 228 +++++++++ .../ExpectColumnValuesToNotBeOutliers.json | 477 ++++++++++++++++++ .../metrics/column_map_metrics/__init__.py | 5 + .../column_values_not_outliers.py | 406 +++++++++++++++ tasks.py | 1 + ...expect_column_values_to_not_be_outliers.py | 67 +++ ...expect_column_values_to_not_be_outliers.py | 153 ++++++ 9 files changed, 1339 insertions(+) create mode 100644 great_expectations/expectations/core/expect_column_values_to_not_be_outliers.py create mode 100644 great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeOutliers.json create mode 100644 great_expectations/expectations/metrics/column_map_metrics/column_values_not_outliers.py create mode 100644 tests/expectations/core/test_expect_column_values_to_not_be_outliers.py create mode 100644 tests/integration/data_sources_and_expectations/expectations/test_expect_column_values_to_not_be_outliers.py diff --git a/great_expectations/expectations/__init__.py b/great_expectations/expectations/__init__.py index 3f98bc88dcff..e1628a5c29f2 100644 --- a/great_expectations/expectations/__init__.py +++ b/great_expectations/expectations/__init__.py @@ -40,6 +40,7 @@ ExpectColumnValuesToMatchStrftimeFormat, ExpectColumnValuesToNotBeInSet, ExpectColumnValuesToNotBeNull, + ExpectColumnValuesToNotBeOutliers, ExpectColumnValuesToNotMatchLikePattern, ExpectColumnValuesToNotMatchLikePatternList, ExpectColumnValuesToNotMatchRegex, diff --git a/great_expectations/expectations/core/__init__.py b/great_expectations/expectations/core/__init__.py index 36e28920ee33..f2db274b1f88 100644 --- a/great_expectations/expectations/core/__init__.py +++ b/great_expectations/expectations/core/__init__.py @@ -74,6 +74,7 @@ ) from .expect_column_values_to_not_be_in_set import ExpectColumnValuesToNotBeInSet from .expect_column_values_to_not_be_null import ExpectColumnValuesToNotBeNull +from .expect_column_values_to_not_be_outliers import ExpectColumnValuesToNotBeOutliers from .expect_column_values_to_not_match_like_pattern import ( ExpectColumnValuesToNotMatchLikePattern, ) diff --git a/great_expectations/expectations/core/expect_column_values_to_not_be_outliers.py b/great_expectations/expectations/core/expect_column_values_to_not_be_outliers.py new file mode 100644 index 000000000000..d5ed27189d73 --- /dev/null +++ b/great_expectations/expectations/core/expect_column_values_to_not_be_outliers.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Type, Union + +from great_expectations.compatibility import pydantic +from great_expectations.compatibility.typing_extensions import override +from great_expectations.core.suite_parameters import ( + SuiteParameterDict, # noqa: TC001 # FIXME CoP +) +from great_expectations.expectations.expectation import ColumnMapExpectation +from great_expectations.expectations.metadata_types import DataQualityIssues, SupportedDataSources +from great_expectations.expectations.model_field_descriptions import ( + COLUMN_DESCRIPTION, + FAILURE_SEVERITY_DESCRIPTION, + MOSTLY_DESCRIPTION, +) +from great_expectations.render.renderer_configuration import ( + RendererConfiguration, + RendererValueType, +) + +if TYPE_CHECKING: + from great_expectations.render.renderer_configuration import AddParamArgs + +EXPECTATION_SHORT_DESCRIPTION = "Expect numeric column values to not be statistical outliers." +METHOD_DESCRIPTION = ( + 'The outlier detection method: "iqr" uses the median and interquartile range; ' + '"std" uses the mean and sample standard deviation.' +) +MULTIPLIER_DESCRIPTION = "The threshold multiplier applied to the selected spread statistic." +DATA_QUALITY_ISSUES = [DataQualityIssues.NUMERIC.value] +SUPPORTED_DATA_SOURCES = [ + SupportedDataSources.PANDAS.value, + SupportedDataSources.SPARK.value, + SupportedDataSources.SQLITE.value, + SupportedDataSources.POSTGRESQL.value, + SupportedDataSources.AURORA.value, + SupportedDataSources.CITUS.value, + SupportedDataSources.ALLOY.value, + SupportedDataSources.NEON.value, + SupportedDataSources.MYSQL.value, + SupportedDataSources.SQL_SERVER.value, + SupportedDataSources.BIGQUERY.value, + SupportedDataSources.SNOWFLAKE.value, + SupportedDataSources.DATABRICKS.value, + SupportedDataSources.REDSHIFT.value, +] + + +class ExpectColumnValuesToNotBeOutliers(ColumnMapExpectation): + __doc__ = f"""{EXPECTATION_SHORT_DESCRIPTION} + + A value is considered an outlier when its absolute distance from the column center is + greater than or equal to the configured multiplier times the column spread. Null values + are excluded from both the aggregate statistics and row-level evaluation. + + ExpectColumnValuesToNotBeOutliers is a Column Map Expectation. + + Column Map Expectations are evaluated for a single column and ask a yes/no question for + every non-null row. The percentage of rows that satisfy the condition is compared with + the configured `mostly` value. + + Args: + column (str): \ + {COLUMN_DESCRIPTION} + + Keyword Args: + method (str): \ + {METHOD_DESCRIPTION} Default "iqr". + multiplier (float): \ + {MULTIPLIER_DESCRIPTION} Default 1.5. + + Other Parameters: + mostly (None or a float between 0 and 1): \ + {MOSTLY_DESCRIPTION} \ + For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly). Default 1. + result_format (str or None): \ + Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. \ + For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format). + catch_exceptions (boolean or None): \ + If True, catch exceptions and include them in the result. + meta (dict or None): \ + A JSON-serializable dictionary included in the result without modification. + severity (str or None): \ + {FAILURE_SEVERITY_DESCRIPTION} + + Returns: + An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result) + + Supported Data Sources: + [{SUPPORTED_DATA_SOURCES[0]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[1]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[2]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[3]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[4]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[5]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[6]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[7]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[8]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[9]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[10]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[11]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[12]}](https://docs.greatexpectations.io/docs/application_integration_support/) + [{SUPPORTED_DATA_SOURCES[13]}](https://docs.greatexpectations.io/docs/application_integration_support/) + + Data Quality Issues: + {DATA_QUALITY_ISSUES[0]} + + Example Data: + amount + 0 10 + 1 11 + 2 12 + 3 13 + 4 100 + + Code Examples: + Passing Case: + Input: + ExpectColumnValuesToNotBeOutliers( + column="amount", + method="std", + multiplier=3.0, + ) + + Failing Case: + Input: + ExpectColumnValuesToNotBeOutliers( + column="amount", + method="iqr", + multiplier=1.5, + ) + """ # noqa: E501 # FIXME CoP + + method: Union[str, SuiteParameterDict] = pydantic.Field( + default="iqr", + description=METHOD_DESCRIPTION, + ) + multiplier: Union[float, SuiteParameterDict] = pydantic.Field( + default=1.5, + description=MULTIPLIER_DESCRIPTION, + ) + + library_metadata: ClassVar[Dict[str, Union[str, list, bool]]] = { + "maturity": "production", + "tags": ["core expectation", "column map expectation", "outlier detection"], + "contributors": [ + "@chavalasantosh", + "@rexboyce", + "@lodeous", + "@bragleg", + ], + "requirements": [], + "has_full_test_suite": True, + "manually_reviewed_code": True, + } + _library_metadata = library_metadata + + map_metric = "column_values.not_outliers" + success_keys = ("mostly", "method", "multiplier") + args_keys = ("column",) + + class Config: + title = "Expect column values to not be outliers" + + @staticmethod + def schema_extra( + schema: Dict[str, Any], model: Type[ExpectColumnValuesToNotBeOutliers] + ) -> None: + ColumnMapExpectation.Config.schema_extra(schema, model) + schema["properties"]["metadata"]["properties"].update( + { + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": DATA_QUALITY_ISSUES, + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": model._library_metadata, + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": EXPECTATION_SHORT_DESCRIPTION, + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": SUPPORTED_DATA_SOURCES, + }, + } + ) + + @override + @classmethod + def _prescriptive_template( + cls, + renderer_configuration: RendererConfiguration, + ) -> RendererConfiguration: + add_param_args: AddParamArgs = ( + ("column", RendererValueType.STRING), + ("method", RendererValueType.STRING), + ("multiplier", RendererValueType.NUMBER), + ("mostly", RendererValueType.NUMBER), + ) + for name, param_type in add_param_args: + renderer_configuration.add_param(name=name, param_type=param_type) + + template_str = ( + "values must not be statistical outliers using the $method method " + "with a multiplier of $multiplier" + ) + params = renderer_configuration.params + if params.mostly and params.mostly.value < 1.0: + renderer_configuration = cls._add_mostly_pct_param( + renderer_configuration=renderer_configuration + ) + template_str += ", at least $mostly_pct % of the time." + else: + template_str += "." + + if renderer_configuration.include_column_name: + template_str = f"$column {template_str}" + + renderer_configuration.template_str = template_str + return renderer_configuration diff --git a/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeOutliers.json b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeOutliers.json new file mode 100644 index 000000000000..ce72b7aa734e --- /dev/null +++ b/great_expectations/expectations/core/schemas/ExpectColumnValuesToNotBeOutliers.json @@ -0,0 +1,477 @@ +{ + "title": "Expect column values to not be outliers", + "description": "Expect numeric column values to not be statistical outliers.\n\nA value is considered an outlier when its absolute distance from the column center is\ngreater than or equal to the configured multiplier times the column spread. Null values\nare excluded from both the aggregate statistics and row-level evaluation.\n\nExpectColumnValuesToNotBeOutliers is a Column Map Expectation.\n\nColumn Map Expectations are evaluated for a single column and ask a yes/no question for\nevery non-null row. The percentage of rows that satisfy the condition is compared with\nthe configured `mostly` value.\n\nArgs:\n column (str): The column name.\n\nKeyword Args:\n method (str): The outlier detection method: \"iqr\" uses the median and interquartile range; \"std\" uses the mean and sample standard deviation. Default \"iqr\".\n multiplier (float): The threshold multiplier applied to the selected spread statistic. Default 1.5.\n\nOther Parameters:\n mostly (None or a float between 0 and 1): Successful if at least `mostly` fraction of values match the Expectation. For more detail, see [mostly](https://docs.greatexpectations.io/docs/reference/expectations/standard_arguments/#mostly). Default 1.\n result_format (str or None): Which output mode to use: BOOLEAN_ONLY, BASIC, COMPLETE, or SUMMARY. For more detail, see [result_format](https://docs.greatexpectations.io/docs/reference/expectations/result_format).\n catch_exceptions (boolean or None): If True, catch exceptions and include them in the result.\n meta (dict or None): A JSON-serializable dictionary included in the result without modification.\n severity (str or None): The impact of this Expectation failing: critical, warning, or info. Defaults to critical if not set. Severity levels can be used to trigger different alerting patterns and actions.\n\nReturns:\n An [ExpectationSuiteValidationResult](https://docs.greatexpectations.io/docs/terms/validation_result)\n\nSupported Data Sources:\n [Pandas](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Spark](https://docs.greatexpectations.io/docs/application_integration_support/)\n [SQLite](https://docs.greatexpectations.io/docs/application_integration_support/)\n [PostgreSQL](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Amazon Aurora PostgreSQL](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Citus](https://docs.greatexpectations.io/docs/application_integration_support/)\n [AlloyDB](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Neon](https://docs.greatexpectations.io/docs/application_integration_support/)\n [MySQL](https://docs.greatexpectations.io/docs/application_integration_support/)\n [SQL Server](https://docs.greatexpectations.io/docs/application_integration_support/)\n [BigQuery](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Snowflake](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Databricks (SQL)](https://docs.greatexpectations.io/docs/application_integration_support/)\n [Redshift](https://docs.greatexpectations.io/docs/application_integration_support/)\n\nData Quality Issues:\n Numeric\n\nExample Data:\n amount\n 0 10\n 1 11\n 2 12\n 3 13\n 4 100\n\nCode Examples:\n Passing Case:\n Input:\n ExpectColumnValuesToNotBeOutliers(\n column=\"amount\",\n method=\"std\",\n multiplier=3.0,\n )\n\n Failing Case:\n Input:\n ExpectColumnValuesToNotBeOutliers(\n column=\"amount\",\n method=\"iqr\",\n multiplier=1.5,\n )", + "type": "object", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "meta": { + "title": "Meta", + "type": "object" + }, + "notes": { + "title": "Notes", + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "result_format": { + "title": "Result Format", + "default": "BASIC", + "anyOf": [ + { + "$ref": "#/definitions/ResultFormat" + }, + { + "type": "object" + } + ] + }, + "description": { + "title": "Description", + "description": "A short description of your Expectation", + "type": "string" + }, + "catch_exceptions": { + "title": "Catch Exceptions", + "default": true, + "type": "boolean" + }, + "rendered_content": { + "title": "Rendered Content", + "type": "array", + "items": { + "type": "object" + } + }, + "severity": { + "description": "Indicate the impact of this Expectation failing. Severity levels can be used to trigger different alerting patterns and actions.", + "default": "critical", + "allOf": [ + { + "$ref": "#/definitions/FailureSeverity" + } + ] + }, + "windows": { + "title": "Windows", + "description": "Definition(s) for evaluation of temporal windows", + "type": "array", + "items": { + "$ref": "#/definitions/Window" + } + }, + "batch_id": { + "title": "Batch Id", + "type": "string" + }, + "column": { + "title": "Column", + "description": "The column name.", + "minLength": 1, + "type": "string" + }, + "mostly": { + "title": "Mostly", + "description": "Successful if at least `mostly` fraction of values match the Expectation.", + "default": 1, + "anyOf": [ + { + "type": "number", + "minimum": 0.0, + "maximum": 1.0 + }, + { + "type": "object" + } + ], + "multipleOf": 0.01 + }, + "row_condition": { + "title": "Row Condition", + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/ComparisonCondition" + }, + { + "$ref": "#/definitions/NullityCondition" + }, + { + "$ref": "#/definitions/AndCondition" + }, + { + "$ref": "#/definitions/OrCondition" + }, + { + "$ref": "#/definitions/PassThroughCondition" + } + ] + }, + "condition_parser": { + "title": "Condition Parser", + "enum": [ + "great_expectations", + "great_expectations__experimental__", + "pandas", + "spark" + ], + "type": "string" + }, + "method": { + "title": "Method", + "description": "The outlier detection method: \"iqr\" uses the median and interquartile range; \"std\" uses the mean and sample standard deviation.", + "default": "iqr", + "anyOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] + }, + "multiplier": { + "title": "Multiplier", + "description": "The threshold multiplier applied to the selected spread statistic.", + "default": 1.5, + "anyOf": [ + { + "type": "number" + }, + { + "type": "object" + } + ] + }, + "metadata": { + "type": "object", + "properties": { + "expectation_class": { + "title": "Expectation Class", + "type": "string", + "const": "ExpectColumnValuesToNotBeOutliers" + }, + "expectation_type": { + "title": "Expectation Type", + "type": "string", + "const": "expect_column_values_to_not_be_outliers" + }, + "domain_type": { + "title": "Domain Type", + "type": "string", + "const": "column", + "description": "Column Map" + }, + "data_quality_issues": { + "title": "Data Quality Issues", + "type": "array", + "const": [ + "Numeric" + ] + }, + "library_metadata": { + "title": "Library Metadata", + "type": "object", + "const": { + "maturity": "production", + "tags": [ + "core expectation", + "column map expectation", + "outlier detection" + ], + "contributors": [ + "@chavalasantosh", + "@rexboyce", + "@lodeous", + "@bragleg" + ], + "requirements": [], + "has_full_test_suite": true, + "manually_reviewed_code": true + } + }, + "short_description": { + "title": "Short Description", + "type": "string", + "const": "Expect numeric column values to not be statistical outliers." + }, + "supported_data_sources": { + "title": "Supported Data Sources", + "type": "array", + "const": [ + "Pandas", + "Spark", + "SQLite", + "PostgreSQL", + "Amazon Aurora PostgreSQL", + "Citus", + "AlloyDB", + "Neon", + "MySQL", + "SQL Server", + "BigQuery", + "Snowflake", + "Databricks (SQL)", + "Redshift" + ] + } + } + } + }, + "required": [ + "column" + ], + "additionalProperties": false, + "definitions": { + "ResultFormat": { + "title": "ResultFormat", + "description": "An enumeration.", + "enum": [ + "BOOLEAN_ONLY", + "BASIC", + "COMPLETE", + "SUMMARY" + ], + "type": "string" + }, + "FailureSeverity": { + "title": "FailureSeverity", + "description": "Severity levels for Expectation failures.", + "enum": [ + "critical", + "warning", + "info" + ], + "type": "string" + }, + "Offset": { + "title": "Offset", + "description": "A threshold in which a metric will be considered passable", + "type": "object", + "properties": { + "positive": { + "title": "Positive", + "type": "number" + }, + "negative": { + "title": "Negative", + "type": "number" + } + }, + "required": [ + "positive", + "negative" + ], + "additionalProperties": false + }, + "Window": { + "title": "Window", + "description": "A definition for a temporal window across <`range`> number of previous invocations", + "type": "object", + "properties": { + "constraint_fn": { + "title": "Constraint Fn", + "type": "string" + }, + "parameter_name": { + "title": "Parameter Name", + "type": "string" + }, + "range": { + "title": "Range", + "type": "integer" + }, + "offset": { + "$ref": "#/definitions/Offset" + }, + "strict": { + "title": "Strict", + "default": false, + "type": "boolean" + } + }, + "required": [ + "constraint_fn", + "parameter_name", + "range", + "offset" + ], + "additionalProperties": false + }, + "Column": { + "title": "Column", + "description": "--Public API--\nSpecify the column in a condition statement.", + "type": "object", + "properties": { + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "name" + ] + }, + "Operator": { + "title": "Operator", + "description": "An enumeration.", + "enum": [ + "==", + "!=", + "<", + "<=", + ">", + ">=", + "IN", + "NOT_IN" + ], + "type": "string" + }, + "ComparisonCondition": { + "title": "ComparisonCondition", + "description": "--Public API--Condition representing the comparison of a column with a parameter.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "comparison", + "enum": [ + "comparison" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "operator": { + "$ref": "#/definitions/Operator" + }, + "parameter": { + "title": "Parameter" + } + }, + "required": [ + "column", + "operator", + "parameter" + ] + }, + "NullityCondition": { + "title": "NullityCondition", + "description": "--Public API--Condition representing the whether or not a column is null.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "nullity", + "enum": [ + "nullity" + ], + "type": "string" + }, + "column": { + "$ref": "#/definitions/Column" + }, + "is_null": { + "title": "Is Null", + "type": "boolean" + } + }, + "required": [ + "column", + "is_null" + ] + }, + "Condition": { + "title": "Condition", + "description": "Base class for conditions.", + "type": "object", + "properties": {} + }, + "AndCondition": { + "title": "AndCondition", + "description": "--Public API--Represents an AND condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "and", + "enum": [ + "and" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "OrCondition": { + "title": "OrCondition", + "description": "--Public API--Represents an OR condition composed of multiple conditions.", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "or", + "enum": [ + "or" + ], + "type": "string" + }, + "conditions": { + "title": "Conditions", + "type": "array", + "items": { + "$ref": "#/definitions/Condition" + } + } + }, + "required": [ + "conditions" + ] + }, + "PassThroughCondition": { + "title": "PassThroughCondition", + "description": "Condition that passes a filter string directly to the execution engine.\n\nThis is used for legacy pandas/spark condition_parser syntax where the\nrow_condition string is passed directly to DataFrame.query() or DataFrame.filter().", + "type": "object", + "properties": { + "type": { + "title": "Type", + "default": "pass_through", + "enum": [ + "pass_through" + ], + "type": "string" + }, + "pass_through_filter": { + "title": "Pass Through Filter", + "type": "string" + } + }, + "required": [ + "pass_through_filter" + ] + } + } +} diff --git a/great_expectations/expectations/metrics/column_map_metrics/__init__.py b/great_expectations/expectations/metrics/column_map_metrics/__init__.py index 9d6c765c0400..eeb438506757 100644 --- a/great_expectations/expectations/metrics/column_map_metrics/__init__.py +++ b/great_expectations/expectations/metrics/column_map_metrics/__init__.py @@ -20,6 +20,11 @@ ) from .column_values_not_match_regex import ColumnValuesNotMatchRegex from .column_values_not_match_regex_list import ColumnValuesNotMatchRegexList +from .column_values_not_outliers import ( + ColumnOutlierCenter, + ColumnOutlierSpread, + ColumnValuesNotOutliers, +) from .column_values_null import ColumnValuesNull from .column_values_of_type import ColumnValuesOfType from .column_values_unique import ColumnValuesUnique diff --git a/great_expectations/expectations/metrics/column_map_metrics/column_values_not_outliers.py b/great_expectations/expectations/metrics/column_map_metrics/column_values_not_outliers.py new file mode 100644 index 000000000000..5d9d9efb6309 --- /dev/null +++ b/great_expectations/expectations/metrics/column_map_metrics/column_values_not_outliers.py @@ -0,0 +1,406 @@ +from __future__ import annotations + +import math +from typing import TYPE_CHECKING, Any, Optional + +from great_expectations.compatibility.pyspark import functions as F +from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa +from great_expectations.compatibility.typing_extensions import override +from great_expectations.core.metric_domain_types import MetricDomainTypes +from great_expectations.core.metric_function_types import ( + MetricPartialFunctionTypeSuffixes, +) +from great_expectations.execution_engine import ( + ExecutionEngine, + PandasExecutionEngine, + SparkDFExecutionEngine, + SqlAlchemyExecutionEngine, +) +from great_expectations.execution_engine.sqlalchemy_dialect import GXSqlDialect +from great_expectations.expectations.metrics.column_aggregate_metric_provider import ( + ColumnAggregateMetricProvider, + column_aggregate_partial, + column_aggregate_value, +) +from great_expectations.expectations.metrics.map_metric_provider import ( + ColumnMapMetricProvider, + column_condition_partial, +) +from great_expectations.expectations.metrics.metric_provider import metric_value +from great_expectations.validator.metric_configuration import MetricConfiguration + +if TYPE_CHECKING: + import pandas as pd + + from great_expectations.expectations.expectation_configuration import ( + ExpectationConfiguration, + ) + +_IQR_METHOD = "iqr" +_STD_METHOD = "std" +_SUPPORTED_METHODS = (_IQR_METHOD, _STD_METHOD) +_SPARK_PERCENTILE_ACCURACY = 100_000 +_MINIMUM_SAMPLE_SIZE_FOR_STANDARD_DEVIATION = 2 + + +def _validate_method(method: str) -> None: + if method not in _SUPPORTED_METHODS: + raise NotImplementedError(f"method {method!r} has not been implemented") + + +def _is_missing_statistic(value: Any) -> bool: + if value is None: + return True + try: + return math.isnan(float(value)) + except (TypeError, ValueError): + return False + + +def _get_sql_compute_domain( + execution_engine: SqlAlchemyExecutionEngine, + metric_domain_kwargs: dict, +): + nonnull_domain_kwargs = execution_engine.add_column_row_condition(metric_domain_kwargs) + selectable, _, accessor_domain_kwargs = execution_engine.get_compute_domain( + nonnull_domain_kwargs, + domain_type=MetricDomainTypes.COLUMN, + ) + if isinstance(selectable, sa.sql.Select): + selectable = selectable.subquery() + return selectable, sa.column(accessor_domain_kwargs["column"]) + + +def _get_window_linear_percentiles( + *, + column, + quantiles: tuple[float, ...], + selectable, + execution_engine: SqlAlchemyExecutionEngine, +) -> tuple[Optional[float], ...]: + """Calculate continuous percentiles for SQL dialects without PERCENTILE_CONT.""" + value_label = "_gx_outlier_value" + row_number_label = "_gx_outlier_row_number" + count_label = "_gx_outlier_count" + + ordered_values = ( + sa.select( + column.label(value_label), + sa.func.row_number().over(order_by=column.asc()).label(row_number_label), + sa.func.count(column).over().label(count_label), + ) + .where(column.is_not(None)) + .select_from(selectable) + .subquery() + ) + + value = sa.cast(ordered_values.c[value_label], sa.Float) + row_number = ordered_values.c[row_number_label] + row_count = ordered_values.c[count_label] + + percentile_expressions = [] + for index, quantile in enumerate(quantiles): + position = quantile * (row_count - 1) + 1 + aggregate_position = sa.func.max(position) + lower_value = sa.func.max(sa.case((row_number <= position, value), else_=None)) + upper_value = sa.func.min(sa.case((row_number >= position, value), else_=None)) + interpolation_fraction = aggregate_position - sa.cast(aggregate_position, sa.Integer) + percentile_expressions.append( + (lower_value + interpolation_fraction * (upper_value - lower_value)).label( + f"_gx_outlier_quantile_{index}" + ) + ) + + row = execution_engine.execute_query( + sa.select(*percentile_expressions).select_from(ordered_values) + ).fetchone() + if row is None: + return tuple(None for _ in quantiles) + return tuple(None if value is None else float(value) for value in row) + + +def _get_sql_percentiles( + *, + column, + quantiles: tuple[float, ...], + selectable, + execution_engine: SqlAlchemyExecutionEngine, +) -> tuple[Optional[float], ...]: + dialect_name = execution_engine.dialect_name + percentile_expressions: list[Any] + + if dialect_name in (GXSqlDialect.SQLITE, GXSqlDialect.MYSQL): + return _get_window_linear_percentiles( + column=column, + quantiles=quantiles, + selectable=selectable, + execution_engine=execution_engine, + ) + + if dialect_name == GXSqlDialect.SQL_SERVER: + percentile_expressions = [ + sa.func.percentile_cont(quantile).within_group(column.asc()).over() + for quantile in quantiles + ] + query = sa.select(*percentile_expressions).select_from(selectable).limit(1) + elif dialect_name == GXSqlDialect.BIGQUERY: + percentile_expressions = [ + sa.func.percentile_cont(column, quantile).over() for quantile in quantiles + ] + query = sa.select(*percentile_expressions).select_from(selectable).limit(1) + elif dialect_name in ( + GXSqlDialect.POSTGRESQL, + GXSqlDialect.REDSHIFT, + GXSqlDialect.SNOWFLAKE, + GXSqlDialect.DATABRICKS, + ): + percentile_expressions = [ + sa.func.percentile_cont(quantile).within_group(column.asc()) for quantile in quantiles + ] + query = sa.select(*percentile_expressions).select_from(selectable) + else: + raise NotImplementedError( + f"IQR outlier detection is not implemented for SQL dialect {dialect_name!r}" + ) + + row = execution_engine.execute_query(query).fetchone() + if row is None: + return tuple(None for _ in quantiles) + return tuple(None if value is None else float(value) for value in row) + + +def _get_sql_standard_deviation_statistics( + *, + column, + selectable, + execution_engine: SqlAlchemyExecutionEngine, +) -> tuple[Optional[float], Optional[float]]: + numeric_column = sa.cast(column, sa.Float) + row = execution_engine.execute_query( + sa.select( + sa.func.count(column), + sa.func.avg(numeric_column), + sa.func.sum(numeric_column * numeric_column), + ).select_from(selectable) + ).fetchone() + + if row is None or row[0] == 0: + return None, None + + count = int(row[0]) + mean = float(row[1]) + if count < _MINIMUM_SAMPLE_SIZE_FOR_STANDARD_DEVIATION or row[2] is None: + return mean, None + + sum_of_squares = float(row[2]) + variance = (sum_of_squares - count * mean * mean) / (count - 1) + # Floating-point cancellation can produce a tiny negative value for a + # constant or near-constant column. + standard_deviation = math.sqrt(max(variance, 0.0)) + return mean, standard_deviation + + +def _get_sql_outlier_statistic( + *, + execution_engine: SqlAlchemyExecutionEngine, + metric_domain_kwargs: dict, + method: str, + statistic: str, +) -> Optional[float]: + _validate_method(method) + selectable, column = _get_sql_compute_domain( + execution_engine=execution_engine, + metric_domain_kwargs=metric_domain_kwargs, + ) + + if method == _STD_METHOD: + mean, standard_deviation = _get_sql_standard_deviation_statistics( + column=column, + selectable=selectable, + execution_engine=execution_engine, + ) + return mean if statistic == "center" else standard_deviation + + first_quartile, median, third_quartile = _get_sql_percentiles( + column=column, + quantiles=(0.25, 0.5, 0.75), + selectable=selectable, + execution_engine=execution_engine, + ) + if statistic == "center": + return median + if first_quartile is None or third_quartile is None: + return None + return third_quartile - first_quartile + + +class ColumnOutlierCenter(ColumnAggregateMetricProvider): + """Return the center used by the configured outlier detection method.""" + + metric_name = "column.outlier_center" + value_keys = ("method",) + filter_column_isnull = True + + @column_aggregate_value(engine=PandasExecutionEngine) + def _pandas(cls, column, method, **kwargs): + _validate_method(method) + if method == _IQR_METHOD: + return column.median() + return column.mean() + + @metric_value(engine=SqlAlchemyExecutionEngine) + def _sqlalchemy( + cls, + execution_engine: SqlAlchemyExecutionEngine, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, + ): + return _get_sql_outlier_statistic( + execution_engine=execution_engine, + metric_domain_kwargs=metric_domain_kwargs, + method=metric_value_kwargs["method"], + statistic="center", + ) + + @column_aggregate_partial(engine=SparkDFExecutionEngine) + def _spark(cls, column, method, **kwargs): + _validate_method(method) + if method == _IQR_METHOD: + return F.percentile_approx(column, 0.5, _SPARK_PERCENTILE_ACCURACY) + return F.mean(column) + + +class ColumnOutlierSpread(ColumnAggregateMetricProvider): + """Return the IQR or sample standard deviation used for outlier detection.""" + + metric_name = "column.outlier_spread" + value_keys = ("method",) + filter_column_isnull = True + + @column_aggregate_value(engine=PandasExecutionEngine) + def _pandas(cls, column, method, **kwargs): + _validate_method(method) + if method == _IQR_METHOD: + from scipy import stats + + return stats.iqr(column, nan_policy="omit") + return column.std() + + @metric_value(engine=SqlAlchemyExecutionEngine) + def _sqlalchemy( + cls, + execution_engine: SqlAlchemyExecutionEngine, + metric_domain_kwargs: dict, + metric_value_kwargs: dict, + metrics: dict[str, Any], + runtime_configuration: dict, + ): + return _get_sql_outlier_statistic( + execution_engine=execution_engine, + metric_domain_kwargs=metric_domain_kwargs, + method=metric_value_kwargs["method"], + statistic="spread", + ) + + @column_aggregate_partial(engine=SparkDFExecutionEngine) + def _spark(cls, column, method, **kwargs): + _validate_method(method) + if method == _IQR_METHOD: + first_quartile = F.percentile_approx(column, 0.25, _SPARK_PERCENTILE_ACCURACY) + third_quartile = F.percentile_approx(column, 0.75, _SPARK_PERCENTILE_ACCURACY) + return third_quartile - first_quartile + return F.stddev_samp(column) + + +class ColumnValuesNotOutliers(ColumnMapMetricProvider): + """Determine whether column values fall within the configured outlier threshold.""" + + condition_metric_name = "column_values.not_outliers" + condition_value_keys = ("method", "multiplier") + + @column_condition_partial(engine=PandasExecutionEngine) + def _pandas( + cls, + column, + _metrics, + method: str = _IQR_METHOD, + multiplier: float = 1.5, + **kwargs, + ) -> pd.Series: + _validate_method(method) + center = _metrics["column.outlier_center"] + spread = _metrics["column.outlier_spread"] + if _is_missing_statistic(center) or _is_missing_statistic(spread): + return column.notnull() & False + return (column - center).abs() < multiplier * spread + + @column_condition_partial(engine=SqlAlchemyExecutionEngine) + def _sqlalchemy( + cls, + column, + _metrics, + method: str = _IQR_METHOD, + multiplier: float = 1.5, + **kwargs, + ): + _validate_method(method) + center = _metrics["column.outlier_center"] + spread = _metrics["column.outlier_spread"] + if _is_missing_statistic(center) or _is_missing_statistic(spread): + return sa.false() + return sa.func.abs(column - center) < multiplier * spread + + @column_condition_partial(engine=SparkDFExecutionEngine) + def _spark( + cls, + column, + _metrics, + method: str = _IQR_METHOD, + multiplier: float = 1.5, + **kwargs, + ): + _validate_method(method) + center = _metrics["column.outlier_center"] + spread = _metrics["column.outlier_spread"] + if _is_missing_statistic(center) or _is_missing_statistic(spread): + return F.lit(False) + return F.abs(column - F.lit(center)) < F.lit(multiplier * spread) + + @classmethod + @override + def _get_evaluation_dependencies( + cls, + metric: MetricConfiguration, + configuration: Optional[ExpectationConfiguration] = None, + execution_engine: Optional[ExecutionEngine] = None, + runtime_configuration: Optional[dict] = None, + ): + dependencies: dict = super()._get_evaluation_dependencies( + metric=metric, + configuration=configuration, + execution_engine=execution_engine, + runtime_configuration=runtime_configuration, + ) + + condition_metric_name = ( + f"{cls.condition_metric_name}.{MetricPartialFunctionTypeSuffixes.CONDITION.value}" + ) + if metric.metric_name != condition_metric_name: + return dependencies + + method = metric.metric_value_kwargs.get("method", _IQR_METHOD) + _validate_method(method) + statistic_value_kwargs = {"method": method} + dependencies["column.outlier_center"] = MetricConfiguration( + metric_name="column.outlier_center", + metric_domain_kwargs=metric.metric_domain_kwargs, + metric_value_kwargs=statistic_value_kwargs, + ) + dependencies["column.outlier_spread"] = MetricConfiguration( + metric_name="column.outlier_spread", + metric_domain_kwargs=metric.metric_domain_kwargs, + metric_value_kwargs=statistic_value_kwargs, + ) + return dependencies diff --git a/tasks.py b/tasks.py index c4535ee81ec6..5f79bdfc0b9a 100644 --- a/tasks.py +++ b/tasks.py @@ -624,6 +624,7 @@ def type_schema( # noqa: C901 - too complex core.ExpectColumnValuesToMatchRegexList, core.ExpectColumnValuesToNotBeInSet, core.ExpectColumnValuesToNotBeNull, + core.ExpectColumnValuesToNotBeOutliers, core.ExpectColumnValuesToNotMatchLikePattern, core.ExpectColumnValuesToNotMatchLikePatternList, core.ExpectColumnValuesToNotMatchRegex, diff --git a/tests/expectations/core/test_expect_column_values_to_not_be_outliers.py b/tests/expectations/core/test_expect_column_values_to_not_be_outliers.py new file mode 100644 index 000000000000..8c923cba0d4d --- /dev/null +++ b/tests/expectations/core/test_expect_column_values_to_not_be_outliers.py @@ -0,0 +1,67 @@ +import pandas as pd +import pytest + +import great_expectations as gx +import great_expectations.expectations # register expectation renderers +from great_expectations.expectations.expectation_configuration import ( + ExpectationConfiguration, +) +from great_expectations.expectations.registry import get_renderer_impl +from great_expectations.self_check.util import get_test_validator_with_data + + +@pytest.mark.unit +def test_unknown_method_raises() -> None: + context = gx.get_context(mode="ephemeral") + validator = get_test_validator_with_data( + execution_engine="pandas", + data=pd.DataFrame({"amount": [1, 2, 3]}), + context=context, + ) + + with pytest.raises( + NotImplementedError, + match="method 'unknown' has not been implemented", + ): + validator.expect_column_values_to_not_be_outliers( + column="amount", + method="unknown", + catch_exceptions=False, + ) + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("mostly", "expected_suffix"), + [ + pytest.param(1.0, ".", id="all_values"), + pytest.param( + 0.9, + ", at least $mostly_pct % of the time.", + id="mostly", + ), + ], +) +def test_prescriptive_renderer(mostly: float, expected_suffix: str) -> None: + configuration = ExpectationConfiguration( + type="expect_column_values_to_not_be_outliers", + kwargs={ + "column": "amount", + "method": "iqr", + "multiplier": 1.5, + "mostly": mostly, + }, + ) + renderer = get_renderer_impl( + object_name="expect_column_values_to_not_be_outliers", + renderer_type="atomic.prescriptive.summary", + )[1] + + rendered_content = renderer(configuration=configuration).to_json_dict() + + template = rendered_content["value"]["template"] + assert template.startswith( + "$column values must not be statistical outliers using the $method method " + "with a multiplier of $multiplier" + ) + assert template.endswith(expected_suffix) diff --git a/tests/integration/data_sources_and_expectations/expectations/test_expect_column_values_to_not_be_outliers.py b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_values_to_not_be_outliers.py new file mode 100644 index 000000000000..6542cb475ca0 --- /dev/null +++ b/tests/integration/data_sources_and_expectations/expectations/test_expect_column_values_to_not_be_outliers.py @@ -0,0 +1,153 @@ +from typing import Sequence + +import pandas as pd +import pytest + +import great_expectations.expectations as gxe +from great_expectations.core.result_format import ResultFormat +from great_expectations.datasource.fluent.interfaces import Batch +from tests.integration.conftest import parameterize_batch_for_data_sources +from tests.integration.test_utils.data_source_config import ( + BigQueryDatasourceTestConfig, + DatabricksDatasourceTestConfig, + MySQLDatasourceTestConfig, + PandasDataFrameDatasourceTestConfig, + PostgreSQLDatasourceTestConfig, + RedshiftDatasourceTestConfig, + SnowflakeDatasourceTestConfig, + SparkFilesystemCsvDatasourceTestConfig, + SqliteDatasourceTestConfig, + SQLServerDatasourceTestConfig, +) +from tests.integration.test_utils.data_source_config.base import DataSourceTestConfig + +ALL_SUPPORTED_DATA_SOURCES: Sequence[DataSourceTestConfig] = [ + PandasDataFrameDatasourceTestConfig(), + SparkFilesystemCsvDatasourceTestConfig(), + SqliteDatasourceTestConfig(), + PostgreSQLDatasourceTestConfig(), + MySQLDatasourceTestConfig(), + SQLServerDatasourceTestConfig(), + BigQueryDatasourceTestConfig(), + SnowflakeDatasourceTestConfig(), + DatabricksDatasourceTestConfig(), + RedshiftDatasourceTestConfig(), +] + +COLUMN = "amount" +CLEAN_DATA = pd.DataFrame({COLUMN: list(range(1, 21))}) +DATA_WITH_OUTLIER = pd.DataFrame({COLUMN: [*range(1, 21), 100]}) +DATA_WITH_OUTLIER_AND_NULL = pd.DataFrame( + { + "row_id": range(22), + COLUMN: pd.Series([*range(1, 21), 100, None], dtype=object), + } +) +DATA_WITH_VALUES_ON_IQR_BOUNDARY = pd.DataFrame({COLUMN: [0, 1, 2, 3, 4]}) + + +@pytest.mark.parametrize( + ("method", "multiplier"), + [ + pytest.param("iqr", 1.5, id="iqr"), + pytest.param("std", 3.0, id="standard_deviation"), + ], +) +@parameterize_batch_for_data_sources( + data_source_configs=ALL_SUPPORTED_DATA_SOURCES, + data=CLEAN_DATA, +) +def test_clean_data_passes( + batch_for_datasource: Batch, + method: str, + multiplier: float, +) -> None: + expectation = gxe.ExpectColumnValuesToNotBeOutliers( + column=COLUMN, + method=method, + multiplier=multiplier, + ) + + result = batch_for_datasource.validate(expectation) + + assert result.success + assert result.result["unexpected_count"] == 0 + + +@pytest.mark.parametrize( + ("method", "multiplier"), + [ + pytest.param("iqr", 1.5, id="iqr"), + pytest.param("std", 3.0, id="standard_deviation"), + ], +) +@parameterize_batch_for_data_sources( + data_source_configs=ALL_SUPPORTED_DATA_SOURCES, + data=DATA_WITH_OUTLIER, +) +def test_injected_outlier_fails_consistently_across_engines( + batch_for_datasource: Batch, + method: str, + multiplier: float, +) -> None: + expectation = gxe.ExpectColumnValuesToNotBeOutliers( + column=COLUMN, + method=method, + multiplier=multiplier, + ) + + result = batch_for_datasource.validate( + expectation, + result_format=ResultFormat.COMPLETE, + ) + + assert not result.success + assert result.result["unexpected_count"] == 1 + assert result.result["unexpected_list"] == [100] + + +@parameterize_batch_for_data_sources( + data_source_configs=ALL_SUPPORTED_DATA_SOURCES, + data=DATA_WITH_OUTLIER_AND_NULL, +) +def test_nulls_are_excluded_from_statistics_and_evaluation( + batch_for_datasource: Batch, +) -> None: + expectation = gxe.ExpectColumnValuesToNotBeOutliers( + column=COLUMN, + method="iqr", + multiplier=1.5, + ) + + result = batch_for_datasource.validate( + expectation, + result_format=ResultFormat.COMPLETE, + ) + + assert not result.success + assert result.result["missing_count"] == 1 + assert result.result["unexpected_count"] == 1 + assert result.result["unexpected_list"] == [100] + + +@parameterize_batch_for_data_sources( + data_source_configs=ALL_SUPPORTED_DATA_SOURCES, + data=DATA_WITH_VALUES_ON_IQR_BOUNDARY, +) +def test_values_equal_to_threshold_are_outliers( + batch_for_datasource: Batch, +) -> None: + expectation = gxe.ExpectColumnValuesToNotBeOutliers( + column=COLUMN, + method="iqr", + multiplier=1.0, + ) + + result = batch_for_datasource.validate( + expectation, + result_format=ResultFormat.COMPLETE, + ) + + assert not result.success + assert result.result["unexpected_count"] == 2 + assert sorted(result.result["unexpected_list"]) == [0, 4] From 712e3c76b0ff47327a7d613130afad9ce2646e7d Mon Sep 17 00:00:00 2001 From: Santosh Chavala <76093335+chavalasantosh@users.noreply.github.com> Date: Mon, 3 Aug 2026 20:53:50 +0530 Subject: [PATCH 2/2] Fix null filtering for outlier map metric --- .../metrics/column_map_metrics/column_values_not_outliers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/great_expectations/expectations/metrics/column_map_metrics/column_values_not_outliers.py b/great_expectations/expectations/metrics/column_map_metrics/column_values_not_outliers.py index 5d9d9efb6309..0c005ca8ea54 100644 --- a/great_expectations/expectations/metrics/column_map_metrics/column_values_not_outliers.py +++ b/great_expectations/expectations/metrics/column_map_metrics/column_values_not_outliers.py @@ -319,6 +319,7 @@ class ColumnValuesNotOutliers(ColumnMapMetricProvider): condition_metric_name = "column_values.not_outliers" condition_value_keys = ("method", "multiplier") + filter_column_isnull = True @column_condition_partial(engine=PandasExecutionEngine) def _pandas(