[MINORBUMP] ExpectColumnDistinctValuesToEqualSet with database-pushed comparison

docs/docusaurus/docs/cloud/validations/format_results.md

@@ -186,13 +186,14 @@ Follow the steps below to select a base format, optionally configure additional
     | Field within `result`         | Value                                                                                                                                                                                                  |
     |-------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
     | element_count                 | The total number of values in the column.                                                                                                                                                              |
-    | missing_count                 | The number of missing values in the column.                                                                                                                                                            |
+    | missing_count                 | The number of missing (null) values in the column. For distinct values Expectations, this is the count of expected values not found in the column.                                                        |
     | missing_percent               | The total percent of rows missing values for the column.                                                                                                                                               |
     | unexpected_count              | The total count of unexpected values in a column.                                                                                                                                                   |
     | unexpected_percent            | The overall percent of unexpected values in a column.                                                                                                                                                  |
     | unexpected_percent_nonmissing | The percent of unexpected values in a column, excluding rows that have no value for that column.                                                                                                       |
     | observed_value                | The aggregate statistic computed for the column. This only applies to Expectations that pertain to the aggregate value of a column, rather than the individual values in each row for the column.      |
     | partial_unexpected_list       | A partial list of values that violate the Expectation. (Up to 20 values by default.)                                                                                                                   |
+    | partial_missing_list          | A partial list of expected values that are missing from the column. Applies to distinct values Expectations. (Up to 20 values by default.)                                                             |
     | partial_unexpected_index_list | A partial list of the unexpected values in the column, as defined by the columns in `unexpected_index_column_names`. (Up to 20 indices by default.)                                                    |
     | partial_unexpected_counts     | A partial list of values and counts, showing the number of times each of the unexpected values occurs. (Up to 20 unexpected value/count pairs by default.)                                              |
     | unexpected_index_list         | A list of the indices of the unexpected values in the column, as defined by the columns in `unexpected_index_column_names`. This only applies to Expectations that have a yes/no answer for each row.  |
@@ -214,6 +215,7 @@ Follow the steps below to select a base format, optionally configure additional
     | unexpected_percent_nonmissing      |no              |yes             |yes             |yes             |
     | observed_value                     |no              |yes             |yes             |yes             |
     | partial_unexpected_list            |no              |yes **          |yes **          |yes **          |
+    | partial_missing_list               |no              |yes **          |yes **          |yes **          |
     | partial_unexpected_index_list      |no              |no              |yes **          |yes **          |
     | partial_unexpected_counts          |no              |no              |yes **          |yes **          |
     | unexpected_index_list              |no              |no              |no              |yes             |

great_expectations/constants.py

@@ -1 +1,10 @@
+from typing import Final
+
 DATAFRAME_REPLACEMENT_STR = "<DATAFRAME>"
+
+# Maximum number of result records to return in expectation results
+MAX_RESULT_RECORDS: Final[int] = 200
+
+# Maximum number of distinct values to return in expectation results
+# to prevent payload size issues (e.g., HTTP 413 errors with GX Cloud)
+MAX_DISTINCT_VALUES: Final[int] = 20

great_expectations/expectations/core/expect_column_distinct_values_to_equal_set.py

@@ -3,10 +3,10 @@
 from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Type, Union
 
 from great_expectations.compatibility.typing_extensions import override
+from great_expectations.constants import MAX_DISTINCT_VALUES
 from great_expectations.expectations.expectation import (
     ColumnAggregateExpectation,
     _style_row_condition,
-    parse_value_to_observed_type,
     render_suite_parameter_string,
 )
 from great_expectations.expectations.metadata_types import DataQualityIssues, SupportedDataSources
@@ -218,7 +218,12 @@ class ExpectColumnDistinctValuesToEqualSet(ColumnAggregateExpectation):
     _library_metadata = library_metadata
 
     # Setting necessary computation metric dependencies and defining kwargs, as well as assigning kwargs default values\  # noqa: E501 # FIXME CoP
-    metric_dependencies = ("column.value_counts",)
+    metric_dependencies = (
+        "column.distinct_values.not_in_set.count",
+        "column.distinct_values.not_in_set",
+        "column.distinct_values.missing_from_column.count",
+        "column.distinct_values.missing_from_column",
+    )
     success_keys = ("value_set",)
     args_keys = (
         "column",
@@ -362,25 +367,39 @@ def _validate(
         runtime_configuration: Optional[dict] = None,
         execution_engine: Optional[ExecutionEngine] = None,
     ):
-        observed_value_counts = metrics["column.value_counts"]
-        observed_value_set = set(observed_value_counts.index)
-        value_set = self._get_success_kwargs()["value_set"]
-
-        # Try to coerce string values to match the type of observed values
-        if observed_value_set and value_set:
-            first_observed = next(iter(observed_value_set))
-            expected_value_set = {
-                parse_value_to_observed_type(first_observed, value) for value in value_set
-            }
-        else:
-            expected_value_set = set(value_set)
+        # Get count of unexpected values (values in column but NOT in expected set)
+        unexpected_count = metrics.get("column.distinct_values.not_in_set.count", 0)
+        unexpected_values = metrics.get("column.distinct_values.not_in_set", [])
+
+        # Get count of missing values (values in expected set but NOT in column)
+        missing_count = metrics.get("column.distinct_values.missing_from_column.count", 0)
+        missing_values = metrics.get("column.distinct_values.missing_from_column", [])
+
+        # Success if no unexpected values AND no missing values
+        success = (unexpected_count == 0) and (missing_count == 0)
+
+        # Check partial_unexpected_count setting to determine if partial lists should be included
+        # For distinct values Expectations, always use MAX_DISTINCT_VALUES as the limit
+        # but respect partial_unexpected_count: 0 to exclude the list entirely
+        result_format = (
+            runtime_configuration.get("result_format", {}) if runtime_configuration else {}
+        )
+        partial_unexpected_count = result_format.get("partial_unexpected_count", 20)
+        include_partial_lists = partial_unexpected_count > 0
+
+        result_dict: Dict[str, Any] = {
+            "observed_value": None,
+            "unexpected_count": unexpected_count,
+            "missing_count": missing_count,
+        }
+
+        if include_partial_lists:
+            result_dict["partial_unexpected_list"] = unexpected_values[:MAX_DISTINCT_VALUES]
+            result_dict["partial_missing_list"] = missing_values[:MAX_DISTINCT_VALUES]
 
         return {
-            "success": observed_value_set == expected_value_set,
-            "result": {
-                "observed_value": sorted(list(observed_value_set)),
-                "details": {"value_counts": observed_value_counts},
-            },
+            "success": success,
+            "result": result_dict,
         }
 
     @classmethod
@@ -397,69 +416,54 @@ def _atomic_diagnostic_observed_value(
             result=result,
             runtime_configuration=runtime_configuration,
         )
-        expected_param_prefix = "exp__"
-        expected_param_name = "expected_value"
-        ov_param_prefix = "ov__"
+        # Use original prefixes that frontend expects
+        ov_param_prefix = "ov__"  # for unexpected values (observed but not expected)
         ov_param_name = "observed_value"
+        exp_param_prefix = "exp__"  # for missing values (expected but not observed)
+        exp_param_name = "expected_value"
 
-        renderer_configuration.add_param(
-            name=expected_param_name,
-            param_type=RendererValueType.ARRAY,
-            value=renderer_configuration.kwargs.get("value_set", []),
-        )
-        renderer_configuration = cls._add_array_params(
-            array_param_name=expected_param_name,
-            param_prefix=expected_param_prefix,
-            renderer_configuration=renderer_configuration,
-        )
+        # Get unexpected and missing values from result, limited to MAX_DISTINCT_VALUES
+        result_dict = result.get("result", {}) if result else {}
+        unexpected_values = result_dict.get("partial_unexpected_list", [])[:MAX_DISTINCT_VALUES]
+        missing_values = result_dict.get("partial_missing_list", [])[:MAX_DISTINCT_VALUES]
 
+        # Add unexpected values (values in column but NOT in expected set) using ov__ prefix
         renderer_configuration.add_param(
             name=ov_param_name,
             param_type=RendererValueType.ARRAY,
-            value=result.get("result", {}).get("observed_value", []) if result else [],
+            value=unexpected_values,
         )
         renderer_configuration = cls._add_array_params(
             array_param_name=ov_param_name,
             param_prefix=ov_param_prefix,
             renderer_configuration=renderer_configuration,
         )
-        observed_value_set = set(
-            result.get("result", {}).get("observed_value", []) if result else []
-        )
-        sample_observed_value = next(iter(observed_value_set)) if observed_value_set else None
-        expected_value_set = {
-            parse_value_to_observed_type(observed_value=sample_observed_value, value=value)
-            for value in renderer_configuration.kwargs.get("value_set", [])
-        }
 
-        observed_values = (
-            (name, schema)
-            for name, schema in renderer_configuration.params
-            if name.startswith(ov_param_prefix)
+        # Add missing values (values in expected set but NOT in column) using exp__ prefix
+        renderer_configuration.add_param(
+            name=exp_param_name,
+            param_type=RendererValueType.ARRAY,
+            value=missing_values,
         )
-
-        expected_values = (
-            (name, schema)
-            for name, schema in renderer_configuration.params
-            if name.startswith(expected_param_prefix)
+        renderer_configuration = cls._add_array_params(
+            array_param_name=exp_param_name,
+            param_prefix=exp_param_prefix,
+            renderer_configuration=renderer_configuration,
         )
 
         template_str_list = []
-        for name, schema in observed_values:
-            render_state = (
-                ObservedValueRenderState.EXPECTED.value
-                if schema.value in expected_value_set
-                else ObservedValueRenderState.UNEXPECTED.value
-            )
-            renderer_configuration.params.__dict__[name].render_state = render_state
-            template_str_list.append(f"${name}")
 
-        for name, schema in expected_values:
-            coerced_value = parse_value_to_observed_type(
-                observed_value=sample_observed_value,
-                value=schema.value,
-            )
-            if coerced_value not in observed_value_set:
+        # All unexpected values (ov__) get UNEXPECTED render state
+        for name, schema in renderer_configuration.params:
+            if name.startswith(ov_param_prefix):
+                renderer_configuration.params.__dict__[
+                    name
+                ].render_state = ObservedValueRenderState.UNEXPECTED.value
+                template_str_list.append(f"${name}")
+
+        # All missing values (exp__) get MISSING render state
+        for name, schema in renderer_configuration.params:
+            if name.startswith(exp_param_prefix):
                 renderer_configuration.params.__dict__[
                     name
                 ].render_state = ObservedValueRenderState.MISSING.value

great_expectations/expectations/metrics/column_aggregate_metrics/column_distinct_values_missing_from_column.py

@@ -10,6 +10,7 @@
     __version__ as SQLALCHEMY_VERSION,
 )
 from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa
+from great_expectations.constants import MAX_DISTINCT_VALUES
 from great_expectations.core.metric_domain_types import MetricDomainTypes
 from great_expectations.execution_engine import (
     PandasExecutionEngine,
@@ -144,7 +145,11 @@ class ColumnDistinctValuesMissingFromColumn(ColumnAggregateMetricProvider):
 
     @column_aggregate_value(engine=PandasExecutionEngine)
     def _pandas(
-        cls, column: pd.Series, value_set: List[Any], limit: int = 20, **kwargs
+        cls,
+        column: pd.Series,
+        value_set: List[Any],
+        limit: int = MAX_DISTINCT_VALUES,
+        **kwargs,
     ) -> List[Any]:
         column_set = set(column.dropna().unique())
         expected_set = _coerce_value_set_to_column_type(column_set, value_set)
@@ -161,7 +166,7 @@ def _sqlalchemy(
     ) -> List[Any]:
         """Return values in the expected set that are missing from the column."""
         value_set = _coerce_value_set_for_sql(metric_value_kwargs.get("value_set", []))
-        limit = metric_value_kwargs.get("limit", 20)
+        limit = metric_value_kwargs.get("limit") or MAX_DISTINCT_VALUES
 
         selectable: sqlalchemy.Selectable
         accessor_domain_kwargs: Dict[str, str]
@@ -200,7 +205,7 @@ def _spark(
     ) -> List[Any]:
         """Return values in the expected set that are missing from the column."""
         value_set = metric_value_kwargs.get("value_set", [])
-        limit = metric_value_kwargs.get("limit", 20)
+        limit = metric_value_kwargs.get("limit") or MAX_DISTINCT_VALUES
 
         df: pyspark.DataFrame
         accessor_domain_kwargs: Dict[str, str]

great_expectations/expectations/metrics/column_aggregate_metrics/column_distinct_values_not_in_set.py

@@ -10,6 +10,7 @@
     __version__ as SQLALCHEMY_VERSION,
 )
 from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa
+from great_expectations.constants import MAX_DISTINCT_VALUES
 from great_expectations.core.metric_domain_types import MetricDomainTypes
 from great_expectations.execution_engine import (
     PandasExecutionEngine,
@@ -148,7 +149,11 @@ class ColumnDistinctValuesNotInSet(ColumnAggregateMetricProvider):
 
     @column_aggregate_value(engine=PandasExecutionEngine)
     def _pandas(
-        cls, column: pd.Series, value_set: List[Any], limit: int = 20, **kwargs
+        cls,
+        column: pd.Series,
+        value_set: List[Any],
+        limit: int = MAX_DISTINCT_VALUES,
+        **kwargs,
     ) -> List[Any]:
         column_set = set(column.dropna().unique())
         expected_set = _coerce_value_set_to_column_type(column_set, value_set)
@@ -170,7 +175,7 @@ def _sqlalchemy(
     ) -> List[Any]:
         """Return sample of distinct values in column that are NOT in the expected set."""
         value_set = _coerce_value_set_for_sql(metric_value_kwargs.get("value_set", []))
-        limit = metric_value_kwargs.get("limit", 20)
+        limit = metric_value_kwargs.get("limit") or MAX_DISTINCT_VALUES
 
         selectable: sqlalchemy.Selectable
         accessor_domain_kwargs: Dict[str, str]
@@ -233,7 +238,7 @@ def _spark(
     ) -> List[Any]:
         """Return sample of distinct values in column that are NOT in the expected set."""
         value_set = metric_value_kwargs.get("value_set", [])
-        limit = metric_value_kwargs.get("limit", 20)
+        limit = metric_value_kwargs.get("limit") or MAX_DISTINCT_VALUES
 
         df: pyspark.DataFrame
         accessor_domain_kwargs: Dict[str, str]

great_expectations/expectations/metrics/map_metric_provider/column_map_condition_auxilliary_methods.py

@@ -11,8 +11,8 @@
 )
 
 import great_expectations.exceptions as gx_exceptions
+from great_expectations.constants import MAX_RESULT_RECORDS
 from great_expectations.expectations.metrics.util import (
-    MAX_RESULT_RECORDS,
     get_dbms_compatible_metric_domain_kwargs,
 )
 

great_expectations/expectations/metrics/map_metric_provider/column_pair_map_condition_auxilliary_methods.py

@@ -11,11 +11,11 @@
 
 from great_expectations.compatibility.pyspark import functions as F
 from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa
+from great_expectations.constants import MAX_RESULT_RECORDS
 from great_expectations.expectations.metrics.map_metric_provider.is_sqlalchemy_metric_selectable import (  # noqa: E501 # FIXME CoP
     _is_sqlalchemy_metric_selectable,
 )
 from great_expectations.expectations.metrics.util import (
-    MAX_RESULT_RECORDS,
     get_dbms_compatible_metric_domain_kwargs,
 )
 from great_expectations.util import (

great_expectations/expectations/metrics/map_metric_provider/map_condition_auxilliary_methods.py

@@ -20,6 +20,7 @@
 from great_expectations.compatibility.sqlalchemy import (
     sqlalchemy as sa,
 )
+from great_expectations.constants import MAX_RESULT_RECORDS
 from great_expectations.core.metric_function_types import (
     SummarizationMetricNameSuffixes,
 )
@@ -28,7 +29,6 @@
     _is_sqlalchemy_metric_selectable,
 )
 from great_expectations.expectations.metrics.util import (
-    MAX_RESULT_RECORDS,
     compute_unexpected_pandas_indices,
     get_dbms_compatible_metric_domain_kwargs,
     get_sqlalchemy_source_table_and_schema,

great_expectations/expectations/metrics/map_metric_provider/multicolumn_map_condition_auxilliary_methods.py

@@ -11,11 +11,11 @@
 
 from great_expectations.compatibility.pyspark import functions as F
 from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa
+from great_expectations.constants import MAX_RESULT_RECORDS
 from great_expectations.expectations.metrics.map_metric_provider.is_sqlalchemy_metric_selectable import (  # noqa: E501 # FIXME CoP
     _is_sqlalchemy_metric_selectable,
 )
 from great_expectations.expectations.metrics.util import (
-    MAX_RESULT_RECORDS,
     get_dbms_compatible_metric_domain_kwargs,
 )
 from great_expectations.util import (

great_expectations/expectations/metrics/query_metric_provider.py

@@ -8,9 +8,9 @@
 from great_expectations.compatibility.sqlalchemy import (
     sqlalchemy as sa,
 )
+from great_expectations.constants import MAX_RESULT_RECORDS
 from great_expectations.execution_engine.sqlalchemy_dialect import GXSqlDialect
 from great_expectations.expectations.metrics.metric_provider import MetricProvider
-from great_expectations.expectations.metrics.util import MAX_RESULT_RECORDS
 from great_expectations.util import get_sqlalchemy_subquery_type
 
 if TYPE_CHECKING:

great_expectations/expectations/metrics/query_metrics/query_column.py

@@ -2,6 +2,7 @@
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
+from great_expectations.constants import MAX_RESULT_RECORDS
 from great_expectations.core.metric_domain_types import MetricDomainTypes
 from great_expectations.execution_engine import (
     SparkDFExecutionEngine,
@@ -12,7 +13,6 @@
     QueryMetricProvider,
     QueryParameters,
 )
-from great_expectations.expectations.metrics.util import MAX_RESULT_RECORDS
 
 if TYPE_CHECKING:
     from great_expectations.compatibility import pyspark