[MAINTENANCE] Add a sqlite execution engine and 1 sqlite specific metric. (#11042)

Author: billdirks

great_expectations/datasource/fluent/sqlite_datasource.py

@@ -29,6 +29,7 @@
 from great_expectations.datasource.fluent.sql_datasource import (
     TableAsset as SqlTableAsset,
 )
+from great_expectations.execution_engine.sqlite_execution_engine import SqliteExecutionEngine
 
 if TYPE_CHECKING:
     # min version of typing_extension missing `Self`, so it can't be imported at runtime
@@ -41,6 +42,9 @@
         BatchParameters,
         DataAsset,
     )
+    from great_expectations.execution_engine.sqlalchemy_execution_engine import (
+        SqlAlchemyExecutionEngine,
+    )
 
 # This module serves as an example of how to extend _SQLAssets for specific backends. The steps are:
 # 1. Create a plain class with the extensions necessary for the specific backend.
@@ -161,6 +165,12 @@ class SqliteDatasource(SQLDatasource):
     _TableAsset: Type[SqlTableAsset] = pydantic.PrivateAttr(SqliteTableAsset)
     _QueryAsset: Type[SqlQueryAsset] = pydantic.PrivateAttr(SqliteQueryAsset)
 
+    @property
+    @override
+    def execution_engine_type(self) -> Type[SqlAlchemyExecutionEngine]:
+        """Returns the default execution engine type."""
+        return SqliteExecutionEngine
+
     @public_api
     @override
     def add_table_asset(

great_expectations/execution_engine/sqlite_execution_engine.py

@@ -0,0 +1,60 @@
+from typing import Optional
+
+from great_expectations.compatibility.sqlalchemy import sqlalchemy as sa
+from great_expectations.compatibility.typing_extensions import override
+from great_expectations.core.metric_function_types import SummarizationMetricNameSuffixes
+from great_expectations.execution_engine.execution_engine import ExecutionEngine
+from great_expectations.execution_engine.sqlalchemy_execution_engine import (
+    SqlAlchemyExecutionEngine,
+)
+from great_expectations.expectations.expectation_configuration import ExpectationConfiguration
+from great_expectations.expectations.metrics.column_aggregate_metric_provider import (
+    column_aggregate_partial,
+)
+from great_expectations.expectations.metrics.column_aggregate_metrics.column_standard_deviation import (  # noqa: E501
+    ColumnStandardDeviation as BaseColumnStandardDeviation,
+)
+from great_expectations.validator.metric_configuration import MetricConfiguration
+
+
+class SqliteExecutionEngine(SqlAlchemyExecutionEngine):
+    """SqlAlchemyExecutionEngine for SQLite databases."""
+
+    pass
+
+
+class ColumnStandardDeviation(BaseColumnStandardDeviation):
+    """MetricProvider Class for Aggregate Standard Deviation metric for SQLite databases."""
+
+    # We should change this decorator to compute this metric a completely new way
+    @column_aggregate_partial(engine=SqlAlchemyExecutionEngine)
+    def _sqlalchemy(cls, column, _dialect, _metrics, **kwargs):
+        """Sqlite Standard Deviation implementation"""
+        mean = _metrics["column.mean"]
+        nonnull_row_count = _metrics[
+            f"column_values.null.{SummarizationMetricNameSuffixes.UNEXPECTED_COUNT.value}"
+        ]
+        standard_deviation = sa.func.sqrt(
+            sa.func.sum((1.0 * column - mean) * (1.0 * column - mean))
+            / ((1.0 * nonnull_row_count) - 1.0)
+        )
+        return standard_deviation
+
+    @classmethod
+    @override
+    def _get_evaluation_dependencies(
+        cls,
+        metric: MetricConfiguration,
+        configuration: Optional[ExpectationConfiguration] = None,
+        execution_engine: Optional[ExecutionEngine] = None,
+        runtime_configuration: Optional[dict] = None,
+    ):
+        # We don't need to override this here but I wanted to show for completeness
+        # If we are changing the decorator on the provider method or we are completely
+        # implementing a new datasource, we'll want to override.
+        return super()._get_evaluation_dependencies(
+            metric=metric,
+            configuration=configuration,
+            execution_engine=execution_engine,
+            runtime_configuration=runtime_configuration,
+        )

great_expectations/expectations/metrics/column_aggregate_metrics/column_standard_deviation.py

@@ -48,15 +48,6 @@ def _sqlalchemy(cls, column, _dialect, _metrics, **kwargs):
         """SqlAlchemy Standard Deviation implementation"""
         if _dialect.name.lower() == GXSqlDialect.MSSQL:
             standard_deviation = sa.func.stdev(column)
-        elif _dialect.name.lower() == GXSqlDialect.SQLITE:
-            mean = _metrics["column.mean"]
-            nonnull_row_count = _metrics[
-                f"column_values.null.{SummarizationMetricNameSuffixes.UNEXPECTED_COUNT.value}"
-            ]
-            standard_deviation = sa.func.sqrt(
-                sa.func.sum((1.0 * column - mean) * (1.0 * column - mean))
-                / ((1.0 * nonnull_row_count) - 1.0)
-            )
         else:
             standard_deviation = sa.func.stddev_samp(column)
 

great_expectations/expectations/registry.py

@@ -312,8 +312,25 @@ def get_metric_provider(
 ) -> Tuple[MetricProvider, Callable]:
     try:
         metric_definition = _registered_metrics[metric_name]
+    except KeyError:
+        raise gx_exceptions.MetricProviderError(  # noqa: TRY003 # FIXME CoP
+            f"No metric named {metric_name} found."
+        )
+
+    try:
         return metric_definition["providers"][type(execution_engine).__name__]
     except KeyError:
+        # Search up class hierarchy for a match. We skip the first entry since that's the
+        # execution engine type itself, type(execution_engine), which we just checked and
+        # resulted in the KeyError we're handling here.
+        for cls in type(execution_engine).mro()[1:]:
+            possible_key = cls.__name__
+            if metric_definition["providers"].get(possible_key) is not None:
+                metric_def = metric_definition["providers"][possible_key]
+                # Register the metric definition for this engine so we don't have to search again
+                metric_definition["providers"][type(execution_engine).__name__] = metric_def
+                return metric_def
+        # no matches when search hierarchy so we raise
         raise gx_exceptions.MetricProviderError(  # noqa: TRY003 # FIXME CoP
             f"No provider found for {metric_name} using {type(execution_engine).__name__}"
         )

tests/validator/test_validation_graph.py

@@ -344,10 +344,7 @@ class PandasExecutionEngineStub:
             ),
         )
 
-    assert (
-        e.value.message
-        == "No provider found for column_values.not_a_metric using PandasExecutionEngine"
-    )
+    assert e.value.message == "No metric named column_values.not_a_metric found."
 
 
 @pytest.mark.unit