add include_tests flag for presets (#1530)

* add include_tests flag for presets

* add tests

Author: mike0sv

src/evidently/future/container.py

@@ -19,6 +19,9 @@
 
 
 class MetricContainer(abc.ABC):
+    def __init__(self, include_tests: bool = True):
+        self.include_tests = include_tests
+
     @abc.abstractmethod
     def generate_metrics(self, context: "Context") -> Sequence[MetricOrContainer]:
         raise NotImplementedError()
@@ -50,7 +53,15 @@ def list_metrics(self, context: "Context") -> Generator[Metric, None, None]:
             else:
                 raise ValueError(f"invalid metric type {type(item)}")
 
+    def _get_tests(self, tests):
+        if tests is not None:
+            return tests
+        if self.include_tests:
+            return None
+        return []
+
 
 class ColumnMetricContainer(MetricContainer, abc.ABC):
-    def __init__(self, column: str):
+    def __init__(self, column: str, include_tests: bool = True):
+        super().__init__(include_tests=include_tests)
         self._column = column

src/evidently/future/generators/column.py

@@ -33,6 +33,7 @@ def __init__(
         self.columns = columns
         self.column_types = column_types
         self.metric_kwargs = metric_kwargs or {}
+        super().__init__(include_tests=True)
 
     def _instantiate_metric(self, column: str) -> MetricOrContainer:
         return self.metric_type(column=column, **self.metric_kwargs)

src/evidently/future/metrics/group_by.py

@@ -67,6 +67,7 @@ class GroupBy(MetricContainer):
     def __init__(self, metric: Metric, column_name: str):
         self._column_name = column_name
         self._metric = metric
+        super().__init__(True)
 
     def generate_metrics(self, context: Context) -> Sequence[MetricOrContainer]:
         labels = context.column(self._column_name).labels()

src/evidently/future/presets/classification.py

@@ -56,6 +56,7 @@ def __init__(
         tnr_tests: SingleValueMetricTests = None,
         fpr_tests: SingleValueMetricTests = None,
         fnr_tests: SingleValueMetricTests = None,
+        include_tests: bool = True,
     ):
         self._accuracy_tests = accuracy_tests
         self._precision_tests = precision_tests
@@ -71,6 +72,7 @@ def __init__(
         self._conf_matrix = conf_matrix
         self._pr_curve = pr_curve
         self._pr_table = pr_table
+        super().__init__(include_tests=include_tests)
 
     def generate_metrics(self, context: "Context") -> Sequence[MetricOrContainer]:
         classification = context.data_definition.get_classification("default")
@@ -80,25 +82,25 @@ def generate_metrics(self, context: "Context") -> Sequence[MetricOrContainer]:
         metrics: List[Metric]
 
         metrics = [
-            Accuracy(probas_threshold=self._probas_threshold, tests=self._accuracy_tests),
-            Precision(probas_threshold=self._probas_threshold, tests=self._precision_tests),
-            Recall(probas_threshold=self._probas_threshold, tests=self._recall_tests),
-            F1Score(probas_threshold=self._probas_threshold, tests=self._f1score_tests),
+            Accuracy(probas_threshold=self._probas_threshold, tests=self._get_tests(self._accuracy_tests)),
+            Precision(probas_threshold=self._probas_threshold, tests=self._get_tests(self._precision_tests)),
+            Recall(probas_threshold=self._probas_threshold, tests=self._get_tests(self._recall_tests)),
+            F1Score(probas_threshold=self._probas_threshold, tests=self._get_tests(self._f1score_tests)),
         ]
         if classification.prediction_probas is not None:
             metrics.extend(
                 [
-                    RocAuc(probas_threshold=self._probas_threshold, tests=self._rocauc_test),
-                    LogLoss(probas_threshold=self._probas_threshold, tests=self._logloss_test),
+                    RocAuc(probas_threshold=self._probas_threshold, tests=self._get_tests(self._rocauc_test)),
+                    LogLoss(probas_threshold=self._probas_threshold, tests=self._get_tests(self._logloss_test)),
                 ]
             )
         if isinstance(classification, BinaryClassification):
             metrics.extend(
                 [
-                    TPR(probas_threshold=self._probas_threshold, tests=self._tpr_test),
-                    TNR(probas_threshold=self._probas_threshold, tests=self._tnr_test),
-                    FPR(probas_threshold=self._probas_threshold, tests=self._fpr_test),
-                    FNR(probas_threshold=self._probas_threshold, tests=self._fnr_test),
+                    TPR(probas_threshold=self._probas_threshold, tests=self._get_tests(self._tpr_test)),
+                    TNR(probas_threshold=self._probas_threshold, tests=self._get_tests(self._tnr_test)),
+                    FPR(probas_threshold=self._probas_threshold, tests=self._get_tests(self._fpr_test)),
+                    FNR(probas_threshold=self._probas_threshold, tests=self._get_tests(self._fnr_test)),
                 ]
             )
         return metrics
@@ -144,27 +146,35 @@ def __init__(
         precision_tests: ByLabelMetricTests = None,
         recall_tests: ByLabelMetricTests = None,
         rocauc_tests: ByLabelMetricTests = None,
+        include_tests: bool = True,
     ):
         self._probas_threshold = probas_threshold
         self._k = k
         self._f1score_tests = f1score_tests
         self._precision_tests = precision_tests
         self._recall_tests = recall_tests
         self._rocauc_tests = rocauc_tests
+        super().__init__(include_tests=include_tests)
 
     def generate_metrics(self, context: "Context") -> Sequence[MetricOrContainer]:
         classification = context.data_definition.get_classification("default")
         if classification is None:
             raise ValueError("Cannot use ClassificationPreset without a classification configration")
         return [
-            F1ByLabel(probas_threshold=self._probas_threshold, k=self._k, tests=self._f1score_tests),
-            PrecisionByLabel(probas_threshold=self._probas_threshold, k=self._k, tests=self._precision_tests),
-            RecallByLabel(probas_threshold=self._probas_threshold, k=self._k, tests=self._recall_tests),
+            F1ByLabel(probas_threshold=self._probas_threshold, k=self._k, tests=self._get_tests(self._f1score_tests)),
+            PrecisionByLabel(
+                probas_threshold=self._probas_threshold, k=self._k, tests=self._get_tests(self._precision_tests)
+            ),
+            RecallByLabel(
+                probas_threshold=self._probas_threshold, k=self._k, tests=self._get_tests(self._recall_tests)
+            ),
         ] + (
             []
             if classification.prediction_probas is None
             else [
-                RocAucByLabel(probas_threshold=self._probas_threshold, k=self._k, tests=self._rocauc_tests),
+                RocAucByLabel(
+                    probas_threshold=self._probas_threshold, k=self._k, tests=self._get_tests(self._rocauc_tests)
+                ),
             ]
         )
 
@@ -192,6 +202,7 @@ def __init__(
     ):
         self._probas_threshold = probas_threshold
         self._k = k
+        super().__init__(include_tests=True)
 
     def generate_metrics(self, context: "Context") -> Sequence[MetricOrContainer]:
         return [
@@ -232,7 +243,9 @@ def __init__(
         precision_by_label_tests: ByLabelMetricTests = None,
         recall_by_label_tests: ByLabelMetricTests = None,
         rocauc_by_label_tests: ByLabelMetricTests = None,
+        include_tests: bool = True,
     ):
+        super().__init__(include_tests=include_tests)
         self._probas_threshold = probas_threshold
         self._quality = ClassificationQuality(
             probas_threshold=probas_threshold,
@@ -249,17 +262,19 @@ def __init__(
             tnr_tests=tnr_tests,
             fpr_tests=fpr_tests,
             fnr_tests=fnr_tests,
+            include_tests=include_tests,
         )
         self._quality_by_label = ClassificationQualityByLabel(
             probas_threshold=probas_threshold,
             f1score_tests=f1score_by_label_tests,
             precision_tests=precision_by_label_tests,
             recall_tests=recall_by_label_tests,
             rocauc_tests=rocauc_by_label_tests,
+            include_tests=include_tests,
         )
         self._roc_auc: Optional[RocAuc] = RocAuc(
             probas_threshold=probas_threshold,
-            tests=rocauc_tests,
+            tests=self._get_tests(rocauc_tests),
         )
 
     def generate_metrics(self, context: "Context") -> Sequence[MetricOrContainer]:

src/evidently/future/presets/dataset_stats.py

@@ -57,8 +57,9 @@ def __init__(
         q50_tests: SingleValueMetricTests = None,
         q75_tests: SingleValueMetricTests = None,
         unique_values_count_tests: ByLabelMetricTests = None,
+        include_tests: bool = True,
     ):
-        super().__init__(column=column)
+        super().__init__(column=column, include_tests=include_tests)
         self._row_count_tests = row_count_tests
         self._missing_values_count_tests = missing_values_count_tests
         self._min_tests = min_tests
@@ -72,23 +73,23 @@ def __init__(
 
     def generate_metrics(self, context: Context) -> Sequence[MetricOrContainer]:
         metrics: List[Metric] = [
-            RowCount(tests=self._row_count_tests),
-            MissingValueCount(column=self._column, tests=self._missing_values_count_tests),
+            RowCount(tests=self._get_tests(self._row_count_tests)),
+            MissingValueCount(column=self._column, tests=self._get_tests(self._missing_values_count_tests)),
         ]
         column_type = context.column(self._column).column_type
         if column_type == ColumnType.Numerical:
             metrics += [
-                MinValue(column=self._column, tests=self._min_tests),
-                MaxValue(column=self._column, tests=self._max_tests),
-                MeanValue(column=self._column, tests=self._mean_tests),
-                StdValue(column=self._column, tests=self._std_tests),
-                QuantileValue(column=self._column, quantile=0.25, tests=self._q25_tests),
-                QuantileValue(column=self._column, quantile=0.5, tests=self._q50_tests),
-                QuantileValue(column=self._column, quantile=0.75, tests=self._q75_tests),
+                MinValue(column=self._column, tests=self._get_tests(self._min_tests)),
+                MaxValue(column=self._column, tests=self._get_tests(self._max_tests)),
+                MeanValue(column=self._column, tests=self._get_tests(self._mean_tests)),
+                StdValue(column=self._column, tests=self._get_tests(self._std_tests)),
+                QuantileValue(column=self._column, quantile=0.25, tests=self._get_tests(self._q25_tests)),
+                QuantileValue(column=self._column, quantile=0.5, tests=self._get_tests(self._q50_tests)),
+                QuantileValue(column=self._column, quantile=0.75, tests=self._get_tests(self._q75_tests)),
             ]
         if column_type == ColumnType.Categorical:
             metrics += [
-                UniqueValueCount(column=self._column, tests=self._unique_values_count_tests),
+                UniqueValueCount(column=self._column, tests=self._get_tests(self._unique_values_count_tests)),
             ]
         if column_type == ColumnType.Datetime:
             metrics += [
@@ -313,6 +314,7 @@ def __init__(
         empty_column_count_tests: SingleValueMetricTests = None,
         constant_columns_count_tests: SingleValueMetricTests = None,
         dataset_missing_value_count_tests: SingleValueMetricTests = None,
+        include_tests: bool = True,
     ):
         self.duplicated_row_count_tests = duplicated_row_count_tests
         self.duplicated_column_count_tests = duplicated_column_count_tests
@@ -324,23 +326,24 @@ def __init__(
         self.dataset_missing_value_count_tests = dataset_missing_value_count_tests
         self.column_count_tests = column_count_tests
         self.row_count_tests = row_count_tests
+        super().__init__(include_tests=include_tests)
 
     def generate_metrics(self, context: Context) -> Sequence[MetricOrContainer]:
         return [
-            RowCount(tests=self.row_count_tests),
-            ColumnCount(tests=self.column_count_tests),
+            RowCount(tests=self._get_tests(self.row_count_tests)),
+            ColumnCount(tests=self._get_tests(self.column_count_tests)),
             ColumnCount(column_type=ColumnType.Numerical, tests=[]),
             ColumnCount(column_type=ColumnType.Categorical, tests=[]),
             ColumnCount(column_type=ColumnType.Datetime, tests=[]),
             ColumnCount(column_type=ColumnType.Text, tests=[]),
-            DuplicatedRowCount(tests=self.duplicated_row_count_tests),
-            DuplicatedColumnsCount(tests=self.duplicated_column_count_tests),
-            AlmostDuplicatedColumnsCount(tests=self.almost_duplicated_column_count_tests),
-            AlmostConstantColumnsCount(tests=self.almost_constant_column_count_tests),
-            EmptyRowsCount(tests=self.empty_row_count_tests),
-            EmptyColumnsCount(tests=self.empty_column_count_tests),
-            ConstantColumnsCount(tests=self.constant_columns_count_tests),
-            DatasetMissingValueCount(tests=self.dataset_missing_value_count_tests),
+            DuplicatedRowCount(tests=self._get_tests(self.duplicated_row_count_tests)),
+            DuplicatedColumnsCount(tests=self._get_tests(self.duplicated_column_count_tests)),
+            AlmostDuplicatedColumnsCount(tests=self._get_tests(self.almost_duplicated_column_count_tests)),
+            AlmostConstantColumnsCount(tests=self._get_tests(self.almost_constant_column_count_tests)),
+            EmptyRowsCount(tests=self._get_tests(self.empty_row_count_tests)),
+            EmptyColumnsCount(tests=self._get_tests(self.empty_column_count_tests)),
+            ConstantColumnsCount(tests=self._get_tests(self.constant_columns_count_tests)),
+            DatasetMissingValueCount(tests=self._get_tests(self.dataset_missing_value_count_tests)),
         ]
 
     def render(
@@ -375,20 +378,27 @@ def __init__(
         columns: Optional[List[str]] = None,
         row_count_tests: SingleValueMetricTests = None,
         column_tests: Optional[Dict[str, ValueStatsTests]] = None,
+        include_tests: bool = True,
     ):
         self._columns = columns
         self._value_stats: List[ValueStats] = []
         self._row_count_tests = row_count_tests
         self._column_tests = column_tests
+        super().__init__(include_tests=include_tests)
 
     def generate_metrics(self, context: Context) -> Sequence[MetricOrContainer]:
         if self._columns is None:
             cols = context.data_definition.numerical_descriptors + context.data_definition.categorical_descriptors
         else:
             cols = self._columns
-        metrics: List[MetricOrContainer] = [RowCount(tests=self._row_count_tests)]
+        metrics: List[MetricOrContainer] = [RowCount(tests=self._get_tests(self._row_count_tests))]
         self._value_stats = [
-            ValueStats(column, **(self._column_tests or {}).get(column, ValueStatsTests()).__dict__) for column in cols
+            ValueStats(
+                column,
+                **(self._column_tests or {}).get(column, ValueStatsTests()).__dict__,
+                include_tests=self.include_tests,
+            )
+            for column in cols
         ]
         metrics.extend(list(chain(*[vs.metrics(context)[1:] for vs in self._value_stats])))
         return metrics
@@ -419,6 +429,7 @@ def __init__(
         constant_columns_count_tests: SingleValueMetricTests = None,
         dataset_missing_value_count_tests: SingleValueMetricTests = None,
         column_tests: Optional[Dict[str, ValueStatsTests]] = None,
+        include_tests: bool = True,
     ):
         self.duplicated_row_count_tests = duplicated_row_count_tests
         self.duplicated_column_count_tests = duplicated_column_count_tests
@@ -432,6 +443,7 @@ def __init__(
         self.row_count_tests = row_count_tests
         self._columns = columns
         self._column_tests = column_tests
+        super().__init__(include_tests=include_tests)
 
     def generate_metrics(self, context: Context) -> Sequence[MetricOrContainer]:
         columns_ = context.data_definition.get_categorical_columns() + context.data_definition.get_numerical_columns()
@@ -446,8 +458,11 @@ def generate_metrics(self, context: Context) -> Sequence[MetricOrContainer]:
             empty_column_count_tests=self.empty_column_count_tests,
             constant_columns_count_tests=self.constant_columns_count_tests,
             dataset_missing_value_count_tests=self.dataset_missing_value_count_tests,
+            include_tests=self.include_tests,
+        )
+        self._text_evals = TextEvals(
+            self._columns or columns_, column_tests=self._column_tests, include_tests=self.include_tests
         )
-        self._text_evals = TextEvals(self._columns or columns_, column_tests=self._column_tests)
         return self._dataset_stats.metrics(context) + self._text_evals.metrics(context)
 
     def render(

src/evidently/future/presets/drift.py

@@ -53,6 +53,7 @@ def __init__(
         self.embeddings_drift_method = embeddings_drift_method
         self.embeddings = embeddings
         self.columns = columns
+        super().__init__(include_tests=True)
 
     def generate_metrics(self, context: Context) -> Sequence[MetricOrContainer]:
         types = [ColumnType.Numerical, ColumnType.Categorical, ColumnType.Text]