annotation filtering tests

Author: czaloom

src/valor_lite/object_detection/legacy.py

@@ -1,5 +1,6 @@
 import tempfile
 from dataclasses import asdict, dataclass
+from pathlib import Path
 
 import numpy as np
 import pyarrow.compute as pc
@@ -44,33 +45,30 @@ def to_dict(self) -> dict[str, int | bool]:
         return asdict(self)
 
 
-class Evaluator:
+class Evaluator(CachedEvaluator):
     """
     Legacy Object Detection Evaluator
     """
 
-    def __init__(self, name: str = "default"):
-        self._evaluator = CachedEvaluator(name=name)
-
     @property
     def metadata(self) -> Metadata:
         """
         Evaluation metadata.
         """
         return Metadata(
-            number_of_datums=self._evaluator.info.number_of_datums,
-            number_of_labels=self._evaluator.info.number_of_labels,
-            number_of_ground_truths=self._evaluator.info.number_of_groundtruth_annotations,
-            number_of_predictions=self._evaluator.info.number_of_prediction_annotations,
+            number_of_datums=self.info.number_of_datums,
+            number_of_labels=self.info.number_of_labels,
+            number_of_ground_truths=self.info.number_of_groundtruth_annotations,
+            number_of_predictions=self.info.number_of_prediction_annotations,
         )
 
     @property
     def _detailed_pairs(self) -> np.ndarray:
         return np.concatenate(
             [
                 pairs
-                for pairs in self._evaluator.iterate_pairs(
-                    self._evaluator._dataset,
+                for pairs in self.iterate_pairs(
+                    self._dataset,
                     columns=[
                         "datum_id",
                         "gt_id",
@@ -87,7 +85,7 @@ def _detailed_pairs(self) -> np.ndarray:
     @property
     def _label_metadata(self) -> np.ndarray:
         label_metadata = np.zeros(
-            (len(self._evaluator._index_to_label), 2), dtype=np.int32
+            (len(self._index_to_label), 2), dtype=np.int32
         )
 
         # groundtruth labels
@@ -114,9 +112,10 @@ def _label_metadata(self) -> np.ndarray:
 
         return label_metadata
 
-    def filter(
-        self, filter_: Filter
-    ) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.int32],]:
+    def filter(  # type: ignore - legacy function override does not match
+        self,
+        filter_: Filter,
+    ) -> tuple[NDArray[np.float64], NDArray[np.float64], NDArray[np.int32]]:
         """
         Performs filtering over the internal cache.
 
@@ -135,12 +134,17 @@ def filter(
             Label metadata.
         """
         with tempfile.TemporaryDirectory() as tmpdir:
-            evaluator = Evaluator()
-            evaluator._evaluator = self._evaluator.filter(
+            name = "filtered"
+            _evaluator = super().filter(
                 directory=tmpdir,
-                name="filtered",
+                name=name,
                 filter_expr=filter_,
             )
+            evaluator = Evaluator(
+                name=name,
+                directory=tmpdir,
+                labels_override=_evaluator._index_to_label,
+            )
             detailed_pairs = evaluator._detailed_pairs
             label_metadata = evaluator._label_metadata
             return detailed_pairs, detailed_pairs, label_metadata
@@ -229,7 +233,7 @@ def compute_precision_recall(
         """
         if filter_ is not None:
             with tempfile.TemporaryDirectory() as tmpdir:
-                evaluator = self._evaluator.filter(
+                evaluator = super().filter(
                     directory=tmpdir,
                     name="filtered",
                     filter_expr=filter_,
@@ -238,7 +242,7 @@ def compute_precision_recall(
                     iou_thresholds=iou_thresholds,
                     score_thresholds=score_thresholds,
                 )
-        return self._evaluator.compute_precision_recall(
+        return super().compute_precision_recall(
             iou_thresholds=iou_thresholds,
             score_thresholds=score_thresholds,
         )
@@ -268,7 +272,7 @@ def compute_confusion_matrix(
         """
         if filter_ is not None:
             with tempfile.TemporaryDirectory() as tmpdir:
-                evaluator = self._evaluator.filter(
+                evaluator = super().filter(
                     directory=tmpdir,
                     name="filtered",
                     filter_expr=filter_,
@@ -278,7 +282,7 @@ def compute_confusion_matrix(
                     score_thresholds=score_thresholds,
                 )
         else:
-            metrics = self._evaluator.compute_confusion_matrix_with_examples(
+            metrics = super().compute_confusion_matrix_with_examples(
                 iou_thresholds=iou_thresholds,
                 score_thresholds=score_thresholds,
             )
@@ -328,12 +332,29 @@ class DataLoader(CachedLoader):
     Legacy Object Detection DataLoader
     """
 
-    def __init__(self):
-        super().__init__(
-            batch_size=1_000,
-            rows_per_file=10_000,
+    def finalize(self) -> Evaluator:  # type: ignore - switching type
+        evaluator = super().finalize()
+        return Evaluator(
+            name=evaluator._name,
+            directory=evaluator._directory,
         )
 
-    def finalize(self) -> Evaluator:  # type: ignore - switching type
-        _ = super().finalize()
-        return Evaluator()
+    @classmethod
+    def filter(
+        cls,
+        directory: str | Path,
+        name: str,
+        evaluator: CachedEvaluator,
+        filter_expr: Filter,
+    ) -> Evaluator:
+        evaluator = super().filter(
+            directory=directory,
+            name=name,
+            evaluator=evaluator,
+            filter_expr=filter_expr,
+        )
+        return Evaluator(
+            directory=evaluator._directory,
+            name=evaluator._name,
+            labels_override=evaluator._index_to_label,
+        )

src/valor_lite/object_detection/loader.py

@@ -390,24 +390,27 @@ def filter(
             gt_ids = pairs[:, (0, 1)].astype(np.int64)
             pd_ids = pairs[:, (0, 2)].astype(np.int64)
 
-            mask_valid_gt = np.zeros(n_pairs, dtype=np.bool_)
-            mask_valid_pd = np.zeros(n_pairs, dtype=np.bool_)
-
             if filter_expr.groundtruths is not None:
+                mask_valid_gt = np.zeros(n_pairs, dtype=np.bool_)
                 gt_tbl = tbl.filter(filter_expr.groundtruths)
                 gt_pairs = np.column_stack(
                     [gt_tbl[col].to_numpy() for col in ("datum_id", "gt_id")]
                 ).astype(np.int64)
                 for gt in np.unique(gt_pairs, axis=0):
                     mask_valid_gt |= (gt_ids == gt).all(axis=1)
+            else:
+                mask_valid_gt = np.ones(n_pairs, dtype=np.bool_)
 
             if filter_expr.predictions is not None:
+                mask_valid_pd = np.zeros(n_pairs, dtype=np.bool_)
                 pd_tbl = tbl.filter(filter_expr.predictions)
                 pd_pairs = np.column_stack(
                     [pd_tbl[col].to_numpy() for col in ("datum_id", "pd_id")]
                 ).astype(np.int64)
                 for pd in np.unique(pd_pairs, axis=0):
                     mask_valid_pd |= (pd_ids == pd).all(axis=1)
+            else:
+                mask_valid_pd = np.ones(n_pairs, dtype=np.bool_)
 
             mask_valid = mask_valid_gt | mask_valid_pd
             mask_valid_gt &= mask_valid

tests/object_detection/conftest.py

@@ -181,6 +181,9 @@ def basic_detections(
                     ymin=rect1[2],
                     ymax=rect1[3],
                     labels=["v1"],
+                    metadata={
+                        "gt_rect": "rect1",
+                    },
                 ),
                 BoundingBox(
                     uid=str(uuid4()),
@@ -189,6 +192,9 @@ def basic_detections(
                     ymin=rect3[2],
                     ymax=rect3[3],
                     labels=["v2"],
+                    metadata={
+                        "gt_rect": "rect3",
+                    },
                 ),
             ],
             predictions=[
@@ -200,6 +206,9 @@ def basic_detections(
                     ymax=rect1[3],
                     labels=["v1"],
                     scores=[0.3],
+                    metadata={
+                        "pd_rect": "rect1",
+                    },
                 ),
             ],
         ),
@@ -213,6 +222,9 @@ def basic_detections(
                     ymin=rect2[2],
                     ymax=rect2[3],
                     labels=["v1"],
+                    metadata={
+                        "gt_rect": "rect2",
+                    },
                 ),
             ],
             predictions=[
@@ -224,6 +236,9 @@ def basic_detections(
                     ymax=rect2[3],
                     labels=["v2"],
                     scores=[0.98],
+                    metadata={
+                        "pd_rect": "rect2",
+                    },
                 ),
             ],
         ),

tests/object_detection/test_dataloader.py

@@ -76,7 +76,7 @@ def test_iou_computation():
     loader.add_bounding_boxes([detection])
     evaluator = loader.finalize()
 
-    tbl = evaluator._evaluator._dataset.to_table()
+    tbl = evaluator._dataset.to_table()
     assert tbl.shape == (7, 12)
 
     # show that three unique IOUs exist

tests/object_detection/test_filtering.py

@@ -2,15 +2,18 @@
 from uuid import uuid4
 
 import numpy as np
+import pyarrow.compute as pc
 import pytest
 
+from valor_lite.cache import DataType
 from valor_lite.exceptions import EmptyCacheError, EmptyFilterError
 from valor_lite.object_detection import (
     BoundingBox,
     DataLoader,
     Detection,
     MetricType,
 )
+from valor_lite.object_detection.evaluator import Filter
 
 
 @pytest.fixture
@@ -590,3 +593,110 @@ def test_filtering_four_detections_by_indices(
         assert m in expected_metrics
     for m in expected_metrics:
         assert m in actual_metrics
+
+
+def test_filtering_four_detections_by_annotation_metadata(
+    four_detections: list[Detection],
+):
+    """
+    Basic object detection test that combines the labels of basic_detections_first_class and basic_detections_second_class.
+
+    groundtruths
+        datum uid1
+            box 1 - label v1 - tp
+            box 3 - label v2 - fn unmatched ground truths
+        datum uid2
+            box 2 - label v1 - fn misclassification
+        datum uid3
+            box 1 - label v1 - tp
+            box 3 - label v2 - fn unmatched ground truths
+        datum uid4
+            box 2 - label v1 - fn misclassification
+
+    predictions
+        datum uid1
+            box 1 - label v1 - score 0.3 - tp
+        datum uid2
+            box 2 - label v2 - score 0.98 - fp misclassification
+        datum uid3
+            box 1 - label v1 - score 0.3 - tp
+        datum uid4
+            box 2 - label v2 - score 0.98 - fp misclassification
+    """
+
+    loader = DataLoader(
+        groundtruth_metadata_types={
+            "gt_rect": DataType.STRING,
+        },
+        prediction_metadata_types={
+            "pd_rect": DataType.STRING,
+        },
+    )
+    loader.add_bounding_boxes(four_detections)
+    evaluator = loader.finalize()
+
+    # remove all FN groundtruths
+    filter_ = Filter(
+        groundtruths=pc.field("gt_rect") == "rect1",
+    )
+    metrics = evaluator.evaluate(
+        iou_thresholds=[0.5], score_thresholds=[0.1], filter_=filter_
+    )
+    actual_metrics = [m.to_dict() for m in metrics[MetricType.Counts]]
+    expected_metrics = [
+        {
+            "type": "Counts",
+            "value": {"tp": 2, "fp": 0, "fn": 0},
+            "parameters": {
+                "iou_threshold": 0.5,
+                "score_threshold": 0.1,
+                "label": "v1",
+            },
+        },
+        {
+            "type": "Counts",
+            "value": {"tp": 0, "fp": 2, "fn": 0},
+            "parameters": {
+                "iou_threshold": 0.5,
+                "score_threshold": 0.1,
+                "label": "v2",
+            },
+        },
+    ]
+    for m in actual_metrics:
+        assert m in expected_metrics
+    for m in expected_metrics:
+        assert m in actual_metrics
+
+    # remove TP ground truths
+    filter_ = Filter(
+        groundtruths=pc.field("gt_rect") != "rect1",
+    )
+    metrics = evaluator.evaluate(
+        iou_thresholds=[0.5], score_thresholds=[0.1], filter_=filter_
+    )
+    actual_metrics = [m.to_dict() for m in metrics[MetricType.Counts]]
+    expected_metrics = [
+        {
+            "type": "Counts",
+            "value": {"tp": 0, "fp": 2, "fn": 2},
+            "parameters": {
+                "iou_threshold": 0.5,
+                "score_threshold": 0.1,
+                "label": "v1",
+            },
+        },
+        {
+            "type": "Counts",
+            "value": {"tp": 0, "fp": 2, "fn": 2},
+            "parameters": {
+                "iou_threshold": 0.5,
+                "score_threshold": 0.1,
+                "label": "v2",
+            },
+        },
+    ]
+    for m in actual_metrics:
+        assert m in expected_metrics
+    for m in expected_metrics:
+        assert m in actual_metrics