remove label metadata

Author: czaloom

src/valor_lite/semantic_segmentation/computation.py

@@ -2,39 +2,12 @@
 from numpy.typing import NDArray
 
 
-def compute_label_metadata(
-    confusion_matrices: NDArray[np.int64],
-    n_labels: int,
-) -> NDArray[np.int64]:
-    """
-    Computes label metadata returning a count of annotations per label.
-
-    Parameters
-    ----------
-    confusion_matrices : NDArray[np.int64]
-        Confusion matrices per datum with shape (n_datums, n_labels + 1, n_labels + 1).
-    n_labels : int
-        The total number of unique labels.
-
-    Returns
-    -------
-    NDArray[np.int64]
-        The label metadata array with shape (n_labels, 2).
-            Index 0 - Ground truth label count
-            Index 1 - Prediction label count
-    """
-    label_metadata = np.zeros((n_labels, 2), dtype=np.int64)
-    label_metadata[:, 0] = confusion_matrices[:, 1:, :].sum(axis=(0, 2))
-    label_metadata[:, 1] = confusion_matrices[:, :, 1:].sum(axis=(0, 1))
-    return label_metadata
-
-
 def filter_cache(
     confusion_matrices: NDArray[np.int64],
     datum_mask: NDArray[np.bool_],
     label_mask: NDArray[np.bool_],
     number_of_labels: int,
-) -> tuple[NDArray[np.int64], NDArray[np.int64]]:
+) -> tuple[NDArray[np.int64]]:
     """
     Performs the filter operation over the internal cache.
 
@@ -75,11 +48,7 @@ def filter_cache(
 
     confusion_matrices = confusion_matrices[datum_mask]
 
-    label_metadata = compute_label_metadata(
-        confusion_matrices=confusion_matrices,
-        n_labels=number_of_labels,
-    )
-    return confusion_matrices, label_metadata
+    return confusion_matrices
 
 
 def compute_intermediates(
@@ -141,7 +110,6 @@ def compute_intermediates(
 
 def compute_metrics(
     confusion_matrices: NDArray[np.int64],
-    label_metadata: NDArray[np.int64],
     n_pixels: int,
 ) -> tuple[
     NDArray[np.float64],
@@ -183,7 +151,10 @@ def compute_metrics(
     NDArray[np.float64]
         Unmatched ground truth ratios.
     """
-    n_labels = label_metadata.shape[0]
+    n_labels = confusion_matrices.shape[-1] - 1
+    label_metadata = np.zeros((n_labels, 2), dtype=np.int64)
+    label_metadata[:, 0] = confusion_matrices[:, 1:, :].sum(axis=(0, 2))
+    label_metadata[:, 1] = confusion_matrices[:, :, 1:].sum(axis=(0, 1))
     gt_counts = label_metadata[:, 0]
     pd_counts = label_metadata[:, 1]
 

src/valor_lite/semantic_segmentation/evaluator.py

@@ -1,16 +1,20 @@
+import json
+from collections import defaultdict
 from dataclasses import asdict, dataclass
+from pathlib import Path
 
-import json
 import numpy as np
 from numpy.typing import NDArray
-from tqdm import tqdm
-from pathlib import Path
-from collections import defaultdict
-
 from pyarrow import pa
 from pyarrow.compute import pc
 from pyarrow.dataset import ds
+from tqdm import tqdm
 
+from valor_lite.cache import (
+    CacheReader,
+    DataType,
+    convert_type_mapping_to_schema,
+)
 from valor_lite.exceptions import EmptyCacheError, EmptyFilterError
 from valor_lite.semantic_segmentation.annotation import Segmentation
 from valor_lite.semantic_segmentation.computation import (
@@ -23,15 +27,15 @@
 from valor_lite.semantic_segmentation.utilities import (
     unpack_precision_recall_iou_into_metric_lists,
 )
-from valor_lite.cache import CacheReader, DataType, convert_type_mapping_to_schema
 
 
 @dataclass
 class EvaluatorInfo:
     number_of_datums: int = 0
-    number_of_groundtruth_annotations: int = 0
-    number_of_prediction_annotations: int = 0
     number_of_labels: int = 0
+    number_of_pixels: int = 0
+    number_of_groundtruth_pixels: int = 0
+    number_of_prediction_pixels: int = 0
     number_of_rows: int = 0
     datum_metadata_types: dict[str, DataType] | None = None
     groundtruth_metadata_types: dict[str, DataType] | None = None
@@ -120,10 +124,9 @@ def generate_meta(
             tbl = fragment.to_table()
             columns = (
                 "datum_id",
-                "gt_id",
-                "pd_id",
                 "gt_label_id",
                 "pd_label_id",
+                "counts",
             )
             ids = np.column_stack(
                 [tbl[col].to_numpy() for col in columns]
@@ -136,18 +139,8 @@ def generate_meta(
             datum_ids = np.unique(ids[:, 0])
             info.number_of_datums += int(datum_ids.size)
 
-            # count unique groundtruths
-            gt_ids = ids[:, 1]
-            gt_ids = np.unique(gt_ids[gt_ids >= 0])
-            info.number_of_groundtruth_annotations += int(gt_ids.shape[0])
-
-            # count unique predictions
-            pd_ids = ids[:, 2]
-            pd_ids = np.unique(pd_ids[pd_ids >= 0])
-            info.number_of_prediction_annotations += int(pd_ids.shape[0])
-
             # get gt labels
-            gt_label_ids = ids[:, 3]
+            gt_label_ids = ids[:, 1]
             gt_label_ids, gt_indices = np.unique(
                 gt_label_ids, return_index=True
             )
@@ -157,17 +150,17 @@ def generate_meta(
             labels.update(gt_labels)
 
             # get pd labels
-            pd_label_ids = ids[:, 4]
-            pd_label_ids, pd_indices = np.unique(
-                pd_label_ids, return_index=True
+            pd_label_ids = ids[:, 2]
+            pd_label_ids, pd_indices, pd_counts = np.unique(
+                pd_label_ids, return_index=True, return_counts=True
             )
             pd_labels = tbl["pd_label"].take(pd_indices).to_pylist()
             pd_labels = dict(zip(pd_label_ids.astype(int).tolist(), pd_labels))
             pd_labels.pop(-1, None)
             labels.update(pd_labels)
 
             # count gts per label
-            gts = ids[:, (1, 3)].astype(np.int64)
+            gts = ids[:, 1].astype(np.int64)
             unique_ann = np.unique(gts[gts[:, 0] >= 0], axis=0)
             unique_labels, label_counts = np.unique(
                 unique_ann[:, 1], return_counts=True
@@ -181,14 +174,35 @@ def generate_meta(
         # complete info object
         info.number_of_labels = len(labels)
 
-        # convert gt counts to numpy
-        number_of_groundtruths_per_label = np.zeros(
-            len(labels), dtype=np.uint64
-        )
-        for k, v in gt_counts_per_lbl.items():
-            number_of_groundtruths_per_label[int(k)] = v
+        # create confusion matrix
+        n_labels = len(labels)
+        matrix = np.zeros((n_labels + 1, n_labels + 1), dtype=np.uint64)
+        for fragment in dataset.get_fragments():
+            tbl = fragment.to_table()
+            columns = (
+                "datum_id",
+                "gt_label_id",
+                "pd_label_id",
+            )
+            ids = np.column_stack(
+                [tbl[col].to_numpy() for col in columns]
+            ).astype(np.int64)
+            counts = tbl["counts"].to_numpy()
+
+            for idx in range(n_labels):
+                mask_gts = ids[:, 1] == idx
+                for pidx in range(n_labels):
+                    mask_pds = ids[:, 2] == pidx
+                    matrix[idx + 1, pidx + 1] = counts[
+                        mask_gts & mask_pds
+                    ].sum()
+
+                mask_unmatched_gts = mask_gts & (ids[:, 2] == -1)
+                matrix[idx + 1, 0] = counts[mask_unmatched_gts].sum()
+                mask_unmatched_pds = (ids[:, 1] == -1) & (ids[:, 2] == idx)
+                matrix[0, idx + 1] = counts[mask_unmatched_pds]
 
-        return labels, number_of_groundtruths_per_label, info
+        return labels, matrix, info
 
     @staticmethod
     def iterate_pairs(
@@ -241,8 +255,8 @@ def filter(
         from valor_lite.semantic_segmentation.loader import Loader
 
         return Loader.filter(
-            directory=directory,
             name=name,
+            directory=directory,
             evaluator=self,
             filter_expr=filter_expr,
-        )
+        )

src/valor_lite/semantic_segmentation/manager.py

@@ -8,7 +8,6 @@
 from valor_lite.semantic_segmentation.annotation import Segmentation
 from valor_lite.semantic_segmentation.computation import (
     compute_intermediates,
-    compute_label_metadata,
     compute_metrics,
     filter_cache,
 )
@@ -103,28 +102,6 @@ def __init__(self):
     def metadata(self) -> Metadata:
         return self._metadata
 
-    @property
-    def ignored_prediction_labels(self) -> list[str]:
-        """
-        Prediction labels that are not present in the ground truth set.
-        """
-        glabels = set(np.where(self._label_metadata[:, 0] > 0)[0])
-        plabels = set(np.where(self._label_metadata[:, 1] > 0)[0])
-        return [
-            self.index_to_label[label_id] for label_id in (plabels - glabels)
-        ]
-
-    @property
-    def missing_prediction_labels(self) -> list[str]:
-        """
-        Ground truth labels that are not present in the prediction set.
-        """
-        glabels = set(np.where(self._label_metadata[:, 0] > 0)[0])
-        plabels = set(np.where(self._label_metadata[:, 1] > 0)[0])
-        return [
-            self.index_to_label[label_id] for label_id in (glabels - plabels)
-        ]
-
     def create_filter(
         self,
         datums: list[str] | NDArray[np.int64] | None = None,
@@ -208,7 +185,7 @@ def create_filter(
             ).any(axis=1)
             label_mask[~mask_valid_labels] = True
 
-        filtered_confusion_matrices, _ = filter_cache(
+        filtered_confusion_matrices = filter_cache(
             confusion_matrices=self._confusion_matrices.copy(),
             datum_mask=datum_mask,
             label_mask=label_mask,
@@ -223,9 +200,7 @@ def create_filter(
             ),
         )
 
-    def filter(
-        self, filter_: Filter
-    ) -> tuple[NDArray[np.int64], NDArray[np.int64]]:
+    def filter(self, filter_: Filter) -> tuple[NDArray[np.int64]]:
         """
         Performs the filter operation over the internal cache.
 
@@ -260,7 +235,7 @@ def compute_precision_recall_iou(
             A dictionary mapping MetricType enumerations to lists of computed metrics.
         """
         if filter_ is not None:
-            confusion_matrices, label_metadata = self.filter(filter_)
+            confusion_matrices = self.filter(filter_)
             n_pixels = filter_.metadata.number_of_pixels
         else:
             confusion_matrices = self._confusion_matrices
@@ -269,12 +244,10 @@ def compute_precision_recall_iou(
 
         results = compute_metrics(
             confusion_matrices=confusion_matrices,
-            label_metadata=label_metadata,
             n_pixels=n_pixels,
         )
         return unpack_precision_recall_iou_into_metric_lists(
             results=results,
-            label_metadata=label_metadata,
             index_to_label=self.index_to_label,
         )
 
@@ -436,10 +409,6 @@ def finalize(self) -> Evaluator:
         for idx, matrix in enumerate(self.matrices):
             h, w = matrix.shape
             self._evaluator._confusion_matrices[idx, :h, :w] = matrix
-        self._evaluator._label_metadata = compute_label_metadata(
-            confusion_matrices=self._evaluator._confusion_matrices,
-            n_labels=n_labels,
-        )
         self._evaluator._metadata = Metadata.create(
             confusion_matrices=self._evaluator._confusion_matrices,
         )

src/valor_lite/semantic_segmentation/utilities.py

@@ -8,7 +8,6 @@
 
 def unpack_precision_recall_iou_into_metric_lists(
     results: tuple,
-    label_metadata: NDArray[np.int64],
     index_to_label: list[str],
 ) -> dict[MetricType, list[Metric]]:
 
@@ -39,24 +38,20 @@ def unpack_precision_recall_iou_into_metric_lists(
                         "iou": float(ious[gt_label_idx, pd_label_idx])
                     }
                     for pd_label_idx in range(n_labels)
-                    if label_metadata[pd_label_idx, 0] > 0
                 }
                 for gt_label_idx in range(n_labels)
-                if label_metadata[gt_label_idx, 0] > 0
             },
             unmatched_predictions={
                 index_to_label[pd_label_idx]: {
                     "ratio": float(unmatched_prediction_ratios[pd_label_idx])
                 }
                 for pd_label_idx in range(n_labels)
-                if label_metadata[pd_label_idx, 0] > 0
             },
             unmatched_ground_truths={
                 index_to_label[gt_label_idx]: {
                     "ratio": float(unmatched_ground_truth_ratios[gt_label_idx])
                 }
                 for gt_label_idx in range(n_labels)
-                if label_metadata[gt_label_idx, 0] > 0
             },
         )
     ]
@@ -73,10 +68,6 @@ def unpack_precision_recall_iou_into_metric_lists(
             "label": label,
         }
 
-        # if no groundtruths exists for a label, skip it.
-        if label_metadata[label_idx, 0] == 0:
-            continue
-
         metrics[MetricType.Precision].append(
             Metric.precision(
                 value=float(precision[label_idx]),