all passing

Author: czaloom

benchmarks/benchmark_objdet.py

@@ -9,7 +9,8 @@
 import requests
 from tqdm import tqdm
 
-from valor_lite.object_detection import BoundingBox, DataLoader, Detection
+from valor_lite.object_detection import BoundingBox, Detection
+from valor_lite.object_detection.loader import Loader
 
 
 def _get_bbox_extrema(
@@ -156,7 +157,7 @@ def write_results_to_file(write_path: Path, results: list[dict]):
 
 @time_it
 def ingest(
-    manager: DataLoader,
+    loader: Loader,
     gt_path: Path,
     pd_path: Path,
     limit: int,
@@ -184,12 +185,12 @@ def ingest(
                 elif len(detections) < chunk_size or chunk_size == -1:
                     continue
 
-                timer, _ = time_it(manager.add_bounding_boxes)(detections)
+                timer, _ = time_it(loader.add_bounding_boxes)(detections)
                 accumulated_time += timer
                 detections = []
 
             if detections:
-                timer, _ = time_it(manager.add_bounding_boxes)(detections)
+                timer, _ = time_it(loader.add_bounding_boxes)(detections)
                 accumulated_time += timer
 
     return accumulated_time
@@ -297,18 +298,25 @@ def run_benchmarking_analysis(
             pd_filename = prediction_caches[pd_type]
 
             # === Base Evaluation ===
-            manager = DataLoader()
+            loader = Loader.create(
+                ".valor/objdet_benchmark",
+                batch_size=1_000,
+                rows_per_file=10_000,
+                delete_if_exists=True,
+            )
 
             # ingest + preprocess
             (ingest_time, preprocessing_time,) = ingest(
-                manager=manager,
+                loader=loader,
                 gt_path=current_directory / Path(gt_filename),
                 pd_path=current_directory / Path(pd_filename),
                 limit=limit,
                 chunk_size=chunk_size,
             )  # type: ignore - time_it wrapper
 
-            finalization_time, evaluator = time_it(manager.finalize)()
+            finalization_time, evaluator = time_it(loader.finalize)(
+                batch_size=10_000
+            )
 
             if ingest_time > ingestion_timeout and ingestion_timeout != -1:
                 raise TimeoutError(
@@ -322,7 +330,7 @@ def run_benchmarking_analysis(
             )
             if eval_time > evaluation_timeout and evaluation_timeout != -1:
                 raise TimeoutError(
-                    f"Base evaluation timed out with {evaluator.metadata.number_of_datums} datums."
+                    f"Base evaluation timed out with {evaluator.info.number_of_datums} datums."
                 )
 
             # evaluate - base metrics + detailed
@@ -337,16 +345,16 @@ def run_benchmarking_analysis(
                 and evaluation_timeout != -1
             ):
                 raise TimeoutError(
-                    f"Detailed evaluation timed out with {evaluator.metadata.number_of_datums} datums."
+                    f"Detailed evaluation timed out with {evaluator.info.number_of_datums} datums."
                 )
 
             results.append(
                 Benchmark(
                     limit=limit,
-                    n_datums=evaluator.metadata.number_of_datums,
-                    n_groundtruths=evaluator.metadata.number_of_ground_truths,
-                    n_predictions=evaluator.metadata.number_of_predictions,
-                    n_labels=evaluator.metadata.number_of_labels,
+                    n_datums=evaluator.info.number_of_datums,
+                    n_groundtruths=evaluator.info.number_of_groundtruth_annotations,
+                    n_predictions=evaluator.info.number_of_prediction_annotations,
+                    n_labels=evaluator.info.number_of_labels,
                     gt_type=gt_type,
                     pd_type=pd_type,
                     chunk_size=chunk_size,

src/valor_lite/cache.py

@@ -68,21 +68,41 @@ def __init__(self, path: str | Path):
         self._path = Path(path)
 
     @property
-    def files(self) -> list[str]:
+    def path(self) -> Path:
+        return self._path
+
+    @property
+    def files(self) -> list[Path]:
+        if not self.path.exists():
+            return []
+        elif not self.path.is_dir():
+            raise NotADirectoryError(
+                f"Path exists but is not a directory: {self._path}"
+            )
+
         files = []
         for entry in os.listdir(self._path):
             full_path = os.path.join(self._path, entry)
             if os.path.isfile(full_path):
-                files.append(full_path)
+                files.append(Path(full_path))
         return files
 
     @property
     def num_files(self) -> int:
         return len(self.files)
 
     @property
-    def dataset_files(self) -> list[str]:
-        return glob.glob(f"{self._path}/*.parquet")
+    def dataset_files(self) -> list[Path]:
+        if not self.path.exists():
+            return []
+        elif not self.path.is_dir():
+            raise NotADirectoryError(
+                f"Path exists but is not a directory: {self._path}"
+            )
+
+        return [
+            Path(filepath) for filepath in glob.glob(f"{self._path}/*.parquet")
+        ]
 
     @property
     def num_dataset_files(self) -> int:
@@ -92,16 +112,19 @@ def num_dataset_files(self) -> int:
     def _generate_config_path(path: str | Path) -> Path:
         return Path(path) / ".cfg"
 
-    @staticmethod
-    def _get_dataset_from_path(path: str | Path) -> ds.Dataset:
-        return ds.dataset(path, format="parquet")
-
 
 class CacheReader(CacheFiles):
-    def __init__(self, path: str | Path):
+    def __init__(
+        self,
+        path: str | Path,
+        batch_size: int,
+        rows_per_file: int,
+        compression: str,
+    ):
         self._path = Path(path)
-        self._cfg = None
-        self._dataset = None
+        self._batch_size = batch_size
+        self._rows_per_file = rows_per_file
+        self._compression = compression
 
         # validate path
         if not self._path.exists():
@@ -111,45 +134,48 @@ def __init__(self, path: str | Path):
                 f"Path exists but is not a directory: {self._path}"
             )
 
+    @classmethod
+    def load(cls, path: str | Path):
+        def _retrieve(config: dict, key: str):
+            if value := config.get(key, None):
+                return value
+            raise KeyError(
+                f"'{key}' is not defined within {cls._generate_config_path(path)}"
+            )
+
+        cfg_path = cls._generate_config_path(path)
+        with open(cfg_path, "r") as f:
+            cfg = json.load(f)
+            batch_size = _retrieve(cfg, "batch_size")
+            rows_per_file = _retrieve(cfg, "rows_per_file")
+            compression = _retrieve(cfg, "compression")
+
+        return cls(
+            path=path,
+            batch_size=batch_size,
+            rows_per_file=rows_per_file,
+            compression=compression,
+        )
+
     @property
     def dataset(self) -> ds.Dataset:
-        if not self._dataset:
-            self._dataset = ds.dataset(
-                self._path,
-                format="parquet",
-            )
-        return self._dataset
+        return ds.dataset(self._path, format="parquet")
 
     @property
     def schema(self) -> pa.Schema:
         return self.dataset.schema
 
-    @property
-    def config(self) -> dict:
-        if self._cfg is None:
-            cfg_path = self._generate_config_path(self._path)
-            with open(cfg_path, "r") as f:
-                self._cfg = json.load(f)
-        return self._cfg
-
-    def _read_config(self, key: str):
-        if value := self.config.get(key, None):
-            return value
-        raise KeyError(
-            f"'{key}' is not defined within {self._generate_config_path(self._path)}"
-        )
-
     @property
     def batch_size(self) -> int:
-        return int(self._read_config("batch_size"))
+        return self._batch_size
 
     @property
     def rows_per_file(self) -> int:
-        return int(self._read_config("rows_per_file"))
+        return self._rows_per_file
 
     @property
     def compression(self) -> str:
-        return str(self._read_config("compression"))
+        return self._compression
 
 
 class CacheWriter(CacheFiles):
@@ -209,7 +235,7 @@ def create(
     @classmethod
     def load(cls, path: str | Path):
         cfg_path = cls._generate_config_path(path)
-        dataset = cls._get_dataset_from_path(path)
+        dataset = ds.dataset(path, format="parquet")
         with open(cfg_path, "r") as f:
             cfg = json.load(f)
         return cls(
@@ -218,6 +244,23 @@ def load(cls, path: str | Path):
             **cfg,
         )
 
+    @classmethod
+    def delete(cls, path: str | Path):
+        path = Path(path)
+        if not path.exists():
+            return
+        cache = cls.load(path)
+        # delete config file
+        cfg_path = cls._generate_config_path(path)
+        if cfg_path.exists() and cfg_path.is_file():
+            cfg_path.unlink()
+        # delete parquet files
+        for file in cache.dataset_files:
+            if file.exists() and file.is_file() and file.suffix == ".parquet":
+                file.unlink()
+        # delete empty cache directory
+        path.rmdir()
+
     def write_rows(
         self,
         rows: list[dict[str, Any]],
@@ -286,10 +329,6 @@ def flush(self):
         self._count = 0
         self._close_writer()
 
-    def delete(self):
-        for file in self.files:
-            Path(file).unlink()
-
     def _next_filename(self) -> Path:
         files = self.dataset_files
         if not files:

src/valor_lite/object_detection/evaluator.py

@@ -68,30 +68,16 @@ def __init__(
             number_of_groundtruths_per_label
         )
 
-    @property
-    def path(self) -> Path:
-        return self._path
-
-    @property
-    def detailed(self) -> CacheReader:
-        return self._detailed_cache
-
-    @property
-    def ranked(self) -> CacheReader:
-        return self._ranked_cache
-
-    @property
-    def info(self) -> EvaluatorInfo:
-        return self._info
-
     @classmethod
     def load(
         cls,
         path: str | Path,
         index_to_label_override: dict[int, str] | None = None,
     ):
-        detailed_cache = CacheReader(cls._generate_detailed_cache_path(path))
-        ranked_cache = CacheReader(cls._generate_ranked_cache_path(path))
+        detailed_cache = CacheReader.load(
+            cls._generate_detailed_cache_path(path)
+        )
+        ranked_cache = CacheReader.load(cls._generate_ranked_cache_path(path))
 
         # build evaluator meta
         (
@@ -215,6 +201,30 @@ def filter(
             index_to_label_override=self._index_to_label,
         )
 
+    def delete(self):
+        """
+        Delete evaluator cache.
+        """
+        from valor_lite.object_detection.loader import Loader
+
+        Loader.delete(self.path)
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    @property
+    def detailed(self) -> CacheReader:
+        return self._detailed_cache
+
+    @property
+    def ranked(self) -> CacheReader:
+        return self._ranked_cache
+
+    @property
+    def info(self) -> EvaluatorInfo:
+        return self._info
+
     @staticmethod
     def generate_meta(
         dataset: ds.Dataset,

src/valor_lite/object_detection/legacy.py

@@ -1,6 +1,5 @@
 import tempfile
 from dataclasses import asdict, dataclass
-from pathlib import Path
 
 import numpy as np
 import pyarrow.compute as pc
@@ -334,20 +333,3 @@ class DataLoader(CachedLoader):
     def finalize(self) -> Evaluator:  # type: ignore - switching type
         evaluator = super().finalize()
         return Evaluator.load(evaluator.path)
-
-    @classmethod
-    def filter(
-        cls,
-        path: str | Path,
-        evaluator: CachedEvaluator,
-        filter_expr: Filter,
-    ) -> Evaluator:
-        evaluator = super().filter(
-            path=path,
-            evaluator=evaluator,
-            filter_expr=filter_expr,
-        )
-        return Evaluator.load(
-            path=path,
-            index_to_label_override=evaluator._index_to_label,
-        )