exclude invalid files reading the parquet dataset

Author: BerndDoser

src/spherinator/data/parquet_dataset.py

@@ -33,7 +33,11 @@ def __init__(
         if not isinstance(data_column, list):
             data_column = [data_column]
 
-        dataset = ds.dataset(data_directory, format="parquet")
+        dataset = ds.dataset(
+            data_directory,
+            format="parquet",
+            exclude_invalid_files=True,
+        )
         table = dataset.to_table(columns=data_column)
         self.transform = transform
         self.with_index = with_index