fix: csv reader resolve_type handle empty values

Author: hanchchch

lavender_data/shard/readers/csv.py

@@ -1,4 +1,4 @@
-import os
+import numpy as np
 import csv
 import sys
 import ast
@@ -17,14 +17,28 @@ class CsvReader(UntypedReader):
 
     def resolve_type(self, value: Any, typestr: str) -> type:
         if typestr in ["int", "int32", "int64"]:
+            if value == "":
+                return np.nan
             return int(value)
         elif typestr in ["float", "double"]:
+            if value == "":
+                return np.nan
             return float(value)
         elif typestr in ["string", "text", "str"]:
             return str(value)
         elif typestr in ["bool", "boolean"]:
             return value.lower() in ["true", "t", "yes", "y", "1"]
-        elif typestr in ["list", "map", "binary"]:
+        elif typestr in ["list"]:
+            if value == "":
+                return []
+            return ast.literal_eval(value)
+        elif typestr in ["map"]:
+            if value == "":
+                return {}
+            return ast.literal_eval(value)
+        elif typestr in ["binary"]:
+            if value == "":
+                return b""
             return ast.literal_eval(value)
         return value
 

lavender_data/shard/statistics.py

@@ -45,6 +45,10 @@ def _is_text_column(values: list[Any]) -> bool:
 
 
 def _is_categorical_column(values: list[Any]) -> bool:
+    # unhashable types are not categorical
+    if not _is_numeric_column(values) and not _is_text_column(values):
+        return False
+
     unique_values = set(values)
 
     if _is_numeric_column(values):