Use string type for null-columns in CSV (#1147)

Author: dreadatour

src/datachain/lib/arrow.py

@@ -241,6 +241,8 @@ def arrow_type_mapper(col_type: pa.DataType, column: str = "") -> type:  # noqa:
         return dict
     if isinstance(col_type, pa.lib.DictionaryType):
         return arrow_type_mapper(col_type.value_type)  # type: ignore[return-value]
+    if pa.types.is_null(col_type):
+        return str  # use strings for null columns
     raise TypeError(f"{col_type!r} datatypes not supported, column: {column}")
 
 

tests/unit/lib/test_arrow.py

@@ -154,18 +154,19 @@ def test_arrow_generator_partitioned(tmp_path, catalog, cache):
         (pa.map_(pa.string(), pa.int32()), dict),
         (pa.dictionary(pa.int64(), pa.string()), str),
         (pa.list_(pa.string()), list[str]),
+        (pa.null(), str),
     ),
 )
 def test_arrow_type_mapper(col_type, expected):
     assert arrow_type_mapper(col_type) == expected
 
 
 def test_arrow_type_mapper_struct():
-    col_type = pa.struct({"x": pa.int32(), "y": pa.string()})
+    col_type = pa.struct({"x": pa.int32(), "y": pa.string(), "z": pa.null()})
     fields = arrow_type_mapper(col_type).model_fields
-    assert list(fields.keys()) == ["x", "y"]
+    assert list(fields.keys()) == ["x", "y", "z"]
     dtypes = [field.annotation for field in fields.values()]
-    assert dtypes == [Optional[int], Optional[str]]
+    assert dtypes == [Optional[int], Optional[str], Optional[str]]
 
 
 def test_arrow_type_error():