added cache tests

Author: czaloom

src/valor_lite/cache.py

@@ -1,5 +1,6 @@
 import glob
 import json
+import os
 from datetime import datetime
 from enum import StrEnum
 from pathlib import Path
@@ -75,7 +76,12 @@ def __init__(self, where: str | Path):
 
     @property
     def files(self) -> list[str]:
-        return glob.glob(f"{self._dir}/*")
+        files = []
+        for entry in os.listdir(self._dir):
+            full_path = os.path.join(self._dir, entry)
+            if os.path.isfile(full_path):
+                files.append(full_path)
+        return files
 
     @property
     def num_files(self) -> int:

tests/common/test_cache.py

@@ -5,7 +5,12 @@
 import numpy as np
 import pyarrow as pa
 
-from valor_lite.cache import CacheWriter, DataType
+from valor_lite.cache import (
+    CacheReader,
+    CacheWriter,
+    DataType,
+    convert_type_mapping_to_schema,
+)
 
 
 def test_datatype_casting_to_arrow():
@@ -22,6 +27,22 @@ def test_datatype_casting_to_python():
     assert DataType.TIMESTAMP.to_py() is datetime
 
 
+def test_convert_type_mapping_to_schema():
+    x = convert_type_mapping_to_schema(
+        {
+            "a": DataType.FLOAT,
+            "b": DataType.STRING,
+        }
+    )
+    assert x == [
+        ("a", pa.float64()),
+        ("b", pa.string()),
+    ]
+
+    assert convert_type_mapping_to_schema({}) == []
+    assert convert_type_mapping_to_schema(None) == []
+
+
 def test_cache_write_batch():
     batch_size = 10
     rows_per_file = 100
@@ -47,7 +68,7 @@ def test_cache_write_batch():
                 }
             )
         cache.flush()
-        assert cache.num_files == 10
+        assert cache.num_files == 11
         for idx, fragment in enumerate(cache.dataset.get_fragments()):
             tbl = fragment.to_table()
             assert tbl["some_int"].to_pylist() == [
@@ -91,7 +112,7 @@ def test_cache_write_rows():
                 ]
             )
         cache.flush()
-        assert cache.num_files == 10
+        assert cache.num_files == 11
         for idx, fragment in enumerate(cache.dataset.get_fragments()):
             tbl = fragment.to_table()
             assert tbl["some_int"].to_pylist() == [
@@ -135,9 +156,9 @@ def test_cache_write_table():
             ]
         )
         cache.write_table(tbl)
-        assert cache.num_files == 1
-        cache.write_table(tbl)
         assert cache.num_files == 2
+        cache.write_table(tbl)
+        assert cache.num_files == 3
         cache.flush()
         for _, fragment in enumerate(cache.dataset.get_fragments()):
             tbl = fragment.to_table()
@@ -146,3 +167,57 @@ def test_cache_write_table():
             assert tbl["some_str"].to_pylist() == [
                 f"str{i}" for i in range(101)
             ]
+
+
+def test_cache_reader():
+    batch_size = 10
+    rows_per_file = 100
+    with tempfile.TemporaryDirectory() as tmpdir:
+        cache = CacheWriter(
+            where=Path(tmpdir),
+            schema=pa.schema(
+                [
+                    ("some_int", pa.int64()),
+                    ("some_float", pa.float64()),
+                    ("some_str", pa.string()),
+                ]
+            ),
+            batch_size=batch_size,
+            rows_per_file=rows_per_file,
+        )
+        tbl = pa.Table.from_pylist(
+            [
+                {
+                    "some_int": i,
+                    "some_float": np.float64(i),
+                    "some_str": f"str{i}",
+                }
+                for i in range(101)
+            ]
+        )
+        cache.write_table(tbl)
+
+        readonly_cache = CacheReader(where=Path(tmpdir))
+        assert readonly_cache.num_files == 2
+        assert readonly_cache.files == [
+            tmpdir + "/000000.parquet",
+            tmpdir + "/.cfg",
+        ]
+        assert readonly_cache.num_dataset_files == 1
+        assert readonly_cache.dataset_files == [
+            tmpdir + "/000000.parquet",
+        ]
+
+        cache.write_table(tbl)
+        assert readonly_cache.num_files == 3
+        assert readonly_cache.num_dataset_files == 2
+
+        for _, fragment in enumerate(readonly_cache.dataset.get_fragments()):
+            tbl = fragment.to_table()
+            assert tbl["some_int"].to_pylist() == [i for i in range(101)]
+            assert tbl["some_float"].to_pylist() == [i for i in range(101)]
+            assert tbl["some_str"].to_pylist() == [
+                f"str{i}" for i in range(101)
+            ]
+
+        assert readonly_cache.dataset.count_rows() == 202