Skip to content

Commit e4c603f

Browse files
paddymulclaude
andcommitted
test: failing regression tests for tallyman_read_csv reconstruction, location, reader kwargs
Cover three regressions from the polars CSV-reader swap (685ad1d), each reproduced as a failing test before the fix: - #6: after the snapshot is baked, deleting the source CSV must not break reconstruction (baked_snapshot_path / verify_result_faithful). Today _ordered_csv_parquet stats the CSV on every recipe re-run -> FileNotFoundError. - #9: the ordered-CSV intermediate must live in a project-independent location, not under the active project's artifacts dir (reconstruction under a different active project misses the cache and re-reads the CSV). - #10: reader options (separator, skip_rows, ...) must be forwarded to polars.scan_csv; the rewrite dropped **kwargs so they raise TypeError. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 685ad1d commit e4c603f

1 file changed

Lines changed: 69 additions & 0 deletions

File tree

tests/test_tallyman_read_csv.py

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,3 +241,72 @@ def test_tallyman_read_csv_digest_stable_under_repartition(project, repartitione
241241
assert snapshot_file_digest(snap) == recorded, (
242242
"snapshot digest drifted across materialisations of a repartitioned read"
243243
)
244+
245+
246+
def test_tallyman_read_csv_reconstructs_after_source_deleted(project, sample_csv, monkeypatch):
247+
"""#6: once the ordered parquet is baked, reconstruction must not touch the CSV.
248+
249+
The CSV is read exactly once — at ingest. After the snapshot is baked, deleting
250+
(or moving) the source CSV must not break path reconstruction: baked_snapshot_path
251+
and verify_result_faithful re-run the recipe (-> tallyman_read_csv), and that must
252+
resolve the already-written intermediate instead of stat-ing the now-absent CSV.
253+
"""
254+
from tallyman_xorq.result_cache import baked_snapshot_path, verify_result_faithful
255+
256+
monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off")
257+
monkeypatch.setenv("TALLYMAN_PROJECT", project)
258+
res = catalog_create("csv_entry", _read_csv_code(sample_csv))
259+
assert "error" not in res, res
260+
h = _hash_of(project)
261+
assert baked_snapshot_path(project, h) is not None
262+
263+
# Delete the source CSV; the baked snapshot and the ordered intermediate remain.
264+
sample_csv.unlink()
265+
266+
# Reconstruction must still resolve the snapshot (pre-fix: FileNotFoundError on src.stat()).
267+
snap = baked_snapshot_path(project, h)
268+
assert snap is not None and snap.exists(), "snapshot must resolve without the source CSV"
269+
assert verify_result_faithful(project, h) is True, "faithful check must work without the CSV"
270+
271+
272+
def test_ordered_csv_intermediate_lives_outside_project_dir(project, sample_csv, monkeypatch):
273+
"""#9: the ordered-CSV intermediate is keyed on the CSV path, not the active project.
274+
275+
Placing it under a project's artifacts dir (via resolve_project(None)) means a
276+
reconstruction under a *different* active project looks in the wrong dir and
277+
re-reads the CSV. It must live in a project-independent location.
278+
"""
279+
from tallyman_core import artifacts_dir
280+
from tallyman_core.paths import tallyman_home
281+
282+
monkeypatch.setenv("TALLYMAN_PROJECT", project)
283+
res = catalog_create("csv_entry", _read_csv_code(sample_csv))
284+
assert "error" not in res, res
285+
286+
shared = list((tallyman_home() / "csv_ordered").glob("*.parquet"))
287+
assert shared, "ordered-CSV intermediate must live under TALLYMAN_HOME/csv_ordered"
288+
proj_csv_ordered = artifacts_dir(project) / "csv_ordered"
289+
assert not proj_csv_ordered.exists(), "intermediate must NOT be under the project artifacts dir"
290+
291+
292+
def test_tallyman_read_csv_forwards_reader_kwargs(project, monkeypatch):
293+
"""#10: reader options (separator, skip_rows, ...) are forwarded to polars scan_csv.
294+
295+
The rewrite dropped **kwargs; a documented call like tallyman_read_csv(path, schema,
296+
separator=';') must parse the alternate delimiter instead of raising TypeError.
297+
"""
298+
monkeypatch.setenv("TALLYMAN_PROJECT", project)
299+
p = data_dir(project) / "semi.csv"
300+
p.write_text("id;name\n3;charlie\n1;alice\n2;bob\n")
301+
code = f"""
302+
import xorq.vendor.ibis as ibis
303+
from tallyman_xorq.io import tallyman_read_csv
304+
schema = ibis.schema({{"id": "int64", "name": "string"}})
305+
expr = tallyman_read_csv({str(p)!r}, schema=schema, separator=";")
306+
"""
307+
res = catalog_create("semi", code)
308+
assert "error" not in res, res
309+
fields = {f["name"] for f in res["schema"]["fields"]}
310+
assert {"id", "name", "original_row_order"} <= fields, (
311+
f"separator=';' not forwarded — columns did not split: {fields}"
312+
)

0 commit comments

Comments
 (0)