From 77ce9f5963da3ae5db6bca02db4a32bcb3467223 Mon Sep 17 00:00:00 2001 From: Paddy Mullen Date: Wed, 1 Jul 2026 16:01:35 -0400 Subject: [PATCH] test(csv-import): add the dastardly CSV pathology corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CSV sibling of buckaroo's dastardly-dataframe dataset: one builder per pathological CSV (embedded newlines, BOM, non-UTF-8, decimal comma, mixed types past the infer window, ragged rows, NUL bytes, ambiguous dates, int overflow, tz-pinned naive timestamps, ...), each drawn from docs/research/csv-importers/edge-cases.md with the per-importer contrast in its docstring. test_dastardly_csv.py runs the corpus through tallyman_read_csv in three layers: - TestCorpusIsDeterministic — every case parses or raises a contract ValueError, never an unhandled exception or silent crash. - TestDocumentedBehavior — characterization guards pinning tallyman's current behavior pathology by pathology. - TestKnownGaps — xfail markers for the behavior the ADR/edge-cases.md want but tallyman does not yet implement (#142 ragged-short null-fill chief among them), ready-made red tests for the holistic edge-case pass. 47 passed, 3 xfailed against current main. Co-Authored-By: Claude Opus 4.8 --- tests/dastardly_csv_library.py | 394 +++++++++++++++++++++++++++++++++ tests/test_dastardly_csv.py | 290 ++++++++++++++++++++++++ 2 files changed, 684 insertions(+) create mode 100644 tests/dastardly_csv_library.py create mode 100644 tests/test_dastardly_csv.py diff --git a/tests/dastardly_csv_library.py b/tests/dastardly_csv_library.py new file mode 100644 index 0000000..e7fac2b --- /dev/null +++ b/tests/dastardly_csv_library.py @@ -0,0 +1,394 @@ +"""The dastardly CSV dataset. + +The weirdest CSVs that cause trouble frequently — the pathology corpus that +tallyman's ``tallyman_read_csv`` must handle deterministically (parse with a +known schema, or raise a clean ``ValueError`` — never silently corrupt, never +crash). + +This is the CSV sibling of buckaroo's ``ddd_library.py`` ("the dastardly +dataframe dataset"): one builder per pathology, each docstring naming the +hazard, how the three reference importers (DuckDB / Polars / Pandas) behave, and +the behavior tallyman wants. Cases and citations are drawn from +``docs/research/csv-importers/edge-cases.md`` (the seed corpus) — case numbers +below match that file. + +Every builder returns the raw CSV **bytes** (not str), because several +pathologies live below the text layer: BOM bytes, NUL bytes, non-UTF-8 +encodings. The accompanying tests in ``test_dastardly_csv.py`` write these +bytes to a project data dir and run them through ``tallyman_read_csv``. + +A builder returns only the dastardly *data*; any reader option needed to tame a +case (``separator``, ``encoding``, ``decimal_comma``, a pinned ``schema``) is +supplied by the test, never baked into the fixture — same split buckaroo keeps +between ``ddd_library`` (the frames) and the widget tests (the config). +""" + +from __future__ import annotations + +from collections.abc import Callable + + +# --------------------------------------------------------------------------- # +# 1. Embedded newline inside a quoted field +# --------------------------------------------------------------------------- # +def csv_embedded_newline() -> bytes: + """A real ``\\n`` inside a ``"..."``-quoted field (data cell and header cell). + + Why: the most common "row count is wrong" bug — an in-quote newline must be + data, not a record terminator. DuckDB/Polars/Pandas all honor it when the + quote char is set. tallyman (polars) should keep 2 data rows, not 3+. + """ + return b'note,n\n"line one\nline two",1\n"single",2\n' + + +def csv_embedded_newline_in_header() -> bytes: + """A quoted newline inside a *header* cell — the name spans two physical lines.""" + return b'"first\nsecond",val\nx,1\ny,2\n' + + +# --------------------------------------------------------------------------- # +# 2. Byte-order mark (BOM) +# --------------------------------------------------------------------------- # +def csv_bom_utf8() -> bytes: + """A UTF-8 BOM (``EF BB BF``) prefixing the first header cell. + + Why: an unstripped BOM corrupts the first column name (``id``), + silently breaking by-name schema binding (#141). DuckDB/Polars/Pandas all + strip a valid UTF-8 BOM; tallyman must too, so the header is ``id`` not + ``id``. + """ + return b"\xef\xbb\xbfid,name\n1,alice\n2,bob\n" + + +def csv_bom_on_quoted_header() -> bytes: + """A UTF-8 BOM in front of a *quoted* first header cell.""" + return b'\xef\xbb\xbf"id",name\n1,alice\n2,bob\n' + + +# --------------------------------------------------------------------------- # +# 3. Alternate / non-UTF-8 encoding +# --------------------------------------------------------------------------- # +def csv_latin1() -> bytes: + """A latin-1 (ISO-8859-1) file with ``café`` / ``£`` — invalid as UTF-8. + + Why: encoding is undecidable from content; a wrong guess is mojibake or a + mid-parse failure. Default strict-UTF-8 should fail loud; reading it needs an + explicit ``encoding`` knob. The ``\xe9``/``\xa3`` bytes are not valid UTF-8. + """ + return "name,price\ncafé,£5\nrésumé,£9\n".encode("latin-1") + + +# --------------------------------------------------------------------------- # +# 4. Decimal comma (European numerics) +# --------------------------------------------------------------------------- # +def csv_decimal_comma() -> bytes: + """Floats written ``1,5`` with ``;`` as the field separator (European style). + + Why: with ``,`` as the decimal point you must NOT also use it as the + delimiter. Read with ``separator=';'`` + ``decimal_comma=True``. Polars: + ``decimal_comma`` switches ``FLOAT_RE``; without it ``1,5`` stays String. + """ + return b"label;amount\na;1,5\nb;2,75\n" + + +# --------------------------------------------------------------------------- # +# 5. Thousands separator +# --------------------------------------------------------------------------- # +def csv_thousands_separator() -> bytes: + """Grouped integers ``1,234`` — the grouping char must be stripped only from + numeric columns, never from the string column. + + Why: Polars has **no** thousands option (the gap called out in the research), + so grouped numbers stay String unless pre-cleaned. Pandas guards against + stripping dots from non-numeric columns. + """ + return b'qty,label\n"1,234","a,b"\n"5,678","c,d"\n' + + +# --------------------------------------------------------------------------- # +# 6. Mixed-type column past the inference window +# --------------------------------------------------------------------------- # +def csv_mixed_type_late(n_clean: int = 12_000) -> bytes: + """Column ``v`` is integer for ``n_clean`` rows, then a string far past the + default 100-row (and 10k) infer window. + + Why: *the* canonical importer bug — the schema is decided from the window + and the violator is unseen until parse time. tallyman's escalation ladder + (ADR D6) must widen to whole-file and fall ``v`` back to ``string`` on a + no-schema read; an explicit ``int64`` must raise with a suggestion. + """ + rows = [b"k,v"] + rows += [f"{i},{i * 2}".encode() for i in range(n_clean)] + rows.append(b"oops,not_a_number") + rows += [f"{i},{i * 2}".encode() for i in range(n_clean + 1, n_clean + 10)] + return b"\n".join(rows) + b"\n" + + +# --------------------------------------------------------------------------- # +# 7. All-null column +# --------------------------------------------------------------------------- # +def csv_all_null_column() -> bytes: + """A column whose every value is empty — no evidence for a type. + + Why: whatever the importer guesses may conflict with a later widening. + Polars resolves an all-null column to ``null``/``String``; with an explicit + schema it is a non-issue. + """ + return b"a,b\n1,\n2,\n3,\n" + + +# --------------------------------------------------------------------------- # +# 8. Leading-zero identifiers (ZIP / FIPS / account numbers) +# --------------------------------------------------------------------------- # +def csv_leading_zeros() -> bytes: + """``01234`` ZIP-style identifiers that are NOT integers. + + Why: promoting to int destroys the leading zero irreversibly. DuckDB keeps + leading-zero values as VARCHAR by a deliberate sniffer rule; Polars (and so + tallyman by default) infers ``Int64`` and loses the zero. Per ADR D2 the + parse layer does **no** semantic cleaning — keeping zeros means pinning the + column to ``string`` in the schema. + """ + return b"zip,city\n01234,boston\n02115,boston\n" + + +# --------------------------------------------------------------------------- # +# 9. Scientific notation and signed numbers +# --------------------------------------------------------------------------- # +def csv_scientific_notation() -> bytes: + """An int-looking column that holds one ``1e5`` value. + + Why: scientific notation is float-shaped — an int-typed column that meets + ``1e5`` fails the cast (the mixed-type trap in disguise). Must surface as a + float (inference) or a value-error (explicit int schema), never silently + truncate. Also a scientific number must never be coerced to a date. + """ + return b"energy\n10\n20\n1e5\n30\n" + + +# --------------------------------------------------------------------------- # +# 10. Duplicate header names +# --------------------------------------------------------------------------- # +def csv_duplicate_headers() -> bytes: + """Two columns both named ``id``. + + Why: by-name schema binding (#141) is ambiguous when names collide; a silent + de-dup can bind the wrong column. Polars dedups to ``id``/``id_duplicated_0``; + tallyman binds by name so the post-dedup names must be deterministic and + surfaced. + """ + return b"id,id\n1,2\n3,4\n" + + +# --------------------------------------------------------------------------- # +# 11. Empty / blank header cell +# --------------------------------------------------------------------------- # +def csv_empty_header_cell() -> bytes: + """A header row with an empty middle cell (``a,,c``).""" + return b"a,,c\n1,2,3\n4,5,6\n" + + +# --------------------------------------------------------------------------- # +# 12. Headerless file (all-string, header ambiguous) +# --------------------------------------------------------------------------- # +def csv_headerless_all_string() -> bytes: + """No header; every field is a string, so row 0 can't be type-distinguished + from a header. + + Why: an importer that assumes a header eats a real data row as names. DuckDB + documents that an all-VARCHAR file is assumed to *have* a header; Polars + (tallyman) assumes the same unless told ``has_header=False`` + names. + """ + return b"alpha,beta\ngamma,delta\nepsilon,zeta\n" + + +# --------------------------------------------------------------------------- # +# 13. Trailing delimiter (ghost column) +# --------------------------------------------------------------------------- # +def csv_trailing_delimiter() -> bytes: + """Every line ends with the delimiter (``a,b,``) → a spurious empty final + column. + + Why: the ghost column shifts width and can be mistaken for an index column. + Polars emits an extra all-null/empty-named column; the decision (keep vs + drop) should be deterministic and surfaced. + """ + return b"a,b,\n1,2,\n3,4,\n" + + +# --------------------------------------------------------------------------- # +# 14. Ragged SHORT row (too few fields) — tallyman #142 +# --------------------------------------------------------------------------- # +def csv_ragged_short() -> bytes: + """A row with fewer fields than the header: ``a,b\\n1,2\\n3\\n4,5``. + + Why: **this is tallyman #142.** The old datafusion path raised; the polars + ``scan_csv`` path silently null-fills, baking ``(3, null)`` into the + canonical snapshot with no signal. Silent-by-direction is the trap — DuckDB + raises ``MISSING COLUMNS`` by default. Empirically polars cannot be made to + raise on a short row (pola-rs/polars#10585); the fail-loud detector is a + ``pyarrow.csv`` validation gate (edge-cases.md, 2026-06-26). + """ + return b"a,b\n1,2\n3\n4,5\n" + + +# --------------------------------------------------------------------------- # +# 15. Ragged LONG row (too many fields) — #142 sibling +# --------------------------------------------------------------------------- # +def csv_ragged_long() -> bytes: + """A row with more fields than the header: ``a,b\\n1,2\\n3,4,5\\n6,7``. + + Why: the asymmetric twin of the short row — most tools are *loud* here and + *silent* on short rows. Polars raises ``found more fields than defined`` + unless ``truncate_ragged_lines=True``; tallyman surfaces that as a + ``ValueError``. The asymmetry between short and long is what to eliminate. + """ + return b"a,b\n1,2\n3,4,5\n6,7\n" + + +# --------------------------------------------------------------------------- # +# 16. Quoted-empty vs unquoted-empty as null +# --------------------------------------------------------------------------- # +def csv_quoted_vs_bare_empty() -> bytes: + """``""`` (quoted empty) vs a bare empty field — should one be null and the + other empty string? + + Why: the null-vs-empty distinction changes downstream joins/aggregations and + the tools disagree. DuckDB ``allow_quoted_nulls`` / ``force_not_null``; + Polars ``missing_utf8_is_empty_string``; Pandas treats bare empty as NA. + """ + return b'a,b\n"",x\n,y\nz,w\n' + + +# --------------------------------------------------------------------------- # +# 17. Whitespace sensitivity +# --------------------------------------------------------------------------- # +def csv_leading_whitespace() -> bytes: + """Leading spaces after the delimiter (`` 1, 2``) that can flip inferred type. + + Why: Polars does **not** strip surrounding whitespace before inference, so + `` 1`` may not match ``INTEGER_RE`` and the column stays String. Pandas has + opt-in ``skipinitialspace``. The fixture pairs a clean numeric column with a + space-prefixed one so the type difference is visible. + """ + return b"clean,spaced\n1, 1\n2, 2\n3, 3\n" + + +# --------------------------------------------------------------------------- # +# 18. Windows vs Unix newlines (and mixed) +# --------------------------------------------------------------------------- # +def csv_mixed_newlines() -> bytes: + """One file mixing ``\\n`` and ``\\r\\n`` line terminators. + + Why: a trailing ``\\r`` can ride along on the last field's value; mixed + newlines can confuse record boundaries. Default behavior should normalize + CRLF/CR and strip the trailing ``\\r`` so values don't carry it. + """ + return b"a,b\r\n1,2\n3,4\r\n5,6\n" + + +# --------------------------------------------------------------------------- # +# 19. NUL bytes and other control bytes mid-field +# --------------------------------------------------------------------------- # +def csv_nul_byte() -> bytes: + """A ``\\x00`` byte embedded in a data field. + + Why: NUL bytes corrupt C string handling and signal a binary file mis-fed as + CSV — a classic fuzzer crash. Pandas raises ``"NULL byte detected"``; DuckDB + has dedicated null-byte tests. edge-cases.md suggests tallyman treat a NUL as + a hard structural error rather than silently absorbing it into a string. + """ + return b"a,b\n1,2\n3,4\x005\n" + + +# --------------------------------------------------------------------------- # +# 20. Huge / wide files and over-long lines +# --------------------------------------------------------------------------- # +def csv_over_long_line(width: int = 2_000_000) -> bytes: + """A single field roughly ``width`` bytes long. + + Why: an over-long line can blow read buffers. DuckDB bounds it with + ``max_line_size`` (~2 MB) and raises ``LINE SIZE OVER MAXIMUM``. The fixture + is a deterministic 'x'*width cell so the boundary behavior is exercised. + """ + big = b"x" * width + return b"a,b\n1,2\n" + big + b",3\n" + + +# --------------------------------------------------------------------------- # +# 21. Datetime format zoo +# --------------------------------------------------------------------------- # +def csv_ambiguous_date() -> bytes: + """Ambiguous ``1/6/2000`` (Jan-6 vs Jun-1) and an impossible ``32/32/2019``. + + Why: date parsing is the richest source of silent month/day swaps. tallyman + keeps dates opt-in/explicit (no auto-parse), so without a ``date`` schema + these stay ``string`` — never silently swapped, never a wrong date. + """ + return b"d\n1/6/2000\n2/7/2001\n32/32/2019\n" + + +# --------------------------------------------------------------------------- # +# 22. Integer overflow / large-int fidelity +# --------------------------------------------------------------------------- # +def csv_integer_overflow() -> bytes: + """A value past int64 range alongside ordinary ints. + + Why: silent wraparound or precision loss on identifiers/large counts. Polars + falls back i64 → Int128 on overflow; with an explicit narrow type an + out-of-range value should be a value-error, not a silent wrap. + """ + return b"big\n1\n2\n99999999999999999999999\n" + + +# --------------------------------------------------------------------------- # +# 23. Naive timestamp string with a tz-aware schema pin (#145) +# --------------------------------------------------------------------------- # +def csv_naive_ts_pinned_utc() -> bytes: + """Naive timestamp strings (no tz offset) with a schema that pins ``timestamp('UTC')``. + + Why: the timestamp scale → polars time_unit mapping (#145) resolved the + rounding direction but left open a factual question: when polars is given a + naive string and a tz-aware ``Datetime(unit, time_zone='UTC')`` override, + does it (a) *attach* UTC (reinterpret the value as UTC, no shift), (b) + *convert* from local time to UTC (shift the clock), or (c) raise? + + Verdict (polars 1.x, verified 2026-06-26): **attach** — ``2024-01-15 + 10:30:00`` becomes ``2024-01-15T10:30:00+00:00`` with no shift. tallyman + inherits this semantics: pinning ``timestamp('UTC')`` on a naive-string + column is a declaration that the values *are* UTC, not a conversion request. + """ + return b"ts,val\n2024-01-15 10:30:00,1\n2024-06-20 08:00:00.123456,2\n" + + +# --------------------------------------------------------------------------- # +# Registry — every dastardly case, for the parametrized corpus sweep. +# (Builders are also importable individually, buckaroo-ddd style.) +# --------------------------------------------------------------------------- # +ALL_CASES: list[Callable[[], bytes]] = [ + csv_embedded_newline, + csv_embedded_newline_in_header, + csv_bom_utf8, + csv_bom_on_quoted_header, + csv_latin1, + csv_decimal_comma, + csv_thousands_separator, + csv_mixed_type_late, + csv_all_null_column, + csv_leading_zeros, + csv_scientific_notation, + csv_duplicate_headers, + csv_empty_header_cell, + csv_headerless_all_string, + csv_trailing_delimiter, + csv_ragged_short, + csv_ragged_long, + csv_quoted_vs_bare_empty, + csv_leading_whitespace, + csv_mixed_newlines, + csv_nul_byte, + csv_over_long_line, + csv_ambiguous_date, + csv_integer_overflow, + csv_naive_ts_pinned_utc, +] diff --git a/tests/test_dastardly_csv.py b/tests/test_dastardly_csv.py new file mode 100644 index 0000000..e0f4bbd --- /dev/null +++ b/tests/test_dastardly_csv.py @@ -0,0 +1,290 @@ +"""Run the dastardly CSV corpus through ``tallyman_read_csv``. + +The CSV sibling of buckaroo's ``test_widget_weird_types.py``: take each +pathological fixture from :mod:`dastardly_csv_library` and assert how +``tallyman_read_csv`` handles it. Three layers: + +- :class:`TestCorpusIsDeterministic` — the corpus-level guard. *Every* case must + reach a deterministic outcome: a parsed schema, or a ``ValueError`` carrying + the ``tallyman_read_csv:`` contract prefix. Never an unhandled exception, a + polars panic, or a silent crash. +- :class:`TestDocumentedBehavior` — characterization / regression guards pinning + tallyman's *current* behavior on each pathology, with the per-tool contrast in + each docstring (drawn from ``docs/research/csv-importers/``). +- :class:`TestKnownGaps` — ``xfail`` tests that assert the behavior the ADR / + edge-cases.md *wants* but tallyman does not yet implement. These are the + ready-made red tests for the holistic edge-case pass (#142 chief among them). + +All citations: ``docs/research/csv-importers/edge-cases.md`` (case numbers) and +``plans/adr-intelligent-csv-import.md`` (decisions D2/D6). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from tallyman_core import data_dir +from tallyman_xorq.io import tallyman_read_csv +from tests import dastardly_csv_library as ddd + + +@dataclass +class ReadOutcome: + """The deterministic result of feeding one dastardly CSV to the reader.""" + + ok: bool + schema: dict[str, str] | None # name -> ibis type str (sans original_row_order) + nrows: int | None + error: str | None + + +def read_dastardly(project: str, builder, *, count: bool = False, **reader_kwargs) -> ReadOutcome: + """Write ``builder()``'s bytes under the project data dir and read them. + + Returns a :class:`ReadOutcome` for either path — a successful parse (schema, + optional row count) or a raised ``ValueError`` (its message). Any *other* + exception type propagates: the corpus contract is parse-or-clean-ValueError, + so a different exception is a real failure, not an outcome. + """ + p: Path = data_dir(project) / f"{builder.__name__}.csv" + p.write_bytes(builder()) + try: + expr = tallyman_read_csv(str(p), **reader_kwargs) + except ValueError as exc: + return ReadOutcome(ok=False, schema=None, nrows=None, error=str(exc)) + schema = {k: str(v) for k, v in expr.schema().items() if k != "original_row_order"} + nrows = int(expr.count().execute()) if count else None + return ReadOutcome(ok=True, schema=schema, nrows=nrows, error=None) + + +# --------------------------------------------------------------------------- # +# Corpus-level guard: every case is handled deterministically. +# --------------------------------------------------------------------------- # +class TestCorpusIsDeterministic: + @pytest.mark.parametrize("builder", ddd.ALL_CASES, ids=lambda f: f.__name__) + def test_parse_or_clean_valueerror(self, project, builder): + """Each pathology yields a schema, or a ValueError with the contract + prefix — never an unhandled exception or silent crash.""" + out = read_dastardly(project, builder) + if out.ok: + assert out.schema, f"{builder.__name__}: parsed but empty schema" + else: + assert "tallyman_read_csv" in out.error or "csv" in out.error.lower(), ( + f"{builder.__name__}: error lacks the contract prefix: {out.error!r}" + ) + + +# --------------------------------------------------------------------------- # +# Characterization: tallyman's current behavior, pathology by pathology. +# --------------------------------------------------------------------------- # +class TestDocumentedBehavior: + # --- 1. embedded newline in a quoted field ------------------------------- + def test_embedded_newline_is_data_not_a_row(self, project): + """Case 1: a quoted ``\\n`` stays inside the field — 2 rows, not 3.""" + out = read_dastardly(project, ddd.csv_embedded_newline, count=True) + assert out.nrows == 2 + assert out.schema == {"note": "string", "n": "int64"} + + def test_embedded_newline_in_header_kept_in_name(self, project): + """Case 1: a quoted newline in a header cell stays in the column name.""" + out = read_dastardly(project, ddd.csv_embedded_newline_in_header, count=True) + assert out.nrows == 2 + assert "first\nsecond" in out.schema + + # --- 2. BOM -------------------------------------------------------------- + def test_utf8_bom_stripped_from_header(self, project): + """Case 2: a UTF-8 BOM is stripped so by-name binding sees ``id``, not ``id``.""" + out = read_dastardly(project, ddd.csv_bom_utf8) + assert "id" in out.schema and "id" not in out.schema + + def test_utf8_bom_stripped_before_quoted_header(self, project): + """Case 2: BOM stripped even ahead of a quoted first header cell.""" + out = read_dastardly(project, ddd.csv_bom_on_quoted_header) + assert "id" in out.schema + + # --- 3. encoding --------------------------------------------------------- + def test_latin1_under_default_utf8_fails_loud(self, project): + """Case 3: invalid UTF-8 bytes raise at parse, never mojibake silently.""" + out = read_dastardly(project, ddd.csv_latin1) + assert not out.ok and "utf-8" in out.error.lower() + + def test_latin1_encoding_kwarg_is_restricted_to_utf8_variants(self, project): + """Case 3: polars scan only supports utf8/utf8-lossy — ``encoding='latin1'`` + is rejected with a clear message (use utf8-lossy or pre-convert).""" + out = read_dastardly(project, ddd.csv_latin1, encoding="latin1") + assert not out.ok and "utf8-lossy" in out.error + + # --- 4. decimal comma ---------------------------------------------------- + def test_decimal_comma_needs_separator_and_flag(self, project): + """Case 4: with the default ``,`` delimiter, ``1,5`` splits into extra + fields and raises; ``separator=';'`` + ``decimal_comma=True`` parses it + as a float.""" + default = read_dastardly(project, ddd.csv_decimal_comma) + assert not default.ok and "more fields" in default.error + tamed = read_dastardly(project, ddd.csv_decimal_comma, separator=";", decimal_comma=True) + assert tamed.ok and tamed.schema["amount"] == "float64" + + # --- 5. thousands separator ---------------------------------------------- + def test_thousands_grouped_numbers_stay_string(self, project): + """Case 5: polars has no thousands option, so ``"1,234"`` stays string + (the research-flagged gap — needs a custom pass to become numeric).""" + out = read_dastardly(project, ddd.csv_thousands_separator) + assert out.schema == {"qty": "string", "label": "string"} + + # --- 6. mixed type past the window --------------------------------------- + def test_mixed_type_late_escalates_to_string(self, project): + """Case 6 / #143: a violator past the 10k window makes no-schema infer + escalate to whole-file and fall the column back to ``string``.""" + out = read_dastardly(project, ddd.csv_mixed_type_late, count=True) + assert out.nrows == 12010 + assert out.schema["v"] == "string" + + # --- 7. all-null column -------------------------------------------------- + def test_all_null_column_infers_string(self, project): + """Case 7: a column with no type evidence resolves to ``string``.""" + out = read_dastardly(project, ddd.csv_all_null_column) + assert out.schema["b"] == "string" + + # --- 8. leading zeros (deliberate, per ADR D2) --------------------------- + def test_leading_zeros_become_int_by_design(self, project): + """Case 8: the parse layer does NO semantic cleaning (ADR D2) — ``01234`` + infers ``int64`` and loses the zero. Keeping it means pinning ``string`` + in the schema; this test pins the deliberate default.""" + out = read_dastardly(project, ddd.csv_leading_zeros) + assert out.schema["zip"] == "int64" + + def test_leading_zeros_preserved_when_pinned_string(self, project): + """Case 8: pinning the column to ``string`` keeps the leading zero.""" + out = read_dastardly(project, ddd.csv_leading_zeros, schema={"zip": "string", "&rest": "infer"}) + assert out.ok and out.schema["zip"] == "string" + + # --- 9. scientific notation ---------------------------------------------- + def test_scientific_notation_widens_to_float(self, project): + """Case 9: an int-looking column holding ``1e5`` infers ``float64``, + never a silently-truncated int and never a date.""" + out = read_dastardly(project, ddd.csv_scientific_notation) + assert out.schema["energy"] == "float64" + + # --- 10. duplicate headers ----------------------------------------------- + def test_duplicate_headers_deduped_deterministically(self, project): + """Case 10: polars dedups the second ``id`` to ``id_duplicated_0`` — a + stable, documented rename (tallyman binds by name, so this matters).""" + out = read_dastardly(project, ddd.csv_duplicate_headers) + assert set(out.schema) == {"id", "id_duplicated_0"} + + # --- 12. headerless all-string ------------------------------------------- + def test_all_string_file_assumes_header(self, project): + """Case 12: when every column is string the header can't be + type-distinguished, so row 0 is assumed to be the header (2 data rows).""" + out = read_dastardly(project, ddd.csv_headerless_all_string, count=True) + assert out.nrows == 2 + assert set(out.schema) == {"alpha", "beta"} + + # --- 13. trailing delimiter ---------------------------------------------- + def test_trailing_delimiter_makes_ghost_column(self, project): + """Case 13: a uniform trailing comma yields an extra empty-named column.""" + out = read_dastardly(project, ddd.csv_trailing_delimiter) + assert "" in out.schema + + # --- 15. ragged long (loud) ---------------------------------------------- + def test_ragged_long_raises(self, project): + """Case 15: a too-many-fields row raises ``found more fields than + defined`` (the loud half of the short/long asymmetry).""" + out = read_dastardly(project, ddd.csv_ragged_long) + assert not out.ok and "more fields" in out.error + + # --- 17. whitespace flips inferred type ---------------------------------- + def test_leading_whitespace_changes_inferred_type(self, project): + """Case 17: polars does not strip surrounding whitespace before + inference, so `` 1`` stays ``string`` while a clean ``1`` is ``int64``.""" + out = read_dastardly(project, ddd.csv_leading_whitespace) + assert out.schema["clean"] == "int64" + assert out.schema["spaced"] == "string" + + # --- 18. mixed newlines --------------------------------------------------- + def test_mixed_newlines_parsed_cleanly(self, project): + """Case 18: a file mixing ``\\n`` and ``\\r\\n`` parses to clean ints — + no trailing ``\\r`` riding along on the last field.""" + out = read_dastardly(project, ddd.csv_mixed_newlines, count=True) + assert out.nrows == 3 + assert out.schema == {"a": "int64", "b": "int64"} + + # --- 20. over-long line --------------------------------------------------- + def test_over_long_line_parses(self, project): + """Case 20: polars has no ~2 MB line cap, so a multi-megabyte field + parses rather than raising ``LINE SIZE OVER MAXIMUM`` (DuckDB's bound). + Pins the absence of a line-size guard so a future cap is a deliberate + change, not a surprise.""" + out = read_dastardly(project, ddd.csv_over_long_line) + assert out.ok and set(out.schema) == {"a", "b"} + + # --- 21. ambiguous / impossible dates ------------------------------------ + def test_ambiguous_date_stays_string(self, project): + """Case 21: dates are opt-in — ``1/6/2000`` and ``32/32/2019`` stay + ``string`` rather than being silently swapped or turned into a wrong + date.""" + out = read_dastardly(project, ddd.csv_ambiguous_date) + assert out.schema["d"] == "string" + + # --- 23. naive timestamp + tz-aware schema pin (#145) -------------------- + def test_naive_ts_pinned_utc_attaches_not_converts(self, project): + """Case 23 / #145: polars *attaches* UTC to a naive timestamp string + when schema_overrides pins Datetime(us, 'UTC') — it reinterprets the + value as UTC in place (no shift, no error). Pinning ``timestamp('UTC')`` + on a naive-string column is a declaration, not a conversion request.""" + out = read_dastardly( + project, + ddd.csv_naive_ts_pinned_utc, + count=True, + schema={"ts": "timestamp('UTC')", "&rest": "infer"}, + ) + assert out.ok, f"unexpected error: {out.error}" + # The ibis type round-trips with the precision polars inferred from the + # sub-second digits in the data (6 digits → scale 6 = us). + assert out.schema["ts"] == "timestamp('UTC', 6)", out.schema + assert out.nrows == 2 + + +# --------------------------------------------------------------------------- # +# Known gaps: the behavior the ADR / edge-cases.md wants but tallyman lacks. +# These xfail today and are the red tests for the holistic edge-case pass. +# --------------------------------------------------------------------------- # +class TestKnownGaps: + @pytest.mark.xfail( + strict=True, + reason="#142: polars scan_csv silently null-fills short rows; tallyman " + "should fail loud (or reject-log). Fix lands via a pyarrow.csv validation " + "gate — see edge-cases.md case 14.", + ) + def test_ragged_short_should_not_silently_nullfill(self, project): + """Case 14 / #142: ``a,b\\n1,2\\n3\\n4,5`` must NOT silently bake + ``(3, null)``. Desired: a clean ValueError (or a routed reject row).""" + out = read_dastardly(project, ddd.csv_ragged_short, count=True) + assert not out.ok, f"short row silently parsed to {out.nrows} rows: {out.schema}" + + @pytest.mark.xfail( + strict=False, + reason="edge-cases.md case 19 (suggestion, not yet decided): a NUL byte " + "should be a hard structural error; today it is absorbed into a string.", + ) + def test_nul_byte_should_be_structural_error(self, project): + """Case 19: a ``\\x00`` mid-field should raise, not silently ride along + inside a string value.""" + out = read_dastardly(project, ddd.csv_nul_byte) + assert not out.ok + + @pytest.mark.xfail( + strict=False, + reason="ADR D6 says infer mode always resolves (mixed → string). An " + "int64-overflow value in a pure-integer column is not a type *mix*, so " + "the merge-to-string fallback never fires and the cast raises instead of " + "resolving. edge-cases.md case 22.", + ) + def test_infer_mode_resolves_int_overflow(self, project): + """Case 22: a no-schema read of a column with a past-int64 value should + resolve (to string, or Int128 where the build supports it), not raise.""" + out = read_dastardly(project, ddd.csv_integer_overflow) + assert out.ok