Skip to content

Commit 28dc4cd

Browse files
authored
Merge pull request #13 from qorud02/fix/explicit-records-key-20261006
Handle explicit empty JSON record selectors
2 parents ab4c69d + 84c8e1a commit 28dc4cd

4 files changed

Lines changed: 127 additions & 3 deletions

File tree

‎README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,9 @@ Contract:
6666

6767
Columns are required by default; set `"required": false` for an optional field. Supported types are `string`, `integer`, `decimal`, and `date` (`YYYY-MM-DD`). Numeric bounds are inclusive. The composite key is checked only when all its components have type-valid, nonmissing values. Unknown contract keys fail early so a misspelled rule cannot silently weaken a check.
6868

69-
JSON arrays work directly. For an API-style envelope such as `{"items": [...]}`, add `--records-key items`. Numeric JSON identifiers fail a string contract instead of silently losing leading zeroes. UTF-8 BOM files are supported; duplicate headers, duplicate JSON keys, ragged CSV/TSV rows, and nonstandard JSON numbers are rejected. JSON numbers outside the supported Decimal range and nesting beyond the Python parser limit return input-error exit `2`, including in contract files. Large finite values such as `1e400` retain exact decimal precision.
69+
JSON arrays work directly when `--records-key` is omitted. For an API-style envelope such as `{"items": [...]}`, add `--records-key items`. Use `--records-key=""` to select a top-level empty-string key, such as `{"": [...]}`. Any explicit selector requires a JSON object containing that key with an array value; root arrays, missing keys, non-array values, CSV and TSV return input-error exit `2`. A selected empty array returns data-violation exit `1` with an `empty_data` issue.
70+
71+
Numeric JSON identifiers fail a string contract instead of silently losing leading zeroes. UTF-8 BOM files are supported; duplicate headers, duplicate JSON keys, ragged CSV/TSV rows, and nonstandard JSON numbers are rejected. JSON numbers outside the supported Decimal range and nesting beyond the Python parser limit return input-error exit `2`, including in contract files. Large finite values such as `1e400` retain exact decimal precision.
7072

7173
## Tab-delimited extracts
7274

‎scripts/package_smoke.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,14 @@ def csv_to_tsv(name, encoding="utf-8"):
8282
"large-finite.json": b'[{"amount":1e400},{"amount":-1e-400}]',
8383
"decimal-contract.json": b'{"columns":{"amount":{"type":"decimal"}}}',
8484
"surrogate-field.json": b'[{"amount":1,"\\ud800":1}]',
85+
"selector-contract.json": b'{"columns":{"id":{"type":"string"}}}',
86+
"selector-array.json": b'[{"id":"001"}]',
87+
"selector-envelope.json": b'{"": [{"id":"001"}], "items": [{"id":"002"}]}',
88+
"selector-empty.json": b'{"": [], "items": [{"id":"001"}]}',
89+
"selector-missing.json": b'{"items": [{"id":"001"}]}',
90+
"selector-nonarray.json": b'{"": {}}',
91+
"selector.csv": b'id\n001\n',
92+
"selector.tsv": b'id\n001\n',
8593
}
8694
payloads["newline-unique-contract.json"] = json.dumps({
8795
"columns": {"id": {"type": "string"}}, "unique_by": ["id"]
@@ -134,6 +142,34 @@ def run(arguments, expected, label, report=True):
134142
assert valid_tsv["valid"] and valid_tsv["records_checked"] == 3 and valid_tsv["error_count"] == 0
135143
invalid_tsv = run([extra("invalid.tsv"), "--contract", contract], 1, "tsv-data-violations")
136144
assert not invalid_tsv["valid"] and invalid_tsv["records_checked"] == 3 and invalid_tsv["error_count"] == 5
145+
for source, options, expected, label in (
146+
("selector-array.json", [], 0, "json-absent-selector"),
147+
("selector-envelope.json", ["--records-key", "items"], 0, "json-named-selector"),
148+
("selector-envelope.json", ["--records-key", ""], 0, "json-empty-string-selector"),
149+
("selector-empty.json", ["--records-key=",], 1, "json-empty-selected-array"),
150+
):
151+
originals = {name: (temp / name).read_bytes() for name in (source, "selector-contract.json")}
152+
report = run([extra(source), "--contract", extra("selector-contract.json")] + options,
153+
expected, label)
154+
assert report["valid"] == (expected == 0) and report["records_checked"] == (1 if expected == 0 else 0), report
155+
assert report["error_count"] == expected, report
156+
if expected == 1:
157+
assert report["issues"][0]["code"] == "empty_data", report
158+
assert {name: (temp / name).read_bytes() for name in originals} == originals
159+
for source, options in (
160+
("selector-envelope.json", []),
161+
("selector-array.json", ["--records-key", ""]),
162+
("selector-missing.json", ["--records-key", ""]),
163+
("selector-nonarray.json", ["--records-key", ""]),
164+
("selector.csv", ["--records-key", ""]),
165+
("selector.tsv", ["--records-key", ""]),
166+
("selector.csv", ["--records-key", "items"]),
167+
("selector.tsv", ["--records-key", "items"]),
168+
):
169+
originals = {name: (temp / name).read_bytes() for name in (source, "selector-contract.json")}
170+
run([extra(source), "--contract", extra("selector-contract.json")] + options,
171+
2, source + "-selector-rejected-" + repr(options), False)
172+
assert {name: (temp / name).read_bytes() for name in originals} == originals
137173
for suffix in ("csv", "tsv"):
138174
source = temp / ("newline-unique." + suffix)
139175
before = source.read_bytes()

‎src/public_data_sentinel/cli.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def read_records(path, records_key=None):
2020
text = stream.read()
2121
if suffix in (".csv", ".tsv"):
2222
format_name = "TSV" if suffix == ".tsv" else "CSV"
23-
if records_key:
23+
if records_key is not None:
2424
raise ValueError("--records-key applies only to JSON")
2525
reader = csv.reader(io.StringIO(text, newline=""), delimiter="\t" if suffix == ".tsv" else ",", strict=True)
2626
headers = next(reader, None)
@@ -39,7 +39,7 @@ def read_records(path, records_key=None):
3939
if suffix != ".json":
4040
raise ValueError("Input filename must end in .csv, .tsv or .json")
4141
records = load_json(text)
42-
if records_key:
42+
if records_key is not None:
4343
if not isinstance(records, dict) or records_key not in records:
4444
raise ValueError(f"JSON object does not contain records key {records_key!r}")
4545
records = records[records_key]

‎tests/test_records_key.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
"""Explicit JSON selectors distinguish an empty key from an absent option."""
2+
import json
3+
import os
4+
from pathlib import Path
5+
import subprocess
6+
import sys
7+
import tempfile
8+
import unittest
9+
10+
11+
ROOT = Path(__file__).resolve().parents[1]
12+
13+
14+
class ExplicitRecordsKeyTests(unittest.TestCase):
15+
def setUp(self):
16+
temporary = tempfile.TemporaryDirectory()
17+
self.addCleanup(temporary.cleanup)
18+
self.root = Path(temporary.name)
19+
self.contract = self.root / "contract.json"
20+
self.contract.write_text('{"columns":{"id":{"type":"string"}}}', encoding="utf-8")
21+
22+
def run_cli(self, name, content, options=()):
23+
source = self.root / name
24+
source.write_bytes(content.encode("utf-8"))
25+
environment = dict(os.environ, PYTHONPATH=str(ROOT / "src"),
26+
PYTHONDONTWRITEBYTECODE="1", PYTHONIOENCODING="utf-8")
27+
return subprocess.run(
28+
[sys.executable, "-m", "public_data_sentinel.cli", str(source),
29+
"--contract", str(self.contract), *options],
30+
cwd=self.root, env=environment, capture_output=True, text=True,
31+
encoding="utf-8", timeout=30,
32+
)
33+
34+
def test_explicit_empty_key_selects_valid_and_empty_record_arrays(self):
35+
for records, expected in (([{"id": "001"}], 0), ([], 1)):
36+
with self.subTest(records=records):
37+
result = self.run_cli(
38+
"envelope.json", json.dumps({"": records, "other": [{"id": 123}]}),
39+
("--records-key", ""),
40+
)
41+
self.assertEqual(result.returncode, expected, result.stderr)
42+
self.assertEqual(result.stderr, "")
43+
report = json.loads(result.stdout)
44+
self.assertEqual(report["records_checked"], len(records))
45+
self.assertEqual(report["valid"], expected == 0)
46+
if not records:
47+
self.assertEqual(report["issues"][0]["code"], "empty_data")
48+
49+
def test_explicit_empty_key_rejects_missing_or_nonarray_selection(self):
50+
for content in ('[{"id":"001"}]', '{"items":[{"id":"001"}]}', '{"":{}}'):
51+
with self.subTest(content=content):
52+
result = self.run_cli("invalid-selection.json", content, ("--records-key", ""))
53+
self.assertEqual(result.returncode, 2, result.stderr)
54+
self.assertEqual(result.stdout, "")
55+
self.assertTrue(result.stderr.startswith("data-sentinel: "))
56+
self.assertNotIn("Traceback", result.stderr)
57+
58+
def test_delimited_inputs_reject_empty_and_named_selectors(self):
59+
for suffix in ("csv", "tsv"):
60+
for key in ("", "items"):
61+
with self.subTest(suffix=suffix, key=key):
62+
result = self.run_cli("data." + suffix, "id\n001\n", ("--records-key", key))
63+
self.assertEqual(result.returncode, 2, result.stderr)
64+
self.assertEqual(result.stdout, "")
65+
self.assertIn("--records-key applies only to JSON", result.stderr)
66+
67+
def test_absent_and_named_selectors_keep_existing_behavior(self):
68+
cases = (
69+
("array.json", '[{"id":"001"}]', ()),
70+
("envelope.json", '{"items":[{"id":"001"}]}', ("--records-key", "items")),
71+
("data.csv", "id\n001\n", ()),
72+
("data.tsv", "id\n001\n", ()),
73+
)
74+
for name, content, options in cases:
75+
with self.subTest(name=name, options=options):
76+
result = self.run_cli(name, content, options)
77+
self.assertEqual(result.returncode, 0, result.stderr)
78+
self.assertEqual(result.stderr, "")
79+
report = json.loads(result.stdout)
80+
self.assertTrue(report["valid"])
81+
self.assertEqual(report["records_checked"], 1)
82+
self.assertEqual(report["error_count"], 0)
83+
84+
85+
if __name__ == "__main__":
86+
unittest.main()

0 commit comments

Comments
 (0)