From 8711d9d2ee6355b2931da135208d86cae5878b0f Mon Sep 17 00:00:00 2001 From: spacedevin Date: Mon, 3 Aug 2026 17:56:06 -0700 Subject: [PATCH] test: run the conformance corpus from Tish, the third target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The corpus exists to make "one source, three targets" a fact rather than a claim, and it was only ever run by two of them: the JS build and the Rust crate. The Tish source itself — what both are generated FROM — was never checked against it. A change that altered the Tish semantics but survived both emits would have gone unnoticed by the very thing meant to catch it. test/conformance.tish imports ../src/index.tish and runs the same inputs against the same expected files. All 11 cases pass, so the three targets are now demonstrably in agreement rather than assumed to be. Two things it has to do differently, both noted where they happen: Run under `tish run` instead of the JS build, because reading the corpus needs `tish:fs` and the JS target has no filesystem at all. Compare semantically rather than textually. This runtime's JSON.stringify ignores its indent argument and emits compact output, so comparing against the pretty-printed expected file failed on whitespace for all 11 cases while the data matched exactly. Re-stringifying the parsed expectation puts both sides in one form; key order survives the round-trip, so real drift is still caught. --- .github/workflows/ci.yml | 7 +++- conformance/README.md | 9 ++++- package.json | 5 ++- test/conformance.tish | 79 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 test/conformance.tish diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c8917d..7383661 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,9 +29,14 @@ jobs: - name: Run tests with coverage run: npm run test:coverage - - name: Conformance corpus + - name: Conformance corpus (JS) run: npm run test:conformance + # The third target. The JS build and the Rust crate were both checked against the corpus; the + # Tish source itself was not, so "one source, three targets" had a leg missing. + - name: Conformance corpus (Tish) + run: npm run test:conformance:tish + - name: Examples run: npm run examples diff --git a/conformance/README.md b/conformance/README.md index 69b41e4..2e5c394 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -5,10 +5,15 @@ must produce the **same** parse: | Implementation | Runner | |----------------|--------| -| JS (`dist/deck.js`) | `npm run test:conformance` | -| Rust (`deckfile` crate, emitted from the same Tish source) | `cargo test` in the crate | +| **Tish** (`src/index.tish` — the source itself) | `npm run test:conformance:tish` | +| **JS** (`dist/deck.js`) | `npm run test:conformance` | +| **Rust** (`deckfile` crate, emitted from the same source) | `cargo test` in the crate | | A restricted host (tish-gba) | its own test, against the `gba` profile | +The Tish runner uses `tish run`, not the JS build, because reading the corpus needs `tish:fs` and the +JS target has no filesystem. It also compares **semantically** — `JSON.stringify` ignores its indent +argument on that runtime, so a text compare would fail on whitespace while the data matched. + Each case is `NNN-name.deck` plus `NNN-name.expected.json`, which holds the full observable parse: ```jsonc diff --git a/package.json b/package.json index ac4b0df..db41afa 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "scripts": { "build": "tish build src/index.tish -o dist/deck.js --target js && node scripts/append-exports.mjs", "tishversion": "tish -V", - "test": "npm run build && node test/coverage.mjs && node test/conformance.mjs && tish build --target js test/smoke.tish -o /tmp/spacedevin-deck-smoke.js && node /tmp/spacedevin-deck-smoke.js", + "test": "npm run build && node test/coverage.mjs && node test/conformance.mjs && npm run test:conformance:tish && tish build --target js test/smoke.tish -o /tmp/spacedevin-deck-smoke.js && node /tmp/spacedevin-deck-smoke.js", "test:conformance": "npm run build && node test/conformance.mjs", "conformance:update": "npm run build && node test/conformance.mjs --update", "test:coverage": "npm run build && c8 --check-coverage --lines 100 --functions 100 --statements 100 --include 'dist/deck.js' node test/coverage.mjs", @@ -51,7 +51,8 @@ "prepack": "npm run build", "prepublishOnly": "npm run build", "build:rust": "node scripts/build-rust.mjs", - "test:rust": "npm run build:rust && cd crate && cargo test" + "test:rust": "npm run build:rust && cd crate && cargo test", + "test:conformance:tish": "tish run test/conformance.tish" }, "c8": { "reporter": [ diff --git a/test/conformance.tish b/test/conformance.tish new file mode 100644 index 0000000..a6ea308 --- /dev/null +++ b/test/conformance.tish @@ -0,0 +1,79 @@ +// Conformance corpus runner — the TISH target. +// +// The corpus is what turns "one source, three targets" from a claim into a fact, and it was only +// being run by the JS build (test/conformance.mjs) and the Rust crate (rust/conformance.rs). This is +// the third leg: it imports ../src/index.tish, so it exercises the Tish source itself rather than +// something compiled from it. +// +// Run with `tish run` rather than the JS build, because the JS target has no filesystem — reading +// the corpus needs `tish:fs`. +// +// npm run test:conformance:tish +import { readDir, readFile } from "tish:fs" +import { parseProgram, parseTrackBody } from "../src/index.tish" + +let failed = 0 +fn check(name, cond) { + if (cond) { + console.log("ok " + name) + } else { + console.log("FAIL " + name) + failed = failed + 1 + } +} + +/// Same observable parse the other two runners snapshot: the program AST plus every track's and +/// clip's parsed body. +fn snapshotOf(src) { + let program = parseProgram(src) + let trackBodies = [] + let ti = 0 + while (ti < program.tracks.length) { + let t = program.tracks[ti] + let parsed = parseTrackBody(t.body) + trackBodies.push({ id: t.id, rows: parsed.rows, errors: parsed.errors }) + ti = ti + 1 + } + let clipBodies = [] + let ci = 0 + while (ci < program.clipBlocks.length) { + let c = program.clipBlocks[ci] + let parsed = parseTrackBody(c.body) + clipBodies.push({ clipId: c.clipId, rows: parsed.rows, errors: parsed.errors }) + ci = ci + 1 + } + return { program: program, trackBodies: trackBodies, clipBodies: clipBodies } +} + +let entries = readDir("conformance") +let cases = [] +let ni = 0 +while (ni < entries.length) { + let n = String(entries[ni]) + if (n.length > 5 && n.substring(n.length - 5) === ".deck") { + cases.push(n.substring(0, n.length - 5)) + } + ni = ni + 1 +} +cases.sort() +check("corpus present", cases.length >= 10) + +let ki = 0 +while (ki < cases.length) { + let name = cases[ki] + // Compare SEMANTICALLY, not textually: this runtime's JSON.stringify ignores the indent argument + // and emits compact output, so a text compare against the pretty-printed expected file would fail + // on whitespace while the data matched exactly. Re-stringifying the parsed expectation puts both + // sides in the same form; key order survives the round-trip, so this still catches real drift. + let actual = JSON.stringify(snapshotOf(readFile("conformance/" + name + ".deck"))) + let expected = JSON.stringify(JSON.parse(readFile("conformance/" + name + ".expected.json"))) + check(name, actual === expected) + ki = ki + 1 +} + +if (failed > 0) { + console.log(String(failed) + " FAILED") + process.exit(1) +} +console.log("") +console.log("CONFORMANCE_OK (tish) — " + String(cases.length) + " cases")