test: run the conformance corpus from Tish, the third target (#12)

The corpus exists to make "one source, three targets" a fact rather than a
claim, and it was only ever run by two of them: the JS build and the Rust crate.
The Tish source itself — what both are generated FROM — was never checked
against it. A change that altered the Tish semantics but survived both emits
would have gone unnoticed by the very thing meant to catch it.

test/conformance.tish imports ../src/index.tish and runs the same inputs against
the same expected files. All 11 cases pass, so the three targets are now
demonstrably in agreement rather than assumed to be.

Two things it has to do differently, both noted where they happen:

Run under `tish run` instead of the JS build, because reading the corpus needs
`tish:fs` and the JS target has no filesystem at all.

Compare semantically rather than textually. This runtime's JSON.stringify
ignores its indent argument and emits compact output, so comparing against the
pretty-printed expected file failed on whitespace for all 11 cases while the
data matched exactly. Re-stringifying the parsed expectation puts both sides in
one form; key order survives the round-trip, so real drift is still caught.

Author: spacedevin

.github/workflows/ci.yml

@@ -29,9 +29,14 @@ jobs:
       - name: Run tests with coverage
         run: npm run test:coverage
 
-      - name: Conformance corpus
+      - name: Conformance corpus (JS)
         run: npm run test:conformance
 
+      # The third target. The JS build and the Rust crate were both checked against the corpus; the
+      # Tish source itself was not, so "one source, three targets" had a leg missing.
+      - name: Conformance corpus (Tish)
+        run: npm run test:conformance:tish
+
       - name: Examples
         run: npm run examples
 

conformance/README.md

@@ -5,10 +5,15 @@ must produce the **same** parse:
 
 | Implementation | Runner |
 |----------------|--------|
-| JS (`dist/deck.js`) | `npm run test:conformance` |
-| Rust (`deckfile` crate, emitted from the same Tish source) | `cargo test` in the crate |
+| **Tish** (`src/index.tish` — the source itself) | `npm run test:conformance:tish` |
+| **JS** (`dist/deck.js`) | `npm run test:conformance` |
+| **Rust** (`deckfile` crate, emitted from the same source) | `cargo test` in the crate |
 | A restricted host (tish-gba) | its own test, against the `gba` profile |
 
+The Tish runner uses `tish run`, not the JS build, because reading the corpus needs `tish:fs` and the
+JS target has no filesystem. It also compares **semantically** — `JSON.stringify` ignores its indent
+argument on that runtime, so a text compare would fail on whitespace while the data matched.
+
 Each case is `NNN-name.deck` plus `NNN-name.expected.json`, which holds the full observable parse:
 
 ```jsonc

package.json

@@ -43,15 +43,16 @@
   "scripts": {
     "build": "tish build src/index.tish -o dist/deck.js --target js && node scripts/append-exports.mjs",
     "tishversion": "tish -V",
-    "test": "npm run build && node test/coverage.mjs && node test/conformance.mjs && tish build --target js test/smoke.tish -o /tmp/spacedevin-deck-smoke.js && node /tmp/spacedevin-deck-smoke.js",
+    "test": "npm run build && node test/coverage.mjs && node test/conformance.mjs && npm run test:conformance:tish && tish build --target js test/smoke.tish -o /tmp/spacedevin-deck-smoke.js && node /tmp/spacedevin-deck-smoke.js",
     "test:conformance": "npm run build && node test/conformance.mjs",
     "conformance:update": "npm run build && node test/conformance.mjs --update",
     "test:coverage": "npm run build && c8 --check-coverage --lines 100 --functions 100 --statements 100 --include 'dist/deck.js' node test/coverage.mjs",
     "examples": "tish build --target js examples/01-parse.tish -o /tmp/deck-ex01.js && node /tmp/deck-ex01.js && tish build --target js examples/02-host-boot.tish -o /tmp/deck-ex02.js && node /tmp/deck-ex02.js && tish build --target js examples/03-helpers.tish -o /tmp/deck-ex03.js && node /tmp/deck-ex03.js",
     "prepack": "npm run build",
     "prepublishOnly": "npm run build",
     "build:rust": "node scripts/build-rust.mjs",
-    "test:rust": "npm run build:rust && cd crate && cargo test"
+    "test:rust": "npm run build:rust && cd crate && cargo test",
+    "test:conformance:tish": "tish run test/conformance.tish"
   },
   "c8": {
     "reporter": [

test/conformance.tish

@@ -0,0 +1,79 @@
+// Conformance corpus runner — the TISH target.
+//
+// The corpus is what turns "one source, three targets" from a claim into a fact, and it was only
+// being run by the JS build (test/conformance.mjs) and the Rust crate (rust/conformance.rs). This is
+// the third leg: it imports ../src/index.tish, so it exercises the Tish source itself rather than
+// something compiled from it.
+//
+// Run with `tish run` rather than the JS build, because the JS target has no filesystem — reading
+// the corpus needs `tish:fs`.
+//
+//   npm run test:conformance:tish
+import { readDir, readFile } from "tish:fs"
+import { parseProgram, parseTrackBody } from "../src/index.tish"
+
+let failed = 0
+fn check(name, cond) {
+  if (cond) {
+    console.log("ok   " + name)
+  } else {
+    console.log("FAIL " + name)
+    failed = failed + 1
+  }
+}
+
+/// Same observable parse the other two runners snapshot: the program AST plus every track's and
+/// clip's parsed body.
+fn snapshotOf(src) {
+  let program = parseProgram(src)
+  let trackBodies = []
+  let ti = 0
+  while (ti < program.tracks.length) {
+    let t = program.tracks[ti]
+    let parsed = parseTrackBody(t.body)
+    trackBodies.push({ id: t.id, rows: parsed.rows, errors: parsed.errors })
+    ti = ti + 1
+  }
+  let clipBodies = []
+  let ci = 0
+  while (ci < program.clipBlocks.length) {
+    let c = program.clipBlocks[ci]
+    let parsed = parseTrackBody(c.body)
+    clipBodies.push({ clipId: c.clipId, rows: parsed.rows, errors: parsed.errors })
+    ci = ci + 1
+  }
+  return { program: program, trackBodies: trackBodies, clipBodies: clipBodies }
+}
+
+let entries = readDir("conformance")
+let cases = []
+let ni = 0
+while (ni < entries.length) {
+  let n = String(entries[ni])
+  if (n.length > 5 && n.substring(n.length - 5) === ".deck") {
+    cases.push(n.substring(0, n.length - 5))
+  }
+  ni = ni + 1
+}
+cases.sort()
+check("corpus present", cases.length >= 10)
+
+let ki = 0
+while (ki < cases.length) {
+  let name = cases[ki]
+  // Compare SEMANTICALLY, not textually: this runtime's JSON.stringify ignores the indent argument
+  // and emits compact output, so a text compare against the pretty-printed expected file would fail
+  // on whitespace while the data matched exactly. Re-stringifying the parsed expectation puts both
+  // sides in the same form; key order survives the round-trip, so this still catches real drift.
+  let actual = JSON.stringify(snapshotOf(readFile("conformance/" + name + ".deck")))
+  let expected = JSON.stringify(JSON.parse(readFile("conformance/" + name + ".expected.json")))
+  check(name, actual === expected)
+  ki = ki + 1
+}
+
+if (failed > 0) {
+  console.log(String(failed) + " FAILED")
+  process.exit(1)
+}
+console.log("")
+console.log("CONFORMANCE_OK (tish) — " + String(cases.length) + " cases")