|
| 1 | +// ───────────────────────────────────────────────────────────────────────────── |
| 2 | +// tsx-highlight.ts — the JSX *highlighter* gate (tsx-conformance.ts is the JSX |
| 3 | +// *parser* gate). It checks the scopes Monogram's GENERATED examples/tsx.tmLanguage.json |
| 4 | +// emits for the JSX-dialect constructs the TS/JS benches can't reach: element & |
| 5 | +// fragment tags, attributes, expression containers, raw text children, and HTML |
| 6 | +// character entities. |
| 7 | +// |
| 8 | +// Two views: |
| 9 | +// 1. CURATED checks — a hard gate: specific JSX tokens must carry specific |
| 10 | +// scopes (the same style as test/issue-cases.ts). These encode the JSX |
| 11 | +// contract and catch regressions in the dialect patterns. |
| 12 | +// 2. DROP-IN agreement (opt-in: set MONOGRAM_OFFICIAL_TSX) — over a JSX corpus, |
| 13 | +// the share of official-emitted scopes Monogram matches at family / exact |
| 14 | +// granularity. For the JSX dialect the official TypeScriptReact grammar is |
| 15 | +// the de-facto reference (tsc exposes no neutral per-token JSX scope roles |
| 16 | +// the way scope-roles.ts does for TS), so this is an agreement measure, not |
| 17 | +// an absolute-accuracy one — hence opt-in and not the hard gate. |
| 18 | +// |
| 19 | +// Run: `node test/tsx-highlight.ts` (set MONOGRAM_OFFICIAL_TSX for the drop-in view) |
| 20 | +// ───────────────────────────────────────────────────────────────────────────── |
| 21 | +import vsctm from 'vscode-textmate'; |
| 22 | +import onig from 'vscode-oniguruma'; |
| 23 | +import { readFileSync, existsSync } from 'node:fs'; |
| 24 | +import { createRequire } from 'node:module'; |
| 25 | +import { scopeFamily } from './highlight-engines.ts'; |
| 26 | + |
| 27 | +const { INITIAL, Registry, parseRawGrammar } = vsctm; |
| 28 | +const require = createRequire(import.meta.url); |
| 29 | +const wasmBin = readFileSync(require.resolve('vscode-oniguruma/release/onig.wasm')); |
| 30 | +await onig.loadWASM(wasmBin.buffer.slice(wasmBin.byteOffset, wasmBin.byteOffset + wasmBin.byteLength)); |
| 31 | + |
| 32 | +function load(scopeName: string, path: string): Promise<vsctm.IGrammar | null> { |
| 33 | + const content = readFileSync(path, 'utf-8'); |
| 34 | + return new Registry({ |
| 35 | + onigLib: Promise.resolve({ |
| 36 | + createOnigScanner: (p: string[]) => new onig.OnigScanner(p), |
| 37 | + createOnigString: (s: string) => new onig.OnigString(s), |
| 38 | + }), |
| 39 | + loadGrammar: async (sn: string) => (sn === scopeName ? parseRawGrammar(content, 'g.json') : null), |
| 40 | + }).loadGrammar(scopeName); |
| 41 | +} |
| 42 | + |
| 43 | +interface Tok { start: number; end: number; text: string; scope: string } |
| 44 | +function tokenize(g: vsctm.IGrammar, text: string): Tok[] { |
| 45 | + const out: Tok[] = []; |
| 46 | + let rs = INITIAL, off = 0; |
| 47 | + for (const line of text.split('\n')) { |
| 48 | + const r = g.tokenizeLine(line, rs); |
| 49 | + for (const t of r.tokens) out.push({ start: off + t.startIndex, end: off + t.endIndex, text: line.slice(t.startIndex, t.endIndex), scope: t.scopes[t.scopes.length - 1] }); |
| 50 | + rs = r.ruleStack; off += line.length + 1; |
| 51 | + } |
| 52 | + return out; |
| 53 | +} |
| 54 | +// Strip only a TRAILING language suffix (`….tsx`/`.ts`/`.js`); never a mid-path |
| 55 | +// `.jsx` (it is structural, e.g. `meta.jsx.children`). |
| 56 | +const norm = (s: string) => s.replace(/\.(tsx|ts|js)$/, ''); |
| 57 | + |
| 58 | +const MONO_PATH = 'examples/tsx.tmLanguage.json'; |
| 59 | +if (!existsSync(MONO_PATH)) { |
| 60 | + console.error(`Monogram TSX grammar not found at ${MONO_PATH}. Run: node src/cli.ts examples/tsx.ts`); |
| 61 | + process.exit(1); |
| 62 | +} |
| 63 | +const mono = (await load('source.tsx', MONO_PATH))!; |
| 64 | + |
| 65 | +// ── 1. Curated checks: (snippet, substring, expected-scope-substring) ── |
| 66 | +// The expected scope is matched modulo the language suffix and as a path prefix, |
| 67 | +// so `entity.name.tag` accepts `entity.name.tag.tsx`. |
| 68 | +const checks: { label: string; code: string; want: { text: string; scope: string }[] }[] = [ |
| 69 | + { |
| 70 | + label: 'element tag + close', |
| 71 | + code: `const a = <div></div>;`, |
| 72 | + want: [ |
| 73 | + { text: 'div', scope: 'entity.name.tag' }, |
| 74 | + { text: '<', scope: 'punctuation.definition.tag.begin' }, |
| 75 | + ], |
| 76 | + }, |
| 77 | + { |
| 78 | + label: 'attributes: name, =, string, expression container', |
| 79 | + code: `const b = <input type="text" value={v} />;`, |
| 80 | + want: [ |
| 81 | + { text: 'type', scope: 'entity.other.attribute-name' }, |
| 82 | + { text: 'text', scope: 'string.quoted.double' }, |
| 83 | + { text: 'value', scope: 'entity.other.attribute-name' }, |
| 84 | + { text: 'v', scope: 'variable.other' }, |
| 85 | + ], |
| 86 | + }, |
| 87 | + { |
| 88 | + label: 'raw text with arbitrary punctuation → meta.jsx.children', |
| 89 | + code: `const c = <p>It's 100% & more (really)!</p>;`, |
| 90 | + want: [ |
| 91 | + { text: `It's 100% & more (really)!`, scope: 'meta.jsx.children' }, |
| 92 | + ], |
| 93 | + }, |
| 94 | + { |
| 95 | + label: 'HTML named entity', |
| 96 | + code: `const d = <span> </span>;`, |
| 97 | + want: [ |
| 98 | + { text: '&', scope: 'punctuation.definition.entity' }, |
| 99 | + { text: 'nbsp', scope: 'constant.character.entity' }, |
| 100 | + { text: ';', scope: 'punctuation.definition.entity' }, |
| 101 | + ], |
| 102 | + }, |
| 103 | + { |
| 104 | + label: 'HTML numeric + hex entity', |
| 105 | + code: `const e = <span>{😀</span>;`, |
| 106 | + want: [ |
| 107 | + { text: '#123', scope: 'constant.character.entity' }, |
| 108 | + { text: '#x1F600', scope: 'constant.character.entity' }, |
| 109 | + ], |
| 110 | + }, |
| 111 | + { |
| 112 | + label: 'lone & in text stays plain children (no false entity)', |
| 113 | + code: `const f = <p>Tom & Jerry</p>;`, |
| 114 | + want: [ |
| 115 | + { text: 'Tom & Jerry', scope: 'meta.jsx.children' }, |
| 116 | + ], |
| 117 | + }, |
| 118 | + { |
| 119 | + label: 'text interleaved with expression container', |
| 120 | + code: `const g = <p>Hello {name}, welcome</p>;`, |
| 121 | + want: [ |
| 122 | + { text: 'Hello ', scope: 'meta.jsx.children' }, |
| 123 | + { text: 'name', scope: 'variable.other' }, |
| 124 | + { text: ', welcome', scope: 'meta.jsx.children' }, |
| 125 | + ], |
| 126 | + }, |
| 127 | + { |
| 128 | + label: 'fragment children', |
| 129 | + code: `const h = <><span>1</span></>;`, |
| 130 | + want: [ |
| 131 | + { text: 'span', scope: 'entity.name.tag' }, |
| 132 | + ], |
| 133 | + }, |
| 134 | +]; |
| 135 | + |
| 136 | +const scopeAt = (toks: Tok[], text: string): string | null => { |
| 137 | + const t = toks.find((x) => x.text === text); |
| 138 | + return t ? norm(t.scope) : null; |
| 139 | +}; |
| 140 | +let pass = 0, total = 0; |
| 141 | +const fails: string[] = []; |
| 142 | +for (const { label, code, want } of checks) { |
| 143 | + const toks = tokenize(mono, code); |
| 144 | + for (const w of want) { |
| 145 | + total++; |
| 146 | + const got = scopeAt(toks, w.text); |
| 147 | + // prefix match (modulo suffix): want `entity.name.tag` accepts `entity.name.tag.begin`? no — |
| 148 | + // we want the LEAF to START WITH the expected path, so `entity.name.tag` ⊆ `entity.name.tag`. |
| 149 | + if (got && (got === w.want || got.startsWith(w.scope))) pass++; |
| 150 | + else fails.push(` [${label}] «${w.text}» want ⊇ ${w.scope} got ${got ?? '(none)'}`); |
| 151 | + } |
| 152 | +} |
| 153 | +console.log('── JSX highlighter — curated scope checks ──'); |
| 154 | +console.log(` ${pass}/${total} checks pass`); |
| 155 | +if (fails.length) { console.log(' FAILURES:'); for (const f of fails) console.log(f); } |
| 156 | + |
| 157 | +// ── 2. Drop-in agreement vs the official TypeScriptReact grammar (opt-in) ── |
| 158 | +const OFF = process.env.MONOGRAM_OFFICIAL_TSX |
| 159 | + ?? '/Applications/Visual Studio Code.app/Contents/Resources/app/extensions/typescript-basics/syntaxes/TypeScriptReact.tmLanguage.json'; |
| 160 | +const CORPUS = [ |
| 161 | + `const a = <div className="x" data-id={5}>It's 100% & more!</div>;`, |
| 162 | + `const b = <ul>{items.map(x => <li key={x.id}>{x.name}</li>)}</ul>;`, |
| 163 | + `const c = <span> & entities {</span>;`, |
| 164 | + `const d = <Foo.Bar baz={1}><Child /></Foo.Bar>;`, |
| 165 | + `const e = <><Header title="Hi" />{children}<Footer /></>;`, |
| 166 | + `const f = <input type="text" value={v} disabled {...rest} />;`, |
| 167 | +]; |
| 168 | +if (existsSync(OFF)) { |
| 169 | + const off = (await load('source.tsx', OFF))!; |
| 170 | + let exact = 0, fam = 0, graded = 0; |
| 171 | + for (const code of CORPUS) { |
| 172 | + const mt = tokenize(mono, code), ot = tokenize(off, code); |
| 173 | + for (const o of ot) { |
| 174 | + if (!o.text.trim() || o.scope === 'source.tsx') continue; // only where official colors |
| 175 | + // only JSX-dialect tokens (skip the shared TS surface already graded elsewhere) |
| 176 | + if (!/\b(tag|jsx|attribute-name|character\.entity|definition\.entity|section\.embedded)\b/.test(o.scope)) continue; |
| 177 | + graded++; |
| 178 | + const m = mt.find((x) => x.start <= o.start && o.start < x.end); |
| 179 | + if (!m) continue; |
| 180 | + if (norm(m.scope) === norm(o.scope)) { exact++; fam++; } |
| 181 | + else if (scopeFamily(m.scope) === scopeFamily(o.scope)) fam++; |
| 182 | + } |
| 183 | + } |
| 184 | + console.log('\n── JSX dialect drop-in vs official TypeScriptReact (agreement, opt-in) ──'); |
| 185 | + console.log(` graded ${graded} JSX tokens · exact ${(exact / graded * 100).toFixed(1)}% family ${(fam / graded * 100).toFixed(1)}%`); |
| 186 | +} else { |
| 187 | + console.log('\n (set MONOGRAM_OFFICIAL_TSX to also measure JSX drop-in agreement vs official)'); |
| 188 | +} |
| 189 | + |
| 190 | +const FLOOR = checks.reduce((n, c) => n + c.want.length, 0); |
| 191 | +if (pass < FLOOR) { console.log(`\n✗ JSX highlighter curated checks ${pass}/${FLOOR}`); process.exit(1); } |
| 192 | +console.log(`\n✓ JSX highlighter: ${pass}/${total} curated scope checks pass`); |
0 commit comments