|
| 1 | +/* |
| 2 | + * Jest tests for the jactoml grammar via tokenizeContent. |
| 3 | + * Uses the fixture at src/test/fixtures/sample.jac.toml. |
| 4 | + */ |
| 5 | + |
| 6 | +import * as path from 'path'; |
| 7 | +import * as fs from 'fs'; |
| 8 | +import { tokenizeContent, TokenizeResult, TokenInfo } from '../commands/inspectTokenScopes'; |
| 9 | + |
| 10 | +const GRAMMAR_PATH = path.join(process.cwd(), 'syntaxes', 'jactoml.tmLanguage.json'); |
| 11 | +const WASM_PATH = path.join(process.cwd(), 'node_modules', 'vscode-oniguruma', 'release', 'onig.wasm'); |
| 12 | +const FIXTURE = path.join(process.cwd(), 'src', 'test', 'fixtures', 'sample.jac.toml'); |
| 13 | + |
| 14 | +const fixtureContent = fs.readFileSync(FIXTURE, 'utf-8'); |
| 15 | + |
| 16 | +function findTokenByText(result: TokenizeResult, line: number, text: string): TokenInfo | undefined { |
| 17 | + return result.tokens.find(t => t.line === line && t.text === text); |
| 18 | +} |
| 19 | + |
| 20 | +describe('jactoml grammar — sample.jac.toml', () => { |
| 21 | + let result: TokenizeResult; |
| 22 | + |
| 23 | + beforeAll(async () => { |
| 24 | + result = await tokenizeContent(fixtureContent, GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 25 | + }); |
| 26 | + |
| 27 | + test('grammar loads under source.jactoml and produces tokens', () => { |
| 28 | + expect(result.tokens.length).toBeGreaterThan(0); |
| 29 | + expect(result.tokens.every(t => t.scopes[0] === 'source.jactoml')).toBe(true); |
| 30 | + }); |
| 31 | + |
| 32 | + test('[project] table head is captured as one token under jac-known', () => { |
| 33 | + const tok = findTokenByText(result, 3, 'project'); |
| 34 | + expect(tok).toBeDefined(); |
| 35 | + expect(tok!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 36 | + }); |
| 37 | + |
| 38 | + test('[dependencies.npm]-style headers capture the full dotted path as one jac-known token (not split per segment)', async () => { |
| 39 | + const r = await tokenizeContent('[dependencies.npm]\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 40 | + const whole = r.tokens.find(t => t.text === 'dependencies.npm'); |
| 41 | + expect(whole).toBeDefined(); |
| 42 | + expect(whole!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 43 | + const justNpm = r.tokens.find(t => t.text === 'npm'); |
| 44 | + expect(justNpm).toBeUndefined(); |
| 45 | + }); |
| 46 | + |
| 47 | + test('[plugins.byllm.model] captures the full dotted path as one jac-known token', () => { |
| 48 | + const tok = findTokenByText(result, 47, 'plugins.byllm.model'); |
| 49 | + expect(tok).toBeDefined(); |
| 50 | + expect(tok!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 51 | + }); |
| 52 | + |
| 53 | + test('[[plugins.scale.microservices.shared_volumes]] array-of-tables captures full path under jac-known', () => { |
| 54 | + const tok = findTokenByText(result, 53, 'plugins.scale.microservices.shared_volumes'); |
| 55 | + expect(tok).toBeDefined(); |
| 56 | + expect(tok!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 57 | + }); |
| 58 | + |
| 59 | + test('arbitrary table names like [jac-shadcn] also get the jac-known scope (no allowlist)', async () => { |
| 60 | + const r = await tokenizeContent('[jac-shadcn]\n[desktop.window]\n[some.deeply.nested.x.y.z]\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 61 | + for (const name of ['jac-shadcn', 'desktop.window', 'some.deeply.nested.x.y.z']) { |
| 62 | + const tok = r.tokens.find(t => t.text === name); |
| 63 | + expect(tok).toBeDefined(); |
| 64 | + expect(tok!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 65 | + } |
| 66 | + }); |
| 67 | + |
| 68 | + test('every LHS key falls back to base taplo property scope (no allowlist)', () => { |
| 69 | + const tok = findTokenByText(result, 48, 'default_model'); |
| 70 | + expect(tok).toBeDefined(); |
| 71 | + expect(tok!.scopes).toContain('support.type.property-name.toml'); |
| 72 | + expect(tok!.scopes).not.toContain('support.type.property-name.jac-known.jactoml'); |
| 73 | + }); |
| 74 | + |
| 75 | + describe('[plugins.scale.sso.github] block in the fixture (env placeholders in real context)', () => { |
| 76 | + test('table header captures the full dotted path as one jac-known token', () => { |
| 77 | + const tok = findTokenByText(result, 57, 'plugins.scale.sso.github'); |
| 78 | + expect(tok).toBeDefined(); |
| 79 | + expect(tok!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 80 | + }); |
| 81 | + |
| 82 | + test('all four ${GITHUB_*} placeholders are scoped as variable.other.env.jactoml inside the double-quoted string', () => { |
| 83 | + const cases: Array<[number, string]> = [ |
| 84 | + [58, '${GITHUB_CLIENT_ID}'], |
| 85 | + [59, '${GITHUB_CLIENT_SECRET}'], |
| 86 | + [60, '${GITHUB_PUBLIC_LINK}'], |
| 87 | + [61, '${GITHUB_APP_SLUG}'], |
| 88 | + ]; |
| 89 | + for (const [line, text] of cases) { |
| 90 | + const tok = findTokenByText(result, line, text); |
| 91 | + expect(tok).toBeDefined(); |
| 92 | + expect(tok!.scopes).toContain('variable.other.env.jactoml'); |
| 93 | + expect(tok!.scopes).toContain('string.quoted.single.basic.line.toml'); |
| 94 | + } |
| 95 | + }); |
| 96 | + |
| 97 | + test('bare $HOST (no braces) is scoped', () => { |
| 98 | + const tok = findTokenByText(result, 62, '$HOST'); |
| 99 | + expect(tok).toBeDefined(); |
| 100 | + expect(tok!.scopes).toContain('variable.other.env.jactoml'); |
| 101 | + }); |
| 102 | + |
| 103 | + test('a string with no placeholder has no env scope', () => { |
| 104 | + const tok = findTokenByText(result, 63, 'plain literal value'); |
| 105 | + expect(tok).toBeDefined(); |
| 106 | + expect(tok!.scopes).not.toContain('variable.other.env.jactoml'); |
| 107 | + }); |
| 108 | + |
| 109 | + test('the LHS keys (client_id, client_secret, public_link, app_slug, host) are recognised as TOML property names by the base grammar', () => { |
| 110 | + for (const [line, name] of [[58, 'client_id'], [59, 'client_secret'], [60, 'public_link'], [61, 'app_slug'], [62, 'host']] as const) { |
| 111 | + const tok = findTokenByText(result, line, name); |
| 112 | + expect(tok).toBeDefined(); |
| 113 | + expect(tok!.scopes).toContain('support.type.property-name.toml'); |
| 114 | + } |
| 115 | + }); |
| 116 | + }); |
| 117 | + |
| 118 | + describe('incomplete-bracket recovery (mirrors jac grammar -incomplete convention)', () => { |
| 119 | + test('an unclosed [project line is marked invalid.illegal.table.jactoml', async () => { |
| 120 | + const r = await tokenizeContent('[project\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 121 | + const tok = r.tokens.find(t => t.text === '[project'); |
| 122 | + expect(tok).toBeDefined(); |
| 123 | + expect(tok!.scopes).toContain('invalid.illegal.table.jactoml'); |
| 124 | + }); |
| 125 | + |
| 126 | + test('lines AFTER an unclosed [project keep their proper scoping (no cascade)', async () => { |
| 127 | + const src = '[project\nname = "jac-ide"\nversion = "1.0.0"\n[dependencies]\nwebsockets = ">=12.0"\n'; |
| 128 | + const r = await tokenizeContent(src, GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 129 | + |
| 130 | + const nameKey = r.tokens.find(t => t.line === 2 && t.text === 'name'); |
| 131 | + expect(nameKey).toBeDefined(); |
| 132 | + expect(nameKey!.scopes).toContain('support.type.property-name.toml'); |
| 133 | + expect(nameKey!.scopes.some(s => s.startsWith('meta.array'))).toBe(false); |
| 134 | + |
| 135 | + const depsTable = r.tokens.find(t => t.line === 4 && t.text === 'dependencies'); |
| 136 | + expect(depsTable).toBeDefined(); |
| 137 | + expect(depsTable!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 138 | + |
| 139 | + const wsKey = r.tokens.find(t => t.line === 5 && t.text === 'websockets'); |
| 140 | + expect(wsKey).toBeDefined(); |
| 141 | + expect(wsKey!.scopes).toContain('support.type.property-name.toml'); |
| 142 | + }); |
| 143 | + |
| 144 | + test('an unclosed [[plugins.x line is marked invalid and does not cascade', async () => { |
| 145 | + const src = '[[plugins.x\nname = "foo"\n[dependencies]\n'; |
| 146 | + const r = await tokenizeContent(src, GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 147 | + |
| 148 | + const badHead = r.tokens.find(t => t.line === 1 && t.text === '[[plugins.x'); |
| 149 | + expect(badHead).toBeDefined(); |
| 150 | + expect(badHead!.scopes).toContain('invalid.illegal.table.jactoml'); |
| 151 | + |
| 152 | + const depsTable = r.tokens.find(t => t.line === 3 && t.text === 'dependencies'); |
| 153 | + expect(depsTable).toBeDefined(); |
| 154 | + expect(depsTable!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 155 | + }); |
| 156 | + }); |
| 157 | + |
| 158 | + describe('env-var placeholders inside strings', () => { |
| 159 | + test('${VAR} inside double-quoted string is scoped as variable.other.env.jactoml', async () => { |
| 160 | + const r = await tokenizeContent('image_registry = "${ECR_REGISTRY}"\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 161 | + const tok = r.tokens.find(t => t.text === '${ECR_REGISTRY}'); |
| 162 | + expect(tok).toBeDefined(); |
| 163 | + expect(tok!.scopes).toContain('variable.other.env.jactoml'); |
| 164 | + }); |
| 165 | + |
| 166 | + test('bare $VAR (no braces) is also scoped', async () => { |
| 167 | + const r = await tokenizeContent('host = "$HOST"\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 168 | + const tok = r.tokens.find(t => t.text === '$HOST'); |
| 169 | + expect(tok).toBeDefined(); |
| 170 | + expect(tok!.scopes).toContain('variable.other.env.jactoml'); |
| 171 | + }); |
| 172 | + |
| 173 | + test('${VAR} inside single-quoted (literal) string is also scoped', async () => { |
| 174 | + const r = await tokenizeContent("literal = '${LITERAL_VAR}'\n", GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 175 | + const tok = r.tokens.find(t => t.text === '${LITERAL_VAR}'); |
| 176 | + expect(tok).toBeDefined(); |
| 177 | + expect(tok!.scopes).toContain('variable.other.env.jactoml'); |
| 178 | + }); |
| 179 | + |
| 180 | + test('mixed text around ${VAR} keeps the surrounding plain string un-scoped', async () => { |
| 181 | + const r = await tokenizeContent('mixed = "prefix-${MID}-suffix"\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 182 | + const placeholder = r.tokens.find(t => t.text === '${MID}'); |
| 183 | + expect(placeholder).toBeDefined(); |
| 184 | + expect(placeholder!.scopes).toContain('variable.other.env.jactoml'); |
| 185 | + const prefix = r.tokens.find(t => t.text === 'prefix-'); |
| 186 | + expect(prefix).toBeDefined(); |
| 187 | + expect(prefix!.scopes).not.toContain('variable.other.env.jactoml'); |
| 188 | + }); |
| 189 | + |
| 190 | + test('a plain string with no $ has no env scope', async () => { |
| 191 | + const r = await tokenizeContent('plain = "no vars here"\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 192 | + for (const t of r.tokens) { |
| 193 | + expect(t.scopes).not.toContain('variable.other.env.jactoml'); |
| 194 | + } |
| 195 | + }); |
| 196 | + }); |
| 197 | + |
| 198 | + describe('edge cases (TOML stress tests)', () => { |
| 199 | + test('triple-double-quoted multiline string spans lines and keeps its scope', async () => { |
| 200 | + const src = 'k = """\nLine 1\nLine 2 with $dollar and "quotes"\n"""\n'; |
| 201 | + const r = await tokenizeContent(src, GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 202 | + const body = r.tokens.find(t => t.line === 3 && t.text.startsWith('Line 2')); |
| 203 | + expect(body).toBeDefined(); |
| 204 | + expect(body!.scopes).toContain('string.quoted.triple.basic.block.toml'); |
| 205 | + const dollarVar = r.tokens.find(t => t.line === 3 && t.text === '$dollar'); |
| 206 | + expect(dollarVar).toBeDefined(); |
| 207 | + expect(dollarVar!.scopes).toContain('variable.other.env.jactoml'); |
| 208 | + }); |
| 209 | + |
| 210 | + test('triple-single-quoted literal multiline does NOT process escape sequences', async () => { |
| 211 | + const src = "regex = '''^\\d{3}-\\d{2}-\\d{4}$'''\n"; |
| 212 | + const r = await tokenizeContent(src, GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 213 | + const lineToks = r.tokens.filter(t => t.line === 1); |
| 214 | + const hasLiteralString = lineToks.some(t => t.scopes.includes('string.quoted.triple.literal.block.toml')); |
| 215 | + expect(hasLiteralString).toBe(true); |
| 216 | + const hasEscape = lineToks.some(t => t.scopes.includes('constant.character.escape.toml')); |
| 217 | + expect(hasEscape).toBe(false); |
| 218 | + }); |
| 219 | + |
| 220 | + test('quoted dotted key "a.b.c" = "..." is a property name, NOT a table header', async () => { |
| 221 | + const r = await tokenizeContent('"a.b.c" = "this is a key, not a table"\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 222 | + const lineToks = r.tokens.filter(t => t.line === 1); |
| 223 | + const isJacKnown = lineToks.some(t => t.scopes.includes('entity.name.tag.jac-known.jactoml')); |
| 224 | + expect(isJacKnown).toBe(false); |
| 225 | + const isProperty = lineToks.some(t => t.scopes.includes('support.type.property-name.toml')); |
| 226 | + expect(isProperty).toBe(true); |
| 227 | + }); |
| 228 | + |
| 229 | + test('deeply nested table header [a.b.c.d.e.f.g] is one whole-path jac-known token', async () => { |
| 230 | + const r = await tokenizeContent('[a.b.c.d.e.f.g]\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 231 | + const tok = r.tokens.find(t => t.text === 'a.b.c.d.e.f.g'); |
| 232 | + expect(tok).toBeDefined(); |
| 233 | + expect(tok!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 234 | + }); |
| 235 | + |
| 236 | + test('repeated [[array.of.tables]] headers are each tagged jac-known', async () => { |
| 237 | + const src = '[[products]]\nname = "A"\n[[products]]\nname = "B"\n [[products.features]]\n k = "v"\n'; |
| 238 | + const r = await tokenizeContent(src, GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 239 | + const headers = r.tokens.filter(t => |
| 240 | + t.scopes.includes('entity.name.tag.jac-known.jactoml') && |
| 241 | + (t.text === 'products' || t.text === 'products.features') |
| 242 | + ); |
| 243 | + expect(headers.length).toBe(3); |
| 244 | + }); |
| 245 | + |
| 246 | + test('inline table { k = "v", n = 42 } scopes keys as meta.table.inline.toml + property name', async () => { |
| 247 | + const r = await tokenizeContent('inline = { k = "v", n = 42 }\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 248 | + const k = r.tokens.find(t => t.line === 1 && t.text === 'k' && t.scopes.includes('meta.table.inline.toml')); |
| 249 | + expect(k).toBeDefined(); |
| 250 | + expect(k!.scopes).toContain('support.type.property-name.toml'); |
| 251 | + const n = r.tokens.find(t => t.line === 1 && t.text === '42' && t.scopes.includes('meta.table.inline.toml')); |
| 252 | + expect(n).toBeDefined(); |
| 253 | + expect(n!.scopes).toContain('constant.numeric.integer.toml'); |
| 254 | + }); |
| 255 | + |
| 256 | + test('datetime value like 1999-12-31T23:59:59Z is a TOML datetime constant', async () => { |
| 257 | + const r = await tokenizeContent('dob = 1999-12-31T23:59:59Z\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 258 | + const tok = r.tokens.find(t => t.text === '1999-12-31T23:59:59Z'); |
| 259 | + expect(tok).toBeDefined(); |
| 260 | + expect(tok!.scopes.some(s => s.includes('time.datetime'))).toBe(true); |
| 261 | + }); |
| 262 | + |
| 263 | + test('empty table [empty] is still tagged as jac-known', async () => { |
| 264 | + const r = await tokenizeContent('[empty]\nkey_in_next_section = 1\n', GRAMMAR_PATH, WASM_PATH, 'source.jactoml'); |
| 265 | + const tok = r.tokens.find(t => t.text === 'empty'); |
| 266 | + expect(tok).toBeDefined(); |
| 267 | + expect(tok!.scopes).toContain('entity.name.tag.jac-known.jactoml'); |
| 268 | + }); |
| 269 | + }); |
| 270 | +}); |
0 commit comments