gen-tm: model import defer * as ns so defer is a keyword (fixes TS #1058)

TS 5.9 deferred import — `import defer * as ns from "x"` — should scope
`defer` as a keyword; both grammars read it as an ordinary alias variable.

Two DATA edits in typescript.ts: ImportClause gains `['defer','*','as',Ident]`
(models the shape so `defer` is structural only before the namespace `*`),
and the scope map gains `keyword.control.import.phase: ['defer']`. `defer`
is NOT in any not()/reserved set, so it stays a valid identifier everywhere
else. A new agnostic detector in gen-tm (`usedAsKeywordOnlyBeforeStar`) finds
any keyword literal whose every occurrence sits right before `*` and emits it
positionally via #import-export-all `(import)\s+(?:(defer)\s+)?(\*)` — derived
from the rule graph, nothing hardcodes "defer". All four generators re-derive
it (TM, Monarch, tree-sitter).

Purely additive to the CFG (the parser already accepted the input as a
degenerate parse): conformance smoke test shows 0 accept/reject deltas vs
master. `defer` as an identifier is preserved (`const defer`, `defer()`,
`import defer from`). TS ledger 26->27/27 — only-official 0, both-miss 0.
agnostic 8/8, test-issues 350/0, tsx/jsx/js-conformance clean, 56/56, tsc
clean; JS/HTML/Vue byte-identical.

Author: johnsoncodehk

README.md

@@ -46,7 +46,7 @@ The same question, every language at once: take the bugs reported against each *
 
 <!-- issues:start -->
 <!-- generated by `npm run bench:issues` — do not edit by hand -->
-_Each hand-written **official** grammar vs Monogram's **derived** one, on the bugs filed against it: **TypeScript 26/27** (official 9/27) · **TSX 11/11** (official 6/11) · **HTML 20/20** (official 13/20) · **Vue 19/19** (official 15/19). Per-issue detail below — auto-generated by `npm run bench:issues`._
+_Each hand-written **official** grammar vs Monogram's **derived** one, on the bugs filed against it: **TypeScript 27/27** (official 9/27) · **TSX 11/11** (official 6/11) · **HTML 20/20** (official 13/20) · **Vue 19/19** (official 15/19). Per-issue detail below — auto-generated by `npm run bench:issues`._
 
 #### TypeScript
 | issue | Monogram | official |
@@ -68,7 +68,7 @@ _Each hand-written **official** grammar vs Monogram's **derived** one, on the bu
 | [#891](https://github.com/microsoft/TypeScript-TmLanguage/issues/891) — `from` as an ordinary variable is not a keyword | ✓ | · |
 | [#814](https://github.com/microsoft/TypeScript-TmLanguage/issues/814) — `a instanceof B & c` keeps the operand a value, not a type | ✓ | · |
 | [#950](https://github.com/microsoft/TypeScript-TmLanguage/issues/950) — default import named `type` — the binding is a variable, not the `type` keyword | ✓ | · |
-| [#1058](https://github.com/microsoft/TypeScript-TmLanguage/issues/1058) — `import defer` should scope `defer` as a keyword | · | · |
+| [#1058](https://github.com/microsoft/TypeScript-TmLanguage/issues/1058) — `import defer` should scope `defer` as a keyword | ✓ | · |
 
 <details><summary>… and 9 more both grammars already handle (✓ / ✓)</summary>
 

src/gen-tm.ts

@@ -4765,9 +4765,41 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
   // grammar/scope map; the rule fires only on the keyword→`*` adjacency that
   // actually occurs in a rule, so an arithmetic `*` is never mis-scoped (import/
   // export keywords are reserved and can never be a multiplication operand).
+  // A phase modifier is a keyword literal whose EVERY keyword occurrence in the
+  // grammar sits immediately before the namespace `*` (e.g. `defer`, only ever in
+  // `import defer * as ns`). Such a word is NOT reserved — it stays a valid binding
+  // identifier elsewhere (`const defer = 1`, `defer()`, `import defer from "m"`) — so
+  // it must be scoped POSITIONALLY (right before `*`, via #import-export-all below),
+  // never in the flat keyword match. Words used as keywords in OTHER positions too
+  // (`import`; `export`, before `*` AND in `export default …`) are NOT phase
+  // modifiers — they keep their normal flat scoping and may introduce the `*`.
+  const usedAsKeywordOnlyBeforeStar = (lit: string): boolean => {
+    let beforeStar = false, elsewhere = false;
+    const walk = (e: RuleExpr | undefined): void => {
+      if (!e) return;
+      if (e.type === 'seq') {
+        for (let i = 0; i < e.items.length; i++) {
+          const it = e.items[i];
+          if (it.type === 'literal' && it.value === lit) {
+            const nx = e.items[i + 1];
+            if (nx && nx.type === 'literal' && nx.value === '*') beforeStar = true; else elsewhere = true;
+          }
+          walk(it);
+        }
+      } else if (e.type === 'alt') e.items.forEach(walk);
+      else if (e.type === 'quantifier' || e.type === 'group' || e.type === 'not') walk(e.body);
+      else if (e.type === 'sep') walk(e.element);
+    };
+    for (const r of grammar.rules) walk(r.body);
+    return beforeStar && !elsewhere;
+  };
+  // Star-introducing keywords (`import`/`export`): carry a keyword.control.import
+  // subtype scope AND introduce a namespace `*`. Phase modifiers (which also carry
+  // such a subtype, since `defer` is a deferred-IMPORT marker) are excluded — they
+  // are not star-introducers, they sit BETWEEN the keyword and the `*`.
   const importExportKws = new Set<string>();
   for (const [lit, scopes] of scopeOverrides) {
-    if (scopes.some(s => s.startsWith('keyword.control.import'))) importExportKws.add(lit);
+    if (scopes.some(s => s.startsWith('keyword.control.import')) && !usedAsKeywordOnlyBeforeStar(lit)) importExportKws.add(lit);
   }
   // Does a rule have an alternative whose first item is the `*` literal? (e.g.
   // `import` → ImportClause, whose namespace branch begins `'*' 'as' Ident`.)
@@ -4785,6 +4817,27 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
     return false;
   };
   const starAllKws = new Set<string>();   // import/export keywords that introduce a namespace `*`
+  // Does an alternative begin `[K, '*', …]` where K is a keyword literal? (the
+  // import phase-modifier shape — `defer * as ns`). Returns each such K reached
+  // directly or through a rule-ref (e.g. `import ImportClause`, an ImportClause alt
+  // being `['defer','*','as',Ident]`).
+  const phaseStarKws = (refName: string, seen: Set<string> = new Set()): string[] => {
+    if (seen.has(refName)) return [];
+    seen.add(refName);
+    const rule = grammar.rules.find(r => r.name === refName);
+    if (!rule) return [];
+    const out: string[] = [];
+    for (const alt of expandAlts(rule.body)) {
+      const head = alt[0], next = alt[1];
+      if (head?.type === 'literal' && isKeywordLiteral(head.value) && next?.type === 'literal' && next.value === '*') out.push(head.value);
+      else if (head?.type === 'ref') out.push(...phaseStarKws(head.name, seen));
+    }
+    return out;
+  };
+  // Map each star-introducing import keyword to the phase modifiers that may sit
+  // between it and the `*` (e.g. `import` → [`defer`]). `export`'s `*` takes no
+  // modifier, so its entry stays empty.
+  const phaseModsByKw = new Map<string, string[]>();
   {
     const walk = (e: RuleExpr | undefined): void => {
       if (!e) return;
@@ -4797,6 +4850,18 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
           if ((b.type === 'literal' && b.value === '*') || (b.type === 'ref' && ruleStartsWithStar(b.name))) {
             starAllKws.add(a.value);
           }
+          // Phase modifier between the keyword and `*` (`import defer * as ns`):
+          // a `[K,'*']`-headed alt reached through the next rule-ref, K used as a
+          // keyword ONLY before `*`.
+          if (b.type === 'ref') {
+            const mods = phaseStarKws(b.name).filter(usedAsKeywordOnlyBeforeStar);
+            if (mods.length) {
+              starAllKws.add(a.value);
+              const cur = phaseModsByKw.get(a.value) ?? [];
+              for (const m of mods) if (!cur.includes(m)) cur.push(m);
+              phaseModsByKw.set(a.value, cur);
+            }
+          }
         }
         for (const item of alt) {
           if (item.type === 'quantifier' || item.type === 'group') walk(item.body);
@@ -4806,17 +4871,51 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
     };
     for (const rule of grammar.rules) walk(rule.body);
   }
+  // Collect the phase modifiers so the flat keyword match (section 5) can exclude
+  // them — they are scoped here, positionally, instead.
+  const phaseModifierKws = new Set<string>([...phaseModsByKw.values()].flat());
   if (starAllKws.size > 0) {
     // Keyword keeps the scope it carries elsewhere (read from the scope map), so
-    // capture 1 is not hardcoded to a specific scope string.
+    // the keyword capture is not hardcoded to a specific scope string.
     const kwScope = getScope(scopeOverrides, [...starAllKws][0]) ?? 'keyword.control.import';
-    repository['import-export-all'] = {
-      match: `\\b(${[...starAllKws].map(escapeRegex).join('|')})\\s+(\\*)`,
-      captures: {
-        '1': { name: `${kwScope}.${langName}` },
-        '2': { name: `constant.language.import-export-all.${langName}` },
-      },
-    };
+    if (phaseModifierKws.size === 0) {
+      // No phase modifier in this language (e.g. JS, or TS without `import defer`):
+      // emit the original flat keyword→`*` match verbatim — byte-identical output.
+      repository['import-export-all'] = {
+        match: `\\b(${[...starAllKws].map(escapeRegex).join('|')})\\s+(\\*)`,
+        captures: {
+          '1': { name: `${kwScope}.${langName}` },
+          '2': { name: `constant.language.import-export-all.${langName}` },
+        },
+      };
+    } else {
+      // A phase modifier exists (`import defer * as ns`). One branch per
+      // star-introducing keyword: a keyword that admits a modifier emits
+      // `(import)\s+(?:(defer)\s+)?(\*)`; a bare one stays `(export)\s+(\*)`.
+      // Capture groups number across branches in open-paren order, assigned as the
+      // branches are built. The phase modifier is scoped from the map (a
+      // keyword.control.import subtype), never a hardcoded word.
+      const phaseScope = getScope(scopeOverrides, [...phaseModifierKws][0]) ?? kwScope;
+      const captures: Record<string, { name: string }> = {};
+      const branches: string[] = [];
+      let g = 0;
+      for (const kw of starAllKws) {
+        const mods = phaseModsByKw.get(kw) ?? [];
+        const kwG = ++g; captures[String(kwG)] = { name: `${kwScope}.${langName}` };
+        let branch = `(${escapeRegex(kw)})\\s+`;
+        if (mods.length) {
+          const modG = ++g; captures[String(modG)] = { name: `${phaseScope}.${langName}` };
+          branch += `(?:(${mods.map(escapeRegex).join('|')})\\s+)?`;
+        }
+        const starG = ++g; captures[String(starG)] = { name: `constant.language.import-export-all.${langName}` };
+        branch += `(\\*)`;
+        branches.push(branch);
+      }
+      repository['import-export-all'] = {
+        match: `\\b(?:${branches.join('|')})`,
+        captures,
+      };
+    }
     topPatterns.push({ include: '#import-export-all' });
   }
 
@@ -5322,7 +5421,11 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
       // Drop keywords whose keyword role is owned by a dedicated declaration context
       // (e.g. `constructor` → #constructor-declaration in class bodies). They double
       // as identifiers everywhere else, so the flat match must not paint them.
-      const globalKws = kws.filter(k => !alwaysBeforeString(k) && !ctxOpSet.has(k) && !ctxModSet.has(k) && !contextDeclaredKws.has(k));
+      // Phase modifiers (`defer`, only ever before the namespace `*`) are likewise
+      // scoped positionally by #import-export-all — never in the flat match, which
+      // would mis-paint their ordinary-identifier uses (`const defer`, `defer()`,
+      // `import defer from "m"`).
+      const globalKws = kws.filter(k => !alwaysBeforeString(k) && !ctxOpSet.has(k) && !ctxModSet.has(k) && !contextDeclaredKws.has(k) && !phaseModifierKws.has(k));
       if (globalKws.length > 0) {
         repository[key] = {
           match: `\\b(${globalKws.map(escapeRegex).join('|')})\\b`,

test/issue-cases.ts

@@ -745,16 +745,14 @@ export const tests: TestCase[] = [
     ],
   },
   // TS 5.9 `import defer * as ns` (deferred-import, valid in tsc 5.9.3): the `defer` modifier should
-  // be a keyword. A PROVEN both-miss: `defer` is a CONTEXTUAL keyword the grammar's vocabulary does
-  // not include — the CFG (like the official, and like TS before 5.9) parses it as an ordinary
-  // binding identifier, so both grammars scope it variable.other.readwrite[.alias]. The agnostic
-  // generator scopes a word as a keyword only when the grammar SAYS it is one; making `defer` a
-  // keyword ONLY in `import defer *` (it stays a valid identifier in `const defer`, `import defer
-  // from`) means modeling brand-new deferred-import syntax with a position-only keyword — neither
-  // grammar does. Both miss new syntax.
+  // be a keyword. The official grammar still misses it (its vocabulary has no `defer`, so it scopes
+  // it variable.other.readwrite[.alias]). Monogram MODELS the deferred-import production in the CFG
+  // (`['defer','*','as',Ident]` in ImportClause) and marks `defer` a keyword.control.import.phase —
+  // and the agnostic generator scopes it as a keyword ONLY in phase-modifier position (immediately
+  // before the namespace `*`, via the import-export-all pattern), so `defer` stays an ordinary
+  // identifier everywhere else (`const defer = 1`, `defer()`, `import defer from "m"`). A Monogram win.
   {
     label: '#1058: `import defer` should scope `defer` as a keyword',
-    monoGap: true,
     input: `import defer * as ns from "x";`,
     checks: [
       { text: 'defer', scope: 'keyword' },

tree-sitter/typescript/grammar.js

@@ -213,7 +213,7 @@ module.exports = grammar({
 
     enum_member: $ => seq($.member_name, optional(seq("=", $.expr))),
 
-    import_clause: $ => choice(seq($.ident, optional(seq(",", choice(seq("{", optional(seq($.import_specifier, repeat(seq(",", $.import_specifier)), optional(","))), "}"), seq("*", "as", $.ident))))), seq("{", optional(seq($.import_specifier, repeat(seq(",", $.import_specifier)), optional(","))), "}"), seq("*", "as", $.ident)),
+    import_clause: $ => choice(seq("defer", "*", "as", $.ident), seq($.ident, optional(seq(",", choice(seq("{", optional(seq($.import_specifier, repeat(seq(",", $.import_specifier)), optional(","))), "}"), seq("*", "as", $.ident))))), seq("{", optional(seq($.import_specifier, repeat(seq(",", $.import_specifier)), optional(","))), "}"), seq("*", "as", $.ident)),
 
     import_specifier: $ => seq($.ident, optional(seq("as", $.ident))),
 

tree-sitter/typescript/queries/highlights.scm

@@ -77,7 +77,9 @@
 [
   "constructor" "function" "=>"
 ] @keyword.function
-"import" @keyword.import
+[
+  "import" "defer"
+] @keyword.import
 [
   "else" "if"
 ] @keyword.conditional

tree-sitter/typescriptreact/grammar.js

@@ -213,7 +213,7 @@ module.exports = grammar({
 
     enum_member: $ => seq($.member_name, optional(seq("=", $.expr))),
 
-    import_clause: $ => choice(seq($.ident, optional(seq(",", choice(seq("{", optional(seq($.import_specifier, repeat(seq(",", $.import_specifier)), optional(","))), "}"), seq("*", "as", $.ident))))), seq("{", optional(seq($.import_specifier, repeat(seq(",", $.import_specifier)), optional(","))), "}"), seq("*", "as", $.ident)),
+    import_clause: $ => choice(seq("defer", "*", "as", $.ident), seq($.ident, optional(seq(",", choice(seq("{", optional(seq($.import_specifier, repeat(seq(",", $.import_specifier)), optional(","))), "}"), seq("*", "as", $.ident))))), seq("{", optional(seq($.import_specifier, repeat(seq(",", $.import_specifier)), optional(","))), "}"), seq("*", "as", $.ident)),
 
     import_specifier: $ => seq($.ident, optional(seq("as", $.ident))),
 

tree-sitter/typescriptreact/queries/highlights.scm

@@ -78,7 +78,9 @@
 [
   "constructor" "function" "=>"
 ] @keyword.function
-"import" @keyword.import
+[
+  "import" "defer"
+] @keyword.import
 [
   "else" "if"
 ] @keyword.conditional

typescript.monarch.json

@@ -426,6 +426,7 @@
             "constructor": "keyword",
             "override": "keyword",
             "accessor": "keyword",
+            "defer": "keyword",
             "delete": "operator",
             "string": "keyword",
             "number": "keyword",
@@ -852,6 +853,10 @@
               "token": "keyword",
               "switchTo": "@root"
             },
+            "defer": {
+              "token": "keyword",
+              "switchTo": "@root"
+            },
             "delete": {
               "token": "operator",
               "switchTo": "@root"
@@ -1180,6 +1185,7 @@
             "constructor": "keyword",
             "override": "keyword",
             "accessor": "keyword",
+            "defer": "keyword",
             "delete": "operator",
             "string": "keyword",
             "number": "keyword",