fix(lexer): Correct handling of comments (#108)

Author: zharinov

lib/lang/__samples__/groovy-01.out.json

@@ -5,7 +5,9 @@ import type { BracketOption, StatesMap } from './types';
 
 describe('lexer/bracket', () => {
   it('works', () => {
-    const states: StatesMap = { $: { unknown: fallbackRule } };
+    const states: StatesMap = {
+      $: { unknown: { ...fallbackRule, type: 'unknown' } },
+    };
     const bracketOptions: BracketOption[] = [
       { startsWith: '{', endsWith: '}' },
       { startsWith: '[', endsWith: ']' },

lib/lexer/bracket.spec.ts

@@ -1,4 +1,3 @@
-import { sortStateRules } from './rules';
 import type {
   BracketOption,
   StateDefinition,
@@ -41,13 +40,17 @@ export function configBrackets(
     const leftTokenKey = `${tokenKey}$left`;
     const leftTokenRule: StringRule = {
       t: 'string',
+      type: leftTokenKey,
       match: option.startsWith,
+      chunk: option.startsWith,
     };
 
     const rightTokenKey = `${tokenKey}$right`;
     const rightTokenRule: StringRule = {
       t: 'string',
+      type: rightTokenKey,
       match: option.endsWith,
+      chunk: option.endsWith,
     };
 
     bracketDefs[leftTokenKey] = leftTokenRule;
@@ -56,9 +59,9 @@ export function configBrackets(
 
   return {
     ...states,
-    $: sortStateRules({
+    $: {
       ...states.$,
       ...bracketDefs,
-    }),
+    },
   };
 }

lib/lexer/bracket.ts

@@ -6,9 +6,7 @@ import type { CommentOption, StatesMap } from './types';
 
 describe('lexer/comment', () => {
   const states: StatesMap = {
-    $: {
-      unknown: fallbackRule,
-    },
+    $: { unknown: { ...fallbackRule, type: 'unknown' } },
   };
 
   const opts: CommentOption[] = [

lib/lexer/comment.spec.ts

@@ -1,4 +1,3 @@
-import { sortStateRules } from './rules';
 import type {
   CommentOption,
   RegexRule,
@@ -22,26 +21,30 @@ export function configComments(
       const anyChars = '.*?';
       const rule: RegexRule = {
         t: 'regex',
+        type: tokenName,
         match: new RegExp(`${start}${anyChars}$`),
+        chunk: option.startsWith,
       };
       commentRules[tokenName] = rule;
     } else if (option.type === 'multiline-comment') {
       const anyChars = '[^]*?';
       const end = esc(option.endsWith);
       const rule: RegexRule = {
         t: 'regex',
+        type: tokenName,
         match: new RegExp(`${start}${anyChars}${end}`),
         lineBreaks: true,
+        chunk: option.startsWith,
       };
       commentRules[tokenName] = rule;
     }
   });
 
   return {
     ...states,
-    $: sortStateRules({
+    $: {
       ...states.$,
       ...commentRules,
-    }),
+    },
   };
 }

lib/lexer/comment.ts

@@ -3,29 +3,51 @@ import { configBrackets } from './bracket';
 import { configComments } from './comment';
 import { configNumbers } from './number';
 import { configOperators } from './operator';
-import { fallbackRule } from './rules';
+import { createOrderedStateMap, fallbackRule } from './rules';
 import { configStrings } from './string';
 import { configSymbols } from './symbol';
 import { coerceToken } from './token';
-import type { Lexer, LexerConfig, RegexRule, StatesMap, Token } from './types';
+import type {
+  Lexer,
+  LexerConfig,
+  OrderedStatesMap,
+  RegexRule,
+  StatesMap,
+  Token,
+} from './types';
 
 export * from './token';
 export * from './types';
 
-export function configureLexerRules(lexerConfig: LexerConfig): StatesMap {
+export function configureLexerRules(
+  lexerConfig: LexerConfig
+): OrderedStatesMap {
   const whitespace: RegexRule = lexerConfig.joinLines
     ? {
         t: 'regex',
+        type: 'whitespace',
         match: new RegExp(`(?:${lexerConfig.joinLines}\\r?\\n|[ \\t\\r])+`),
         lineBreaks: true,
+        chunk: null,
       }
-    : { t: 'regex', match: /[ \t\r]+/ };
+    : {
+        t: 'regex',
+        type: 'whitespace',
+        match: /[ \t\r]+/,
+        chunk: null,
+      };
 
   let result: StatesMap = {
     $: {
       whitespace,
-      newline: { t: 'regex', match: /\r?\n/, lineBreaks: true },
-      _: fallbackRule,
+      newline: {
+        t: 'regex',
+        type: 'newline',
+        match: /\r?\n/,
+        chunk: null,
+        lineBreaks: true,
+      },
+      _: { ...fallbackRule, type: '_' },
     },
   };
 
@@ -37,12 +59,14 @@ export function configureLexerRules(lexerConfig: LexerConfig): StatesMap {
   result = configBrackets(result, brackets);
   result = configStrings(result, strings);
   result = configNumbers(result, { match: numbers });
-  return result;
+
+  const orderedResult = createOrderedStateMap(result);
+  return orderedResult;
 }
 
 export function createLexer(options: LexerConfig): Lexer {
   const rules = configureLexerRules(options);
-  const mooLexer = mooStates(rules);
+  const mooLexer = mooStates(rules as never);
 
   const result: Lexer = {
     reset(input?: string) {

lib/lexer/index.ts

@@ -5,7 +5,9 @@ import type { NumberOption, StatesMap } from './types';
 
 describe('lexer/number', () => {
   it('works', () => {
-    const states: StatesMap = { $: { unknown: fallbackRule } };
+    const states: StatesMap = {
+      $: { unknown: { ...fallbackRule, type: 'unknown' } },
+    };
     const symbolOption: NumberOption = { match: /[0-9]+/ };
     const rules = configNumbers(states, symbolOption);
     const input = '40+2';

lib/lexer/number.spec.ts

@@ -7,8 +7,13 @@ export function configNumbers(
   return {
     ...states,
     $: {
-      number: { t: 'regex', match },
       ...states.$,
+      number: {
+        t: 'regex',
+        type: 'number',
+        match,
+        chunk: null,
+      },
     },
   };
 }

lib/lexer/number.ts

@@ -5,7 +5,9 @@ import { tokenize } from '../../test/test-utils';
 
 describe('lexer/operator', () => {
   it('works', () => {
-    const states: StatesMap = { $: { unknown: fallbackRule } };
+    const states: StatesMap = {
+      $: { unknown: { ...fallbackRule, type: 'unknown' } },
+    };
     const bracketOptions: OperatorOption[] = ['+'];
     const rules = configOperators(states, bracketOptions);
     const input = '2+2';

lib/lexer/operator.spec.ts

@@ -1,4 +1,3 @@
-import { sortStateRules } from './rules';
 import type {
   OperatorOption,
   StateDefinition,
@@ -13,15 +12,20 @@ export function configOperators(
   const operatorRules: StateDefinition = {};
   operators.map((op, idx) => {
     const tokenName = `op$${idx}`;
-    const tokenRule: StringRule = { t: 'string', match: op };
+    const tokenRule: StringRule = {
+      t: 'string',
+      type: tokenName,
+      match: op,
+      chunk: op,
+    };
     operatorRules[tokenName] = tokenRule;
   });
 
   return {
     ...states,
-    $: sortStateRules({
+    $: {
       ...states.$,
       ...operatorRules,
-    }),
+    },
   };
 }