feat: add lexer helpers (lexeme/trivia)

Author: ClaudiuCeia

README.md

@@ -115,6 +115,9 @@ if (!result.success) console.error(formatErrorStack(result));
 If you want the deeper explanations (recursion patterns, `createLanguage`, error
 handling, `cut` vs `context`, and `any` vs `furthest`), see `docs/guide.md`.
 
+The guide also covers the optional lexer layer (`lexeme`, `symbol`, `keyword`,
+`createLexer`) for trivia/comments.
+
 ## License
 
 MIT © [Claudiu Ceia](https://github.com/ClaudiuCeia)

docs/guide.md

@@ -217,3 +217,77 @@ if (res.success) {
   console.log(formatTraceTable(tr.rows()));
 }
 ```
+
+## Lexer layer (optional)
+
+For larger grammars, it's common to separate "lexing" (whitespace/comments and
+token-like units) from higher-level structure, otherwise every rule ends up
+sprinkling `space()`/`regex()`/`optional(...)`.
+
+This repo includes a minimal lexer layer that consumes trailing trivia and drops
+it from the output.
+
+Exports are in `src/lexer.ts`:
+
+- `defaultTrivia()` skips whitespace plus `//` and `/* ... */` comments
+- `lexeme(p, trivia?)` runs `p` and then consumes trailing trivia
+- `symbol("...")` is `lexeme(str("..."))`
+- `keyword("if")` like `symbol`, but enforces an identifier boundary (won't
+  match `ifx`)
+- `createLexer({ trivia? })` builds a small helper object around your trivia
+  policy
+
+Example:
+
+```ts
+import { any, createLexer, map, seq } from "@claudiu-ceia/combine";
+import { eof, int } from "@claudiu-ceia/combine";
+
+const L = createLexer();
+
+const expr = any(
+  L.lexeme(int()),
+  map(seq(L.symbol("("), L.lexeme(int()), L.symbol(")")), ([, n]) => n),
+);
+
+const program = seq(expr, eof());
+```
+
+### With `createLanguageThis`
+
+The lexer layer and `createLanguageThis` are complementary: the lexer keeps
+trivia handling out of your productions, while `createLanguageThis` handles
+mutual recursion without worrying about declaration order.
+
+```ts
+import {
+  any,
+  createLanguageThis,
+  createLexer,
+  map,
+  seq,
+} from "@claudiu-ceia/combine";
+import { eof, int, regex } from "@claudiu-ceia/combine";
+
+const Lx = createLexer();
+
+const Lang = createLanguageThis({
+  Ident() {
+    return Lx.lexeme(regex(/[a-zA-Z_][a-zA-Z0-9_]*/, "identifier"));
+  },
+  Atom() {
+    return any(
+      Lx.lexeme(int()),
+      this.Ident,
+      map(seq(Lx.symbol("("), this.Expr, Lx.symbol(")")), ([, e]) => e),
+    );
+  },
+  Expr() {
+    return this.Atom;
+  },
+  Program() {
+    // Parse leading trivia once at the entry point.
+    return map(seq(Lx.trivia, this.Expr, eof()), ([, e]) => e);
+  },
+});
+```

mod.ts

@@ -4,3 +4,4 @@ export * from "./src/parsers.ts";
 export * from "./src/utility.ts";
 export * from "./src/language.ts";
 export * from "./src/perf.ts";
+export * from "./src/lexer.ts";

src/lexer.ts

@@ -0,0 +1,95 @@
+import { any, not, seq, skipMany, surrounded } from "./combinators.ts";
+import type { Parser } from "./Parser.ts";
+import { regex, space, str } from "./parsers.ts";
+import { map } from "./utility.ts";
+
+export type TriviaParser = Parser<null>;
+
+/**
+ * Match and skip a line comment: `// ...` until (but not including) `\n`.
+ */
+export const lineComment = (): Parser<null> => {
+  return map(regex(/\/\/[^\n]*/, "line comment"), () => null);
+};
+
+/**
+ * Match and skip a block comment (non-greedy).
+ */
+export const blockComment = (): Parser<null> => {
+  return map(regex(/\/\*[\s\S]*?\*\//, "block comment"), () => null);
+};
+
+/**
+ * Default "trivia" parser: whitespace and line/block comments.
+ *
+ * Designed to be used with `lexeme(...)` so most parsers don't need to handle
+ * trivia explicitly.
+ */
+export const defaultTrivia = (): TriviaParser => {
+  const piece = any(space(), lineComment(), blockComment());
+  return skipMany(piece);
+};
+
+/**
+ * Parse `p` and then consume trailing trivia.
+ */
+export const lexeme = <T>(
+  p: Parser<T>,
+  trivia: TriviaParser = defaultTrivia(),
+): Parser<T> => {
+  return map(seq(p, trivia), ([v]) => v);
+};
+
+/**
+ * Parse a fixed string token and consume trailing trivia.
+ */
+export const symbol = (
+  s: string,
+  trivia: TriviaParser = defaultTrivia(),
+): Parser<string> => {
+  return lexeme(str(s), trivia);
+};
+
+const identContinueChar = (): Parser<string> => {
+  return regex(/[a-zA-Z0-9_]/, "identifier char");
+};
+
+/**
+ * Parse a keyword and consume trailing trivia.
+ *
+ * Ensures the keyword is not immediately followed by an identifier character.
+ */
+export const keyword = (
+  s: string,
+  trivia: TriviaParser = defaultTrivia(),
+): Parser<string> => {
+  return lexeme(
+    map(seq(str(s), not(identContinueChar())), ([kw]) => kw),
+    trivia,
+  );
+};
+
+export type Lexer = Readonly<{
+  trivia: TriviaParser;
+  lexeme: <T>(p: Parser<T>) => Parser<T>;
+  symbol: (s: string) => Parser<string>;
+  keyword: (s: string) => Parser<string>;
+  parens: <T>(p: Parser<T>) => Parser<T>;
+}>;
+
+/**
+ * Create a small "lexer layer" around a trivia parser.
+ *
+ * This keeps grammars readable by centralizing whitespace/comment handling.
+ */
+export const createLexer = (opts?: { trivia?: TriviaParser }): Lexer => {
+  const trivia = opts?.trivia ?? defaultTrivia();
+  return {
+    trivia,
+    lexeme: <T>(p: Parser<T>): Parser<T> => lexeme(p, trivia),
+    symbol: (s: string): Parser<string> => symbol(s, trivia),
+    keyword: (s: string): Parser<string> => keyword(s, trivia),
+    parens: <T>(p: Parser<T>): Parser<T> =>
+      surrounded(symbol("(", trivia), p, symbol(")", trivia)),
+  };
+};

tests/lexer.test.ts

@@ -0,0 +1,44 @@
+import { assertEquals } from "@std/assert";
+import {
+  createLexer,
+  defaultTrivia,
+  keyword,
+  lexeme,
+  symbol,
+} from "../src/lexer.ts";
+import { eof, int, str } from "../src/parsers.ts";
+import { seq } from "../src/combinators.ts";
+import { map } from "../src/utility.ts";
+
+Deno.test("lexeme consumes trailing whitespace", () => {
+  const p = seq(lexeme(str("a")), str("b"), eof());
+  const res = p({ text: "a   b", index: 0 });
+  assertEquals(res.success, true);
+});
+
+Deno.test("defaultTrivia consumes line and block comments", () => {
+  const p = seq(
+    symbol("a", defaultTrivia()),
+    symbol("b", defaultTrivia()),
+    eof(),
+  );
+  const res = p({ text: "a // hi\n /* ok */ b", index: 0 });
+  assertEquals(res.success, true);
+});
+
+Deno.test("keyword enforces identifier boundary", () => {
+  const p = seq(keyword("if"), eof());
+  assertEquals(p({ text: "if", index: 0 }).success, true);
+  assertEquals(p({ text: "ifx", index: 0 }).success, false);
+});
+
+Deno.test("createLexer provides a consistent trivia policy", () => {
+  const L = createLexer();
+  const p = map(
+    seq(L.symbol("("), L.lexeme(int()), L.symbol(")"), eof()),
+    ([, n]) => n,
+  );
+  const res = p({ text: "(  12 /*x*/ )", index: 0 });
+  assertEquals(res.success, true);
+  if (res.success) assertEquals(res.value, 12);
+});