wdpr/packages/parser/src/lexer/tokens.ts at b5edd39f5bf3808b79c51b3519b0d5e726236cd9 · r74tech/wdpr · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
import type { Position } from "@wdprlib/ast";

/**
 * Token types for Wikidot markup
 */
export type TokenType =
  // Special
  | "EOF"
  | "TEXT"
  | "IDENTIFIER" // alphanumeric word
  | "NEWLINE"
  | "WHITESPACE"

  // Block delimiters
  | "BLOCK_OPEN" // [[
  | "BLOCK_CLOSE" // ]]
  | "BLOCK_END_OPEN" // [[/

  // Inline formatting
  | "BOLD_MARKER" // **
  | "ITALIC_MARKER" // //
  | "UNDERLINE_MARKER" // __
  | "STRIKE_MARKER" // --
  | "SUPER_MARKER" // ^^
  | "SUB_MARKER" // ,,
  | "MONO_MARKER" // {{
  | "MONO_CLOSE" // }}

  // Special syntax
  | "HEADING_MARKER" // + (at line start)
  | "HR_MARKER" // ---- (at line start)
  | "LIST_BULLET" // * (at line start)
  | "LIST_NUMBER" // # (at line start)
  | "BLOCKQUOTE_MARKER" // > (at line start)
  | "TABLE_MARKER" // || (at line start)
  | "TABLE_HEADER" // ||~ (header cell)
  | "TABLE_LEFT" // ||< (left align)
  | "TABLE_CENTER" // ||= (center align)
  | "TABLE_RIGHT" // ||> (right align)

  // Code blocks
  | "CODE_OPEN" // [[code]]
  | "CODE_CLOSE" // [[/code]]

  // Links
  | "LINK_OPEN" // [[[
  | "LINK_CLOSE" // ]]]
  | "BRACKET_OPEN" // [
  | "BRACKET_CLOSE" // ]
  | "BRACKET_ANCHOR" // [#
  | "BRACKET_STAR" // [*

  // Special characters
  | "PIPE" // |
  | "EQUALS" // =
  | "COLON" // :
  | "SLASH" // /
  | "STAR" // *
  | "HASH" // #
  | "AT" // @
  | "AMPERSAND" // &
  | "BACKSLASH" // \
  | "QUOTED_STRING" // "..."

  // Raw/Escape
  | "RAW_OPEN" // @@
  | "RAW_CLOSE" // @@
  | "RAW_BLOCK_OPEN" // @<
  | "RAW_BLOCK_CLOSE" // >@

  // Color
  | "COLOR_MARKER" // ##

  // Other
  | "UNDERSCORE" // _ (single underscore, for line break)

  // Comment
  | "COMMENT_OPEN" // [!--
  | "COMMENT_CLOSE" // --]

  // Clear float
  | "CLEAR_FLOAT" // ~~~
  | "CLEAR_FLOAT_LEFT" // ~~~<
  | "CLEAR_FLOAT_RIGHT" // ~~~>

  // Double angle (guillemet)
  | "LEFT_DOUBLE_ANGLE" // <<
  | "RIGHT_DOUBLE_ANGLE"; // >> (non-line-start)

/**
 * Token
 */
export interface Token {
  type: TokenType;
  value: string;
  position: Position;
  /** Whether this token appears at the start of a line */
  lineStart: boolean;
}

/**
 * Create a token
 */
export function createToken(
  type: TokenType,
  value: string,
  position: Position,
  lineStart = false,
): Token {
  return { type, value, position, lineStart };
}