Skip to content
This repository was archived by the owner on Jan 6, 2026. It is now read-only.

Commit 34ce50d

Browse files
authored
Merge pull request #10 from yehorkardash/language-overlay
fix: Use language overlay and highlight functions
1 parent 689a186 commit 34ce50d

10 files changed

Lines changed: 189 additions & 120 deletions

package-lock.json

Lines changed: 12 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@
3333
"@lezer/lr": "^1.0.0"
3434
},
3535
"devDependencies": {
36-
"@codemirror/buildhelper": "^0.1.5"
36+
"@codemirror/buildhelper": "^0.1.5",
37+
"@codemirror/text": "^0.19.6"
3738
},
3839
"repository": {
3940
"type": "git",

src/expressions.grammar

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
@top Program { entity* }
2+
3+
entity { Plaintext | Resolvable }
4+
5+
@tokens {
6+
Plaintext { ![{] Plaintext? | "{" (@eof | ![{] Plaintext?) }
7+
8+
OpenMarker[closedBy="CloseMarker"] { "{{" }
9+
10+
CloseMarker[openedBy="OpenMarker"] { "}}" }
11+
12+
Resolvable {
13+
OpenMarker resolvableChar* CloseMarker
14+
}
15+
16+
resolvableChar { unicodeChar | "}" ![}] | "\\}}" }
17+
18+
unicodeChar { $[\u0000-\u007C] | $[\u007E-\u20CF] | $[\u2700-\u27BF] | $[\u{1F300}-\u{1FAF8}] | $[\u4E00-\u9FFF] }
19+
}
20+
21+
@detectDelim

src/expressions.grammar.d.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
import {LRParser} from "@lezer/lr"
2+
export declare const parser: LRParser

src/sql.grammar

Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
Statement { element+ }?
66
}
77

8-
@skip { LineComment | BlockComment }
8+
@skip { Whitespace | LineComment | BlockComment }
99

1010
element {
1111
String |
@@ -19,41 +19,20 @@ element {
1919
Builtin |
2020
SpecialVar |
2121
CompositeIdentifier {
22-
Dot? (QuotedIdentifier | Identifier | SpecialVar | Resolvable) (!dot Dot (QuotedIdentifier | Identifier | SpecialVar | Resolvable))+
22+
Dot? (QuotedIdentifier | Identifier | SpecialVar) (!dot Dot (QuotedIdentifier | Identifier | SpecialVar))+
2323
} |
2424
Keyword |
2525
Type |
2626
Operator |
2727
Punctuation |
2828
Parens { ParenL element* ParenR } |
2929
Braces { BraceL element* BraceR } |
30-
Brackets { BracketL element* BracketR } |
31-
orphanWrappedResolvable {
32-
(OrphanSingleQuote Whitespace? Resolvable Whitespace? OrphanSingleQuote)
33-
} |
34-
Resolvable |
35-
Whitespace
36-
}
37-
38-
@tokens {
39-
OrphanSingleQuote { "'" }
40-
41-
Resolvable { ResolvableStart resolvableContent* ResolvableEnd }
42-
43-
ResolvableStart[closedBy="ResolvableEnd"] { "{{" }
44-
45-
ResolvableEnd[openedBy="ResolvableStart"] { "}}" }
46-
47-
resolvableContent {
48-
unicodeChar |
49-
"}" ![}] |
50-
"\\}}"
51-
}
52-
53-
unicodeChar { $[\u0000-\u007C] | $[\u007E-\u06FF] }
30+
Brackets { BracketL element* BracketR }
31+
Function
5432
}
5533

5634
@external tokens tokens from "./tokens" {
35+
Whitespace
5736
LineComment
5837
BlockComment
5938
String
@@ -78,7 +57,7 @@ element {
7857
Bits
7958
Bytes
8059
Builtin
81-
Whitespace
60+
Function
8261
}
8362

8463
@detectDelim

src/sql.grammar.terms.d.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ export const Whitespace: number, LineComment: number, BlockComment: number,
22
String: number, Number: number, Bool: number, Null: number,
33
ParenL: number, ParenR: number, BraceL: number, BraceR: number, BracketL: number, BracketR: number, Semi: number, Dot: number,
44
Operator: number, Punctuation: number, SpecialVar: number, Identifier: number, QuotedIdentifier: number,
5-
Keyword: number, Type: number, Builtin: number, Bits: number, Bytes: number
5+
Keyword: number, Type: number, Builtin: number, Bits: number, Bytes: number, Function: number

src/sql.ts

Lines changed: 90 additions & 50 deletions
Large diffs are not rendered by default.

src/tokens.ts

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import {ExternalTokenizer, InputStream} from "@lezer/lr"
22
import {LineComment, BlockComment, String as StringToken, Number, Bits, Bytes, Bool, Null,
33
ParenL, ParenR, BraceL, BraceR, BracketL, BracketR, Semi, Dot,
44
Operator, Punctuation, SpecialVar, Identifier, QuotedIdentifier,
5-
Keyword, Type, Builtin, Whitespace} from "./sql.grammar.terms"
5+
Keyword, Type, Builtin, Whitespace, Function} from "./sql.grammar.terms"
66

77
const enum Ch {
88
Newline = 10,
@@ -118,13 +118,14 @@ function inString(ch: number, str: string) {
118118

119119
const Space = " \t\r\n"
120120

121-
function keywords(keywords: string, types: string, builtin?: string) {
121+
function keywords(keywords: string, types: string, builtin?: string, functions?: string) {
122122
let result: {[name: string]: number} = Object.create(null)
123123
result["true"] = result["false"] = Bool
124124
result["null"] = result["unknown"] = Null
125125
for (let kw of keywords.split(" ")) if (kw) result[kw] = Keyword
126126
for (let tp of types.split(" ")) if (tp) result[tp] = Type
127127
for (let kw of (builtin || "").split(" ")) if (kw) result[kw] = Builtin
128+
for (let fn of (functions || "").split(" ")) if (fn) result[fn] = Function
128129
return result
129130
}
130131

@@ -145,7 +146,8 @@ export interface Dialect {
145146
}
146147

147148
export const SQLTypes = "array binary bit boolean char character clob date decimal double float int integer interval large national nchar nclob numeric object precision real smallint time timestamp varchar varying "
148-
export const SQLKeywords = "absolute action add after all allocate alter and any are as asc assertion at authorization before begin between both breadth by call cascade cascaded case cast catalog check close collate collation column commit condition connect connection constraint constraints constructor continue corresponding count create cross cube current current_date current_default_transform_group current_transform_group_for_type current_path current_role current_time current_timestamp current_user cursor cycle data day deallocate declare default deferrable deferred delete depth deref desc describe descriptor deterministic diagnostics disconnect distinct do domain drop dynamic each else elseif end end-exec equals escape except exception exec execute exists exit external fetch first for foreign found from free full function general get global go goto grant group grouping handle having hold hour identity if immediate in indicator initially inner inout input insert intersect into is isolation join key language last lateral leading leave left level like limit local localtime localtimestamp locator loop map match method minute modifies module month names natural nesting new next no none not of old on only open option or order ordinality out outer output overlaps pad parameter partial path prepare preserve primary prior privileges procedure public read reads recursive redo ref references referencing relative release repeat resignal restrict result return returns revoke right role rollback rollup routine row rows savepoint schema scroll search second section select session session_user set sets signal similar size some space specific specifictype sql sqlexception sqlstate sqlwarning start state static system_user table temporary then timezone_hour timezone_minute to trailing transaction translation treat trigger under undo union unique unnest until update usage user using value values view when whenever where while with without work write year zone "
149+
export const SQLFunctions = "abs absolute case check cast concat coalesce cube collate count current_date current_path current_role current_time current_timestamp current_user day exists grouping hour localtime localtimestamp minute month second trim session_user size system_user treat unnest user year equals lower upper pow floor ceil exp log ifnull min max avg sum sqrt round "
150+
export const SQLKeywords = "action add after all allocate alter and any are as asc assertion at authorization before begin between both breadth by call cascade cascaded catalog close collation column commit condition connect connection constraint constraints constructor continue corresponding create cross current current_default_transform_group current_transform_group_for_type cursor cycle data deallocate declare default deferrable deferred delete depth deref desc describe descriptor deterministic diagnostics disconnect distinct do domain drop dynamic each else elseif end end-exec equals escape except exception exec execute exit external fetch first for foreign found from free full function general get global go goto grant group handle having hold identity if immediate in indicator initially inner inout input insert intersect into is isolation join key language last lateral leading leave left level like limit local locator loop map match method modifies module names natural nesting new next no none not of old on only open option or order ordinality out outer output overlaps pad parameter partial path prepare preserve primary prior privileges procedure public read reads recursive redo ref references referencing relative release repeat resignal restrict result return returns revoke right role rollback rollup routine row rows savepoint schema scroll search section select session set sets signal similar some space specific specifictype sql sqlexception sqlstate sqlwarning start state static table temporary then timezone_hour timezone_minute to trailing transaction translation trigger under undo union unique until update usage using value values view when whenever where while with without work write zone "
149151

150152
const defaults: Dialect = {
151153
backslashEscapes: false,
@@ -160,14 +162,14 @@ const defaults: Dialect = {
160162
operatorChars: "*+\-%<>!=&|~^/",
161163
specialVar: "?",
162164
identifierQuotes: '"',
163-
words: keywords(SQLKeywords, SQLTypes)
165+
words: keywords(SQLKeywords, SQLTypes, "", SQLFunctions)
164166
}
165167

166-
export function dialect(spec: Partial<Dialect>, kws?: string, types?: string, builtin?: string): Dialect {
168+
export function dialect(spec: Partial<Dialect>, kws?: string, types?: string, builtin?: string, functions?: string): Dialect {
167169
let dialect = {} as Dialect
168170
for (let prop in defaults)
169171
(dialect as any)[prop] = ((spec.hasOwnProperty(prop) ? spec : defaults) as any)[prop]
170-
if (kws) dialect.words = keywords(kws, types || "", builtin)
172+
if (kws) dialect.words = keywords(kws, types || "", builtin, functions)
171173
return dialect
172174
}
173175

test/test-complete.ts

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,18 @@
11
import {EditorState} from "@codemirror/state"
22
import {CompletionContext, CompletionResult, CompletionSource} from "@codemirror/autocomplete"
3-
import {schemaCompletionSource, PostgreSQL, MySQL, SQLConfig} from "@n8n/codemirror-lang-sql"
3+
import {schemaCompletionSource, PostgreSQL, MySQL, SQLConfig, getParser} from "@n8n/codemirror-lang-sql"
44
import ist from "ist"
5+
import { LanguageSupport } from '@codemirror/language'
56

67
function get(doc: string, conf: SQLConfig & {explicit?: boolean} = {}) {
7-
let cur = doc.indexOf("|"), dialect = conf.dialect || PostgreSQL
8+
let cur = doc.indexOf("|");
9+
const dialect = conf.dialect || PostgreSQL
810
doc = doc.slice(0, cur) + doc.slice(cur + 1)
11+
912
let state = EditorState.create({
1013
doc,
1114
selection: {anchor: cur},
12-
extensions: [dialect, dialect.language.data.of({
15+
extensions: [dialect, dialect.sqlLanguage.data.of({
1316
autocomplete: schemaCompletionSource(Object.assign({dialect}, conf))
1417
})]
1518
})

test/test-tokens.ts

Lines changed: 41 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,69 @@
11
import ist from "ist"
22
import {PostgreSQL, MySQL, SQLDialect} from "@n8n/codemirror-lang-sql"
3+
import { LRParser } from '@lezer/lr'
34

45
const mysqlTokens = MySQL.language
56
const postgresqlTokens = PostgreSQL.language
67
const bigQueryTokens = SQLDialect.define({
78
treatBitsAsBytes: true
89
}).language
910

11+
const parse = (parser: LRParser, input: string) => {
12+
const tree = parser.parse(input);
13+
const props: Record<string, {tree: unknown}> = (tree as any).props;
14+
const key = Object.keys(props)[0];
15+
return props[key].tree;
16+
}
17+
18+
const parseMixed = (parser: LRParser, input: string) => {
19+
return parser.parse(input);
20+
}
21+
1022
describe("Parse MySQL tokens", () => {
1123
const parser = mysqlTokens.parser
1224

1325
it("parses quoted bit-value literals", () => {
14-
ist(parser.parse("SELECT b'0101'"), 'Script(Statement(Keyword,Whitespace,Bits))')
26+
ist(parse(parser, "SELECT b'0101'"), 'Script(Statement(Keyword,Whitespace,Bits))')
1527
})
1628

1729
it("parses unquoted bit-value literals", () => {
18-
ist(parser.parse("SELECT 0b01"), 'Script(Statement(Keyword,Whitespace,Bits))')
30+
ist(parse(parser, "SELECT 0b01"), 'Script(Statement(Keyword,Whitespace,Bits))')
1931
})
2032
})
2133

2234
describe("Parse PostgreSQL tokens", () => {
2335
const parser = postgresqlTokens.parser
2436

2537
it("parses quoted bit-value literals", () => {
26-
ist(parser.parse("SELECT b'0101'"), 'Script(Statement(Keyword,Whitespace,Bits))')
38+
ist(parse(parser, "SELECT b'0101'"), 'Script(Statement(Keyword,Whitespace,Bits))')
2739
})
2840

2941
it("parses quoted bit-value literals", () => {
30-
ist(parser.parse("SELECT B'0101'"), 'Script(Statement(Keyword,Whitespace,Bits))')
42+
ist(parse(parser, "SELECT B'0101'"), 'Script(Statement(Keyword,Whitespace,Bits))')
3143
})
3244

3345
it("parses double dollar quoted Whitespace literals", () => {
34-
ist(parser.parse("SELECT $$hello$$"), 'Script(Statement(Keyword,Whitespace,String))')
46+
ist(parse(parser, "SELECT $$hello$$"), 'Script(Statement(Keyword,Whitespace,String))')
3547
})
3648
})
3749

3850
describe("Parse BigQuery tokens", () => {
3951
const parser = bigQueryTokens.parser
4052

4153
it("parses quoted bytes literals in single quotes", () => {
42-
ist(parser.parse("SELECT b'abcd'"), 'Script(Statement(Keyword,Whitespace,Bytes))')
54+
ist(parse(parser, "SELECT b'abcd'"), 'Script(Statement(Keyword,Whitespace,Bytes))')
4355
})
4456

4557
it("parses quoted bytes literals in double quotes", () => {
46-
ist(parser.parse('SELECT b"abcd"'), 'Script(Statement(Keyword,Whitespace,Bytes))')
58+
ist(parse(parser, 'SELECT b"abcd"'), 'Script(Statement(Keyword,Whitespace,Bytes))')
4759
})
4860

4961
it("parses bytes literals in single quotes", () => {
50-
ist(parser.parse("SELECT b'0101'"), 'Script(Statement(Keyword,Whitespace,Bytes))')
62+
ist(parse(parser, "SELECT b'0101'"), 'Script(Statement(Keyword,Whitespace,Bytes))')
5163
})
5264

5365
it("parses bytes literals in double quotes", () => {
54-
ist(parser.parse('SELECT b"0101"'), 'Script(Statement(Keyword,Whitespace,Bytes))')
66+
ist(parse(parser, 'SELECT b"0101"'), 'Script(Statement(Keyword,Whitespace,Bytes))')
5567
})
5668
})
5769

@@ -60,56 +72,56 @@ describe("Parse n8n resolvables", () => {
6072

6173
it("parses resolvables with dots inside composite identifiers", () => {
6274
ist(
63-
parser.parse("SELECT my_column FROM {{ 'schema' }}.{{ 'table' }}"),
64-
'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,CompositeIdentifier(Resolvable,".",Resolvable)))'
75+
parseMixed(parser, "SELECT my_column FROM {{ 'schema' }}.{{ 'table' }}"),
76+
'Program(Plaintext,Resolvable,Plaintext,Resolvable)'
6577
)
6678
ist(
67-
parser.parse("SELECT my_column FROM {{ 'schema' }}.{{ 'table' }}.{{ 'foo' }}"),
68-
'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,CompositeIdentifier(Resolvable,".",Resolvable,".",Resolvable)))'
79+
parseMixed(parser, "SELECT my_column FROM {{ 'schema' }}.{{ 'table' }}.{{ 'foo' }}"),
80+
'Program(Plaintext,Resolvable,Plaintext,Resolvable,Plaintext,Resolvable)'
6981
)
7082
ist(
71-
parser.parse("SELECT my_column FROM public.{{ 'table' }}"),
72-
'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,CompositeIdentifier(Identifier,".",Resolvable)))'
83+
parseMixed(parser, "SELECT my_column FROM public.{{ 'table' }}"),
84+
'Program(Plaintext,Resolvable)'
7385
)
7486
ist(
75-
parser.parse("SELECT my_column FROM {{ 'schema' }}.users"),
76-
'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,CompositeIdentifier(Resolvable,".",Identifier)))'
87+
parseMixed(parser, "SELECT my_column FROM {{ 'schema' }}.users"),
88+
'Program(Plaintext,Resolvable,Plaintext)'
7789
)
7890
})
7991

8092
it("parses 4-node SELECT variants", () => {
81-
ist(parser.parse("{{ 'SELECT' }} my_column FROM my_table"), 'Script(Statement(Resolvable,Whitespace,Identifier,Whitespace,Keyword,Whitespace,Identifier))')
93+
ist(parseMixed(parser, "{{ 'SELECT' }} my_column FROM my_table"), 'Program(Resolvable,Plaintext)')
8294

83-
ist(parser.parse("SELECT {{ 'my_column' }} FROM my_table"), 'Script(Statement(Keyword,Whitespace,Resolvable,Whitespace,Keyword,Whitespace,Identifier))')
95+
ist(parseMixed(parser, "SELECT {{ 'my_column' }} FROM my_table"), 'Program(Plaintext,Resolvable,Plaintext)')
8496

85-
ist(parser.parse("SELECT my_column {{ 'FROM' }} my_table"), 'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Resolvable,Whitespace,Identifier))')
97+
ist(parseMixed(parser, "SELECT my_column {{ 'FROM' }} my_table"), 'Program(Plaintext,Resolvable,Plaintext)')
8698

87-
ist(parser.parse("SELECT my_column FROM {{ 'my_table' }}"), 'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,Resolvable))')
99+
ist(parseMixed(parser, "SELECT my_column FROM {{ 'my_table' }}"), 'Program(Plaintext,Resolvable)')
88100
})
89101

90102
it("parses 5-node SELECT variants (with semicolon)", () => {
91-
ist(parser.parse("{{ 'SELECT' }} my_column FROM my_table;"), 'Script(Statement(Resolvable,Whitespace,Identifier,Whitespace,Keyword,Whitespace,Identifier,";"))')
103+
ist(parseMixed(parser, "{{ 'SELECT' }} my_column FROM my_table;"), 'Program(Resolvable,Plaintext)')
92104

93-
ist(parser.parse("SELECT {{ 'my_column' }} FROM my_table;"), 'Script(Statement(Keyword,Whitespace,Resolvable,Whitespace,Keyword,Whitespace,Identifier,";"))')
105+
ist(parseMixed(parser, "SELECT {{ 'my_column' }} FROM my_table;"), 'Program(Plaintext,Resolvable,Plaintext)')
94106

95-
ist(parser.parse("SELECT my_column {{ 'FROM' }} my_table;"), 'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Resolvable,Whitespace,Identifier,";"))')
107+
ist(parseMixed(parser, "SELECT my_column {{ 'FROM' }} my_table;"), 'Program(Plaintext,Resolvable,Plaintext)')
96108

97-
ist(parser.parse("SELECT my_column FROM {{ 'my_table' }};"), 'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,Resolvable,";"))')
109+
ist(parseMixed(parser, "SELECT my_column FROM {{ 'my_table' }};"), 'Program(Plaintext,Resolvable,Plaintext)')
98110
})
99111

100112
it("parses single-quoted resolvable with no whitespace", () => {
101-
ist(parser.parse("SELECT my_column FROM '{{ 'my_table' }}';"), 'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,OrphanSingleQuote,Resolvable,OrphanSingleQuote,";"))')
113+
ist(parseMixed(parser, "SELECT my_column FROM '{{ 'my_table' }}';"), 'Program(Plaintext,Resolvable,Plaintext)')
102114
});
103115

104116
it("parses single-quoted resolvable with leading whitespace", () => {
105-
ist(parser.parse("SELECT my_column FROM ' {{ 'my_table' }}';"), 'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,OrphanSingleQuote,Whitespace,Resolvable,OrphanSingleQuote,";"))')
117+
ist(parseMixed(parser, "SELECT my_column FROM ' {{ 'my_table' }}';"), 'Program(Plaintext,Resolvable,Plaintext)')
106118
});
107119

108120
it("parses single-quoted resolvable with trailing whitespace", () => {
109-
ist(parser.parse("SELECT my_column FROM '{{ 'my_table' }} ';"), 'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,OrphanSingleQuote,Resolvable,Whitespace,OrphanSingleQuote,";"))')
121+
ist(parseMixed(parser, "SELECT my_column FROM '{{ 'my_table' }} ';"), 'Program(Plaintext,Resolvable,Plaintext)')
110122
});
111123

112124
it("parses single-quoted resolvable with surrounding whitespace", () => {
113-
ist(parser.parse("SELECT my_column FROM ' {{ 'my_table' }} ';"), 'Script(Statement(Keyword,Whitespace,Identifier,Whitespace,Keyword,Whitespace,OrphanSingleQuote,Whitespace,Resolvable,Whitespace,OrphanSingleQuote,";"))')
125+
ist(parseMixed(parser, "SELECT my_column FROM ' {{ 'my_table' }} ';"), 'Program(Plaintext,Resolvable,Plaintext)')
114126
});
115127
})

0 commit comments

Comments
 (0)