fix: support non-English characters (fixes #34)

gajus · Jan 3, 2025 · 00deae8 · 00deae8
1 parent fe42d04
commit 00deae8
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 3 deletions.
diff --git a/src/grammar.ne b/src/grammar.ne
@@ -292,4 +292,4 @@ regex_flags ->
   [gmiyusd]:+ {% d => d[0].join('') %}
 
 unquoted_value ->
-  [a-zA-Z_*?@#$] [a-zA-Z0-9\.\-_*?@#$]:* {% d => d[0] + d[1].join('') %}
+  [a-zA-Z_*?@#$\u0080-\uFFFF] [a-zA-Z0-9\.\-_*?@#$\u0080-\uFFFF]:* {% d => d[0] + d[1].join('') %}
diff --git a/src/grammar.ts b/src/grammar.ts
@@ -771,12 +771,12 @@ const grammar: Grammar = {
     {
       name: 'unquoted_value$ebnf$1',
       postprocess: (d) => d[0].concat([d[1]]),
-      symbols: ['unquoted_value$ebnf$1', /[\w#$*.?@\-]/],
+      symbols: ['unquoted_value$ebnf$1', /[\w#$*.?@\u0080-\uFFFF\-]/],
     },
     {
       name: 'unquoted_value',
       postprocess: (d) => d[0] + d[1].join(''),
-      symbols: [/[#$*?@A-Z_a-z]/, 'unquoted_value$ebnf$1'],
+      symbols: [/[#$*?@A-Z_a-z\u0080-\uFFFF]/, 'unquoted_value$ebnf$1'],
     },
   ],
   ParserStart: 'main',

diff --git a/test/liqe/parse.ts b/test/liqe/parse.ts
@@ -144,6 +144,26 @@ test('foo123', testQuery, {
   type: 'Tag',
 });
 
+test('测试', testQuery, {
+  expression: {
+    location: {
+      end: 2,
+      start: 0,
+    },
+    quoted: false,
+    type: 'LiteralExpression',
+    value: '测试',
+  },
+  field: {
+    type: 'ImplicitField',
+  },
+  location: {
+    end: 2,
+    start: 0,
+  },
+  type: 'Tag',
+});
+
 test('foo with whitespace at the start', (t) => {
   t.deepEqual(parse(' foo'), {
     expression: {
@@ -547,6 +567,41 @@ test('foo:bar123', testQuery, {
   type: 'Tag',
 });
 
+test('foo:测试', testQuery, {
+  expression: {
+    location: {
+      end: 6,
+      start: 4,
+    },
+    quoted: false,
+    type: 'LiteralExpression',
+    value: '测试',
+  },
+  field: {
+    location: {
+      end: 3,
+      start: 0,
+    },
+    name: 'foo',
+    path: ['foo'],
+    quoted: false,
+    type: 'Field',
+  },
+  location: {
+    end: 6,
+    start: 0,
+  },
+  operator: {
+    location: {
+      end: 4,
+      start: 3,
+    },
+    operator: ':',
+    type: 'ComparisonOperator',
+  },
+  type: 'Tag',
+});
+
 // https://github.com/gajus/liqe/issues/18
 // https://github.com/gajus/liqe/issues/19
 test.skip('foo: bar', testQuery, {