Merge pull request #12 from cucumber/improve-semantic-tokens

Change semantic tokens

Author: aslakhellesoy

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Fixed
+- Generate semantic tokens that are supported by the Monaco / Visual Studio Code `vs` theme. ([#12](https://github.com/cucumber/language-service/pull/12)).
+
 ## [0.10.0] - 2021-11-08
 ### Removed
 - Move `tree-sitter` functionality to `@cucumber/language-server`

src/service/getGherkinSemanticTokens.ts

@@ -5,12 +5,13 @@ import { SemanticTokens, SemanticTokenTypes } from 'vscode-languageserver-types'
 
 import { parseGherkinDocument } from '../gherkin/parseGherkinDocument.js'
 
+// The default vs theme can only highlight certain tokens. See the list of those tokens in
+// https://microsoft.github.io/monaco-editor/monarch.html
 export const semanticTokenTypes: SemanticTokenTypes[] = [
   SemanticTokenTypes.keyword, // Feature, Scenario, Given etc
   SemanticTokenTypes.parameter, // step parameters
-  SemanticTokenTypes.string, // DocString content and ``` delimiter
-  SemanticTokenTypes.type, // DocString ```type
-  SemanticTokenTypes.class, // @tags
+  SemanticTokenTypes.string, // DocString content and ``` delimiter, table cells (except example table header rows)
+  SemanticTokenTypes.type, // @tags and DocString ```type
   SemanticTokenTypes.variable, // step <placeholder>
   SemanticTokenTypes.property, // examples table header row
 ]
@@ -66,7 +67,7 @@ export function getGherkinSemanticTokens(
 
   const data = walkGherkinDocument<number[]>(gherkinDocument, [], {
     tag(tag, arr) {
-      return makeLocationToken(tag.location, tag.name, SemanticTokenTypes.class, arr)
+      return makeLocationToken(tag.location, tag.name, SemanticTokenTypes.type, arr)
     },
     feature(feature, arr) {
       return makeLocationToken(feature.location, feature.keyword, SemanticTokenTypes.keyword, arr)
@@ -155,7 +156,7 @@ export function getGherkinSemanticTokens(
       return arr
     },
     tableRow(tableRow, arr) {
-      const type = inExamples ? SemanticTokenTypes.property : SemanticTokenTypes.parameter
+      const type = inExamples ? SemanticTokenTypes.property : SemanticTokenTypes.string
       for (const cell of tableRow.cells) {
         arr = makeLocationToken(cell.location, cell.value, type, arr)
       }

test/service/getGherkinSemanticTokens.test.ts

@@ -1,8 +1,13 @@
 import { CucumberExpression, ParameterTypeRegistry } from '@cucumber/cucumber-expressions'
 import assert from 'assert'
-import { SemanticTokens } from 'vscode-languageserver-types'
+import { SemanticTokenTypes, uinteger } from 'vscode-languageserver-types'
 
-import { getGherkinSemanticTokens } from '../../src/service/getGherkinSemanticTokens.js'
+import {
+  getGherkinSemanticTokens,
+  semanticTokenTypes,
+} from '../../src/service/getGherkinSemanticTokens.js'
+
+type TokenWithType = [string, SemanticTokenTypes]
 
 describe('getGherkinSemanticTokens', () => {
   it('creates tokens for keywords', () => {
@@ -35,142 +40,58 @@ Feature: a
     )
 
     const semanticTokens = getGherkinSemanticTokens(gherkinSource, [expression])
-    const expectedSemanticTokens: SemanticTokens = {
-      // See https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#textDocument_semanticTokens
-      // for details about how tokens are encoded
-      data: [
-        1,
-        0,
-        4,
-        4,
-        0, // @foo
-        0,
-        5,
-        4,
-        4,
-        0, // @bar
-        1,
-        0,
-        7,
-        0,
-        0, // Feature
-        4,
-        2,
-        8,
-        0,
-        0, // Scenario
-        1,
-        4,
-        6,
-        0,
-        0, // Given
-        0,
-        13,
-        2,
-        1,
-        0, // 42
-        0,
-        15,
-        5,
-        1,
-        0, // belly
-        1,
-        6,
-        3,
-        2,
-        0, // """
-        0,
-        3,
-        8,
-        3,
-        0, // sometype
-        1,
-        5,
-        5,
-        2,
-        0, // hello
-        1,
-        8,
-        5,
-        2,
-        0, // world
-        1,
-        7,
-        3,
-        2,
-        0, // """
-        1,
-        4,
-        4,
-        0,
-        0, // And
-        1,
-        8,
-        1,
-        1,
-        0, // a
-        0,
-        5,
-        3,
-        1,
-        0, // bbb
-        1,
-        8,
-        2,
-        1,
-        0, // cc
-        0,
-        6,
-        2,
-        1,
-        0, // dd
-        2,
-        2,
-        16,
-        0,
-        0, // Scenario Outline
-        1,
-        4,
-        6,
-        0,
-        0, // Given
-        0,
-        8,
-        5,
-        5,
-        0, // <foo>
-        0,
-        10,
-        5,
-        5,
-        0, // <bar>
-        2,
-        4,
-        8,
-        0,
-        0, // Examples
-        1,
-        8,
-        3,
-        6,
-        0, // foo
-        0,
-        6,
-        3,
-        6,
-        0, // bar
-        1,
-        8,
-        1,
-        1,
-        0, // a
-        0,
-        6,
-        1,
-        1,
-        0, // b
-      ],
-    }
-    assert.deepStrictEqual(semanticTokens, expectedSemanticTokens)
+    const actual = tokenize(gherkinSource, semanticTokens.data)
+    const expected: TokenWithType[] = [
+      ['@foo', SemanticTokenTypes.type],
+      ['@bar', SemanticTokenTypes.type],
+      ['Feature', SemanticTokenTypes.keyword],
+      ['Scenario', SemanticTokenTypes.keyword],
+      ['Given ', SemanticTokenTypes.keyword],
+      ['42', SemanticTokenTypes.parameter],
+      ['belly', SemanticTokenTypes.parameter],
+      ['"""', SemanticTokenTypes.string],
+      ['sometype', SemanticTokenTypes.type],
+      ['hello', SemanticTokenTypes.string],
+      ['world', SemanticTokenTypes.string],
+      ['"""', SemanticTokenTypes.string],
+      ['And ', SemanticTokenTypes.keyword],
+      ['a', SemanticTokenTypes.string],
+      ['bbb', SemanticTokenTypes.string],
+      ['cc', SemanticTokenTypes.string],
+      ['dd', SemanticTokenTypes.string],
+      ['Scenario Outline', SemanticTokenTypes.keyword],
+      ['Given ', SemanticTokenTypes.keyword],
+      ['<foo>', SemanticTokenTypes.variable],
+      ['<bar>', SemanticTokenTypes.variable],
+      ['Examples', SemanticTokenTypes.keyword],
+      ['foo', SemanticTokenTypes.property],
+      ['bar', SemanticTokenTypes.property],
+      ['a', SemanticTokenTypes.string],
+      ['b', SemanticTokenTypes.string],
+    ]
+    assert.deepStrictEqual(actual, expected)
   })
 })
+
+// See https://microsoft.github.io/language-server-protocol/specifications/specification-3-17/#textDocument_semanticTokens
+// for details about how tokens are encoded
+function tokenize(source: string, tokenData: readonly uinteger[]): readonly TokenWithType[] {
+  const result: TokenWithType[] = []
+  const lines = source.split('\n')
+  let lineIndex = 0
+  let start = 0
+  for (let i = 0; i < tokenData.length; i += 5) {
+    const deltaLine = tokenData[i]
+    if (deltaLine > 0) {
+      start = 0
+    }
+    lineIndex += deltaLine
+    start += tokenData[i + 1]
+    const length = tokenData[i + 2]
+    const token = lines[lineIndex].substring(start, start + length)
+    const tokenTypeIndex = tokenData[i + 3]
+    const tokenType = semanticTokenTypes[tokenTypeIndex]
+    result.push([token, tokenType])
+  }
+  return result
+}