Skip to content

fix(lsp): prevent negative deltaStart overflow in multi-line semantic token encoding #2095

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
flohessling wants to merge 1 commit into
base: main
Choose a base branch
from
+88 −6
Open

fix(lsp): prevent negative deltaStart overflow in multi-line semantic token encoding #2095

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions internal/lsp/token_encoder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ type TokenEncoder struct {

// lastEncodedTokenIdx tracks index of the last encoded token
// so we can account for any skipped tokens in calculating diff
lastEncodedTokenIdx int
lastEncodedTokenIdx int
lastEncodedLine int // actual last emitted line (0-indexed)
lastEncodedStartChar int // actual last emitted start char (0-indexed)
}

func (te *TokenEncoder) Encode() []uint32 {
Expand Down Expand Up @@ -54,13 +56,12 @@ func (te *TokenEncoder) encodeTokenOfIndex(i int) []uint32 {

tokenLineDelta := token.Range.End.Line - token.Range.Start.Line

previousLine := 0
previousLine := te.lastEncodedLine
previousStartChar := 0
if i > 0 {
previousLine = te.Tokens[te.lastEncodedTokenIdx].Range.End.Line - 1
currentLine := te.Tokens[i].Range.End.Line - 1
if currentLine == previousLine {
previousStartChar = te.Tokens[te.lastEncodedTokenIdx].Range.Start.Column - 1
previousStartChar = te.lastEncodedStartChar
}
}

Expand All @@ -76,6 +77,8 @@ func (te *TokenEncoder) encodeTokenOfIndex(i int) []uint32 {
uint32(tokenTypeIdx),
uint32(modifierBitMask),
}...)
te.lastEncodedLine = token.Range.Start.Line - 1
te.lastEncodedStartChar = token.Range.Start.Column - 1
} else {
// Add entry for each line of a multiline token
for tokenLine := token.Range.Start.Line - 1; tokenLine <= token.Range.End.Line-1; tokenLine++ {
Expand Down Expand Up @@ -103,6 +106,8 @@ func (te *TokenEncoder) encodeTokenOfIndex(i int) []uint32 {

previousLine = tokenLine
}
te.lastEncodedLine = token.Range.End.Line - 1
te.lastEncodedStartChar = 0
}

te.lastEncodedTokenIdx = i
Expand Down Expand Up @@ -156,7 +161,9 @@ func (te *TokenEncoder) resolveTokenType(token lang.SemanticToken) (semtok.Token
return "", false
}

func (te *TokenEncoder) resolveTokenModifiers(tokModifiers []lang.SemanticTokenModifier) semtok.TokenModifiers {
func (te *TokenEncoder) resolveTokenModifiers(
tokModifiers []lang.SemanticTokenModifier,
) semtok.TokenModifiers {
modifiers := make(semtok.TokenModifiers, 0)

for _, modifier := range tokModifiers {
Expand All @@ -180,7 +187,9 @@ func (te *TokenEncoder) resolveTokenModifiers(tokModifiers []lang.SemanticTokenM
return modifiers
}

func (te *TokenEncoder) firstSupportedTokenType(tokenTypes ...semtok.TokenType) (semtok.TokenType, bool) {
func (te *TokenEncoder) firstSupportedTokenType(
tokenTypes ...semtok.TokenType,
) (semtok.TokenType, bool) {
for _, tokenType := range tokenTypes {
if te.tokenTypeSupported(string(tokenType)) {
return tokenType, true
Expand Down
73 changes: 73 additions & 0 deletions internal/lsp/token_encoder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,3 +382,76 @@ func TestTokenEncoder_unsupported(t *testing.T) {
expectedData, data)
}
}

func TestTokenEncoder_multiLineTokenFollowedBySameEndLineToken(t *testing.T) {
// Simulates a heredoc with interpolation in its body:
//
// message = <<-EOT
// Some text on first line
// Hello ${var.name} world
// EOT
//
// hcl-lang produces (simplified):
// 1. A multi-line TokenString for the heredoc content before the interpolation
// Start = {Line: 1, Column: 42} (high column — position on the "message = <<-EOT" line)
// End = {Line: 3, Column: 11} (just before ${var.name} on line 3)
// 2. A single-line TokenReferenceStep for "var" at {Line: 3, Column: 11}
// 3. A single-line TokenReferenceStep for "name" at {Line: 3, Column: 15}
//
// The bug: when encoding "var" (token 2), the encoder checks:
// previousLine = Tokens[0].Range.End.Line - 1 = 3 - 1 = 2
// currentLine = Tokens[1].Range.End.Line - 1 = 3 - 1 = 2
// currentLine == previousLine → previousStartChar = Tokens[0].Range.Start.Column - 1 = 41
// deltaStartChar = var.Start.Column - 1 - 41 = 10 - 41 = -31 → uint32 overflow!
bytes := []byte(
" message = <<-EOT\n Some text on first line\n Hello ${var.name} world\n EOT\n",
)
te := &TokenEncoder{
Lines: source.MakeSourceLines("test.tf", bytes),
Tokens: []lang.SemanticToken{
// Multi-line string: starts at high column on line 1, ends on line 3
{
Type: lang.TokenString,
Range: hcl.Range{
Filename: "test.tf",
Start: hcl.Pos{Line: 1, Column: 42, Byte: 41},
End: hcl.Pos{Line: 3, Column: 11, Byte: 57},
},
},
// "var" reference at line 3, column 11
{
Type: lang.TokenReferenceStep,
Range: hcl.Range{
Filename: "test.tf",
Start: hcl.Pos{Line: 3, Column: 11, Byte: 57},
End: hcl.Pos{Line: 3, Column: 14, Byte: 60},
},
},
// "name" reference at line 3, column 15
{
Type: lang.TokenReferenceStep,
Range: hcl.Range{
Filename: "test.tf",
Start: hcl.Pos{Line: 3, Column: 15, Byte: 61},
End: hcl.Pos{Line: 3, Column: 19, Byte: 65},
},
},
},
ClientCaps: protocol.SemanticTokensClientCapabilities{
TokenTypes: serverTokenTypes.AsStrings(),
TokenModifiers: serverTokenModifiers.AsStrings(),
},
}
data := te.Encode()
// Check for uint32 overflow in any deltaStart or deltaLine value.
// Values > 2^31 indicate a negative int was cast to uint32.
for i := 0; i < len(data); i += 5 {
if data[i] > 1<<31 {
t.Fatalf("token at data index %d has overflowed deltaLine: %d", i, data[i])
}
if data[i+1] > 1<<31 {
t.Fatalf("token at data index %d has overflowed deltaStart: %d", i, data[i+1])
}
}
t.Logf("encoded data: %v", data)
}