Refactor lexer

Author: antonmedv

file/error.go

@@ -8,6 +8,8 @@ import (
 
 type Error struct {
 	Location
+	Line    int    `json:"line"`
+	Column  int    `json:"column"`
 	Message string `json:"message"`
 	Snippet string `json:"snippet"`
 	Prev    error  `json:"prev"`
@@ -18,12 +20,24 @@ func (e *Error) Error() string {
 }
 
 func (e *Error) Bind(source Source) *Error {
-	if snippet, found := source.Snippet(e.Location.Line); found {
+	e.Line = 1
+	for i, r := range source {
+		if i == e.From {
+			break
+		}
+		if r == '\n' {
+			e.Line++
+			e.Column = 0
+		} else {
+			e.Column++
+		}
+	}
+	if snippet, found := source.Snippet(e.Line); found {
 		snippet := strings.Replace(snippet, "\t", " ", -1)
 		srcLine := "\n | " + snippet
 		var bytes = []byte(snippet)
 		var indLine = "\n | "
-		for i := 0; i < e.Location.Column && len(bytes) > 0; i++ {
+		for i := 0; i < e.Column && len(bytes) > 0; i++ {
 			_, sz := utf8.DecodeRune(bytes)
 			bytes = bytes[sz:]
 			if sz > 1 {
@@ -54,7 +68,7 @@ func (e *Error) Wrap(err error) {
 }
 
 func (e *Error) format() string {
-	if e.Location.Empty() {
+	if e.Snippet == "" {
 		return e.Message
 	}
 	return fmt.Sprintf(

file/location.go

@@ -1,12 +1,6 @@
 package file
 
 type Location struct {
-	Line   int `json:"line"`   // The 1-based line of the location.
-	Column int `json:"column"` // The 0-based column number of the location.
-	From   int `json:"from"`   // The 0-based byte offset from the beginning.
-	To     int `json:"to"`     // The 0-based byte offset to the end.
-}
-
-func (l Location) Empty() bool {
-	return l.Column == 0 && l.Line == 0
+	From int `json:"from"`
+	To   int `json:"to"`
 }

parser/lexer/lexer.go

@@ -3,20 +3,18 @@ package lexer
 import (
 	"fmt"
 	"strings"
-	"unicode/utf8"
 
 	"github.com/expr-lang/expr/file"
 )
 
 func Lex(source file.Source) ([]Token, error) {
 	l := &lexer{
-		input:  source.String(),
+		source: source,
 		tokens: make([]Token, 0),
+		start:  0,
+		end:    0,
 	}
-
-	l.loc = file.Location{Line: 1, Column: 0}
-	l.prev = l.loc
-	l.startLoc = l.loc
+	l.commit()
 
 	for state := root; state != nil; {
 		state = state(l)
@@ -30,34 +28,27 @@ func Lex(source file.Source) ([]Token, error) {
 }
 
 type lexer struct {
-	input      string
+	source     file.Source
 	tokens     []Token
-	start, end int           // current position in input
-	width      int           // last rune width
-	startLoc   file.Location // start location
-	prev, loc  file.Location // prev location of end location, end location
+	backed     bool
+	start, end int
 	err        *file.Error
 }
 
 const eof rune = -1
 
+func (l *lexer) commit() {
+	l.start = l.end
+}
+
 func (l *lexer) next() rune {
-	if l.end >= len(l.input) {
-		l.width = 0
+	l.backed = false
+	if l.end >= len(l.source) {
+		l.end++
 		return eof
 	}
-	r, w := utf8.DecodeRuneInString(l.input[l.end:])
-	l.width = w
-	l.end += w
-
-	l.prev = l.loc
-	if r == '\n' {
-		l.loc.Line++
-		l.loc.Column = 0
-	} else {
-		l.loc.Column++
-	}
-
+	r := l.source[l.end]
+	l.end++
 	return r
 }
 
@@ -68,8 +59,11 @@ func (l *lexer) peek() rune {
 }
 
 func (l *lexer) backup() {
-	l.end -= l.width
-	l.loc = l.prev
+	if l.backed {
+		panic(fmt.Sprintf("lexer: cannot backup twice: %q", l.source[l.start:l.end]))
+	}
+	l.backed = true
+	l.end--
 }
 
 func (l *lexer) emit(t Kind) {
@@ -78,35 +72,35 @@ func (l *lexer) emit(t Kind) {
 
 func (l *lexer) emitValue(t Kind, value string) {
 	l.tokens = append(l.tokens, Token{
-		Location: l.startLoc,
+		Location: file.Location{From: l.start, To: l.end},
 		Kind:     t,
 		Value:    value,
 	})
-	l.start = l.end
-	l.startLoc = l.loc
+	l.commit()
 }
 
 func (l *lexer) emitEOF() {
+	from := l.end - 2
+	if from < 0 {
+		from = 0
+	}
+	to := l.end - 1
+	if to < 0 {
+		to = 0
+	}
 	l.tokens = append(l.tokens, Token{
-		Location: l.prev, // Point to previous position for better error messages.
+		Location: file.Location{From: from, To: to},
 		Kind:     EOF,
 	})
-	l.start = l.end
-	l.startLoc = l.loc
+	l.commit()
 }
 
 func (l *lexer) skip() {
-	l.start = l.end
-	l.startLoc = l.loc
+	l.commit()
 }
 
 func (l *lexer) word() string {
-	return l.input[l.start:l.end]
-}
-
-func (l *lexer) ignore() {
-	l.start = l.end
-	l.startLoc = l.loc
+	return string(l.source[l.start:l.end])
 }
 
 func (l *lexer) accept(valid string) bool {
@@ -132,18 +126,18 @@ func (l *lexer) skipSpaces() {
 }
 
 func (l *lexer) acceptWord(word string) bool {
-	pos, loc, prev := l.end, l.loc, l.prev
+	pos := l.end
 
 	l.skipSpaces()
 
 	for _, ch := range word {
 		if l.next() != ch {
-			l.end, l.loc, l.prev = pos, loc, prev
+			l.end = pos
 			return false
 		}
 	}
 	if r := l.peek(); r != ' ' && r != eof {
-		l.end, l.loc, l.prev = pos, loc, prev
+		l.end = pos
 		return false
 	}
 
@@ -153,8 +147,11 @@ func (l *lexer) acceptWord(word string) bool {
 func (l *lexer) error(format string, args ...any) stateFn {
 	if l.err == nil { // show first error
 		l.err = &file.Error{
-			Location: l.loc,
-			Message:  fmt.Sprintf(format, args...),
+			Location: file.Location{
+				From: l.end - 1,
+				To:   l.end,
+			},
+			Message: fmt.Sprintf(format, args...),
 		}
 	}
 	return nil
@@ -230,6 +227,6 @@ func (l *lexer) scanRawString(quote rune) (n int) {
 		ch = l.next()
 		n++
 	}
-	l.emitValue(String, l.input[l.start+1:l.end-1])
+	l.emitValue(String, string(l.source[l.start+1:l.end-1]))
 	return
 }

parser/lexer/lexer_test.go

@@ -5,10 +5,9 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/expr-lang/expr/file"
 	"github.com/expr-lang/expr/internal/testify/assert"
 	"github.com/expr-lang/expr/internal/testify/require"
-
-	"github.com/expr-lang/expr/file"
 	. "github.com/expr-lang/expr/parser/lexer"
 )
 
@@ -17,6 +16,13 @@ func TestLex(t *testing.T) {
 		input  string
 		tokens []Token
 	}{
+		{
+			"1",
+			[]Token{
+				{Kind: Number, Value: "1"},
+				{Kind: EOF},
+			},
+		},
 		{
 			".5 0.025 1 02 1e3 0xFF 0b0101 0o600 1.2e-4 1_000_000 _42 -.5",
 			[]Token{
@@ -265,33 +271,33 @@ func compareTokens(i1, i2 []Token) bool {
 }
 
 func TestLex_location(t *testing.T) {
-	source := file.NewSource("1..2 3..4")
+	source := file.NewSource("1..2\n3..4")
 	tokens, err := Lex(source)
 	require.NoError(t, err)
 	require.Equal(t, []Token{
-		{Location: file.Location{Line: 1, Column: 0}, Kind: Number, Value: "1"},
-		{Location: file.Location{Line: 1, Column: 1}, Kind: Operator, Value: ".."},
-		{Location: file.Location{Line: 1, Column: 3}, Kind: Number, Value: "2"},
-		{Location: file.Location{Line: 1, Column: 5}, Kind: Number, Value: "3"},
-		{Location: file.Location{Line: 1, Column: 6}, Kind: Operator, Value: ".."},
-		{Location: file.Location{Line: 1, Column: 8}, Kind: Number, Value: "4"},
-		{Location: file.Location{Line: 1, Column: 8}, Kind: EOF, Value: ""},
+		{Location: file.Location{From: 0, To: 1}, Kind: "Number", Value: "1"},
+		{Location: file.Location{From: 1, To: 3}, Kind: "Operator", Value: ".."},
+		{Location: file.Location{From: 3, To: 4}, Kind: "Number", Value: "2"},
+		{Location: file.Location{From: 5, To: 6}, Kind: "Number", Value: "3"},
+		{Location: file.Location{From: 6, To: 8}, Kind: "Operator", Value: ".."},
+		{Location: file.Location{From: 8, To: 9}, Kind: "Number", Value: "4"},
+		{Location: file.Location{From: 8, To: 9}, Kind: "EOF", Value: ""},
 	}, tokens)
 }
 
 const errorTests = `
 "\xQA"
-invalid char escape (1:5)
+invalid char escape (1:4)
  | "\xQA"
- | ....^
+ | ...^
 
 id "hello
 literal not terminated (1:10)
  | id "hello
  | .........^
 
 früh ♥︎
-unrecognized character: U+2665 '♥' (1:7)
+unrecognized character: U+2665 '♥' (1:6)
  | früh ♥︎
 `
 

parser/lexer/state.go

@@ -14,7 +14,7 @@ func root(l *lexer) stateFn {
 		l.emitEOF()
 		return nil
 	case utils.IsSpace(r):
-		l.ignore()
+		l.skip()
 		return root
 	case r == '\'' || r == '"':
 		l.scanString(r)
@@ -83,14 +83,14 @@ func (l *lexer) scanNumber() bool {
 		}
 	}
 	l.acceptRun(digits)
-	loc, prev, end := l.loc, l.prev, l.end
+	end := l.end
 	if l.accept(".") {
 		// Lookup for .. operator: if after dot there is another dot (1..2), it maybe a range operator.
 		if l.peek() == '.' {
 			// We can't backup() here, as it would require two backups,
 			// and backup() func supports only one for now. So, save and
 			// restore it here.
-			l.loc, l.prev, l.end = loc, prev, end
+			l.end = end
 			return true
 		}
 		l.acceptRun(digits)
@@ -147,7 +147,7 @@ func not(l *lexer) stateFn {
 
 	l.skipSpaces()
 
-	pos, loc, prev := l.end, l.loc, l.prev
+	end := l.end
 
 	// Get the next word.
 	for {
@@ -164,7 +164,7 @@ func not(l *lexer) stateFn {
 	case "in", "matches", "contains", "startsWith", "endsWith":
 		l.emit(Operator)
 	default:
-		l.end, l.loc, l.prev = pos, loc, prev
+		l.end = end
 	}
 	return root
 }
@@ -193,7 +193,7 @@ func singleLineComment(l *lexer) stateFn {
 			break
 		}
 	}
-	l.ignore()
+	l.skip()
 	return root
 }
 
@@ -207,7 +207,7 @@ func multiLineComment(l *lexer) stateFn {
 			break
 		}
 	}
-	l.ignore()
+	l.skip()
 	return root
 }