File tree Expand file tree Collapse file tree 1 file changed +8
-8
lines changed
Expand file tree Collapse file tree 1 file changed +8
-8
lines changed Original file line number Diff line number Diff line change @@ -238,15 +238,15 @@ var keywordTokenTypes = map[string]models.TokenType{
238238// tkz, _ := New()
239239// tokens, err := tkz.Tokenize([]byte(sql))
240240type Tokenizer struct {
241- input []byte // Input SQL bytes (zero-copy reference)
242- pos Position // Current scanning position
243- lineStart Position // Start of current line
244- lineStarts []int // Byte offsets of line starts (for position tracking)
245- line int // Current line number (1-based)
246- keywords * keywords.Keywords // Keyword classifier for token type determination
241+ input []byte // Input SQL bytes (zero-copy reference)
242+ pos Position // Current scanning position
243+ lineStart Position // Start of current line
244+ lineStarts []int // Byte offsets of line starts (for position tracking)
245+ line int // Current line number (1-based)
246+ keywords * keywords.Keywords // Keyword classifier for token type determination
247247 dialect keywords.SQLDialect // SQL dialect for dialect-specific keyword recognition
248- logger * slog.Logger // Optional structured logger for verbose tracing
249- Comments []models.Comment // Comments captured during tokenization
248+ logger * slog.Logger // Optional structured logger for verbose tracing
249+ Comments []models.Comment // Comments captured during tokenization
250250}
251251
252252// New creates a new Tokenizer with default configuration and keyword support.
You can’t perform that action at this time.
0 commit comments