Add unique single printable character to Token for serialization

Author: ldemailly

token/token.go

@@ -1,7 +1,7 @@
 // There are 2 types of Token, constant ones (with no "value") and the ones with attached
-// value that is variable (e.g. IDENT, INT, FLOAT, STRING, COMMENT).
-// We'd use the upcoming unique https://tip.golang.org/doc/go1.23#new-unique-package
-// but we want this to run on 1.22 and earlier.
+// value that is variable (e.g. IDENT, INT, FLOAT, STRING, *COMMENT).
+// We might use the upcoming unique https://tip.golang.org/doc/go1.23#new-unique-package
+// but we want this to run on 1.22 and earlier and rolled our own not multi threaded.
 package token
 
 import (
@@ -17,13 +17,14 @@ type noCopy struct{}
 func (*noCopy) Lock()   {}
 func (*noCopy) Unlock() {}
 
-type Type uint8
+type Type byte
 
 type Token struct {
 	// Allows go vet to flag accidental copies of this type,
 	// though with an error about lock value which can be confusing
 	_         noCopy
 	tokenType Type
+	charCode  byte // 1 byte serialization printable code.
 	literal   string
 }
 
@@ -42,7 +43,11 @@ func InternToken(t *Token) *Token {
 }
 
 func Intern(t Type, literal string) *Token {
-	return InternToken(&Token{tokenType: t, literal: literal})
+	code := tToCode[t]
+	if code == 0 {
+		panic("no code for " + t.String())
+	}
+	return InternToken(&Token{tokenType: t, literal: literal, charCode: code})
 }
 
 func ResetInterning() {
@@ -56,9 +61,10 @@ const (
 	startValueTokens
 
 	// Identifiers + literals. with attached value.
-	IDENT // add, foobar, x, y, ...
-	INT   // 1343456
-	FLOAT // 1. 1e3
+	IDENT  // add, foobar, x, y, ...
+	INT    // 1343456
+	FLOAT  // .5, 3.14159,...
+	STRING // "foo bar"
 	LINECOMMENT
 	BLOCKCOMMENT
 
@@ -112,7 +118,6 @@ const (
 	IF
 	ELSE
 	RETURN
-	STRING
 	MACRO
 	// Macro magic.
 	QUOTE
@@ -139,56 +144,95 @@ var (
 )
 
 var (
-	keywords map[string]*Token
-	cTokens  map[byte]*Token
-	c2Tokens map[[2]byte]*Token
-	tToChar  map[Type]byte
-	tToT     map[Type]*Token // for all token that are constant.
+	keywords    map[string]*Token
+	cTokens     [256]*Token // for all token that are single char.
+	tToCode     [256]byte   // iota code to char code.
+	codeToT     [256]Type   // char code to Type
+	c2Tokens    map[[2]byte]*Token
+	tToChar     map[Type]byte
+	tToT        map[Type]*Token // for all token that are constant.
+	tokensCount int
 )
 
 func init() {
 	Init()
 }
 
+func assocCodeAndToken(t Type, code byte) {
+	if tToCode[t] != 0 {
+		panic("duplicate code for " + t.String() + " " + string(tToCode[t]) + " vs " + string(code))
+	}
+	tToCode[t] = code
+	if codeToT[code] != 0 {
+		panic("duplicate token for " + string(code) + " " + codeToT[code].String() + " vs " + t.String())
+	}
+	codeToT[code] = t
+	tokensCount++
+}
+
 func assoc(t Type, c byte) {
 	tToChar[t] = c
-	tok := &Token{tokenType: t, literal: string(c)}
+	tok := &Token{tokenType: t, literal: string(c), charCode: c}
 	cTokens[c] = tok
 	tToT[t] = tok
+	assocCodeAndToken(t, c)
 }
 
-func assocS(t Type, s string) *Token {
-	tok := &Token{tokenType: t, literal: s}
+func assocS(t Type, s string, code byte) *Token {
+	tok := &Token{tokenType: t, literal: s, charCode: code}
 	old := InternToken(tok)
 	if old != tok {
 		panic("duplicate token for " + s)
 	}
 	tToT[t] = tok
+	assocCodeAndToken(t, code)
 	return tok
 }
 
-func assocC2(t Type, str string) {
+func assocC2(t Type, str string, code byte) {
 	if len(str) != 2 {
 		panic("assocC2: expected 2 char string")
 	}
-	tok := &Token{tokenType: t, literal: str}
+	tok := &Token{tokenType: t, literal: str, charCode: code}
 	old := InternToken(tok)
 	if old != tok {
 		panic("duplicate token for " + str)
 	}
 	tToT[t] = tok
 	c2Tokens[[2]byte{str[0], str[1]}] = tok
+	assocCodeAndToken(t, code)
 }
 
-func Init() {
+func Init() { //nolint:funlen // we need all this.
 	ResetInterning()
+	tokensCount = 0
 	keywords = make(map[string]*Token)
-	cTokens = make(map[byte]*Token)
+	clear(cTokens[:])
+	clear(tToCode[:])
+	clear(codeToT[:])
 	c2Tokens = make(map[[2]byte]*Token)
 	tToChar = make(map[Type]byte)
 	tToT = make(map[Type]*Token)
 	for i := startIdentityTokens + 1; i < endIdentityTokens; i++ {
-		t := assocS(i, strings.ToLower(i.String()))
+		str := i.String()
+		code := str[0] // will be dup for some, we're fixing this below.
+		switch i {     //nolint:exhaustive // we're only fixing the ones conflicting.
+		case TRUE:
+			code = 't'
+		case FALSE:
+			code = 'f'
+		case FIRST:
+			code = '1'
+		case REST:
+			code = '2'
+		case PRINTLN:
+			code = 'N'
+		case ERROR:
+			code = 'e'
+		case LOG:
+			code = 'p'
+		}
+		t := assocS(i, strings.ToLower(str), code)
 		keywords[t.literal] = t
 	}
 	TRUET = tToT[TRUE]
@@ -218,8 +262,8 @@ func Init() {
 		if !ok {
 			panic("missing single character token char lookup for " + i.String())
 		}
-		v, ok := cTokens[b]
-		if !ok {
+		v := cTokens[b]
+		if v == nil {
 			panic("missing single character token for " + i.String())
 		}
 		if v.tokenType != i {
@@ -238,14 +282,25 @@ func Init() {
 		}
 	}
 	// Multi character non identity tokens.
-	assocC2(LTEQ, "<=")
-	assocC2(GTEQ, ">=")
-	assocC2(EQ, "==")
-	assocC2(NOTEQ, "!=")
-	assocC2(INCR, "++")
-	assocC2(DECR, "--")
+	assocC2(LTEQ, "<=", 'l')
+	assocC2(GTEQ, ">=", 'g')
+	assocC2(EQ, "==", '_')
+	assocC2(NOTEQ, "!=", 'n')
+	assocC2(INCR, "++", 'i')
+	assocC2(DECR, "--", 'd')
 	// Special alias for := to be same as ASSIGN.
 	c2Tokens[[2]byte{':', '='}] = cTokens['=']
+	// Valued tokens.:
+	assocCodeAndToken(IDENT, ' ')
+	assocCodeAndToken(INT, '0')
+	assocCodeAndToken(FLOAT, '.')
+	assocCodeAndToken(STRING, 's')
+	assocCodeAndToken(LINECOMMENT, 'c')
+	assocCodeAndToken(BLOCKCOMMENT, 'b')
+
+	assocCodeAndToken(ILLEGAL, 1) // doesn't need to be printable.
+	assocCodeAndToken(EOL, 10)    // nl but not printed anyway
+	assocCodeAndToken(EOF, 'z')   // also not used/visible
 }
 
 //go:generate stringer -type=Type
@@ -256,14 +311,14 @@ func LookupIdent(ident string) *Token {
 	if t, ok := keywords[ident]; ok {
 		return t
 	}
-	return InternToken(&Token{tokenType: IDENT, literal: ident})
+	return InternToken(&Token{tokenType: IDENT, literal: ident, charCode: ' '})
 }
 
 // ByType is the cheapest lookup for all the tokens whose type
 // only have one possible instance/value
 // (ie all the tokens except for the first 4 value tokens).
 // TODO: codegen all the token constants to avoid needing this function.
-// (even though that's better than string comparaisons).
+// (even though that's better than string comparisons).
 func ByType(t Type) *Token {
 	return tToT[t]
 }
@@ -276,6 +331,10 @@ func (t *Token) Type() Type {
 	return t.tokenType
 }
 
+func (t *Token) Code() byte {
+	return t.charCode
+}
+
 func ConstantTokenChar(literal byte) *Token {
 	return cTokens[literal]
 }
@@ -287,3 +346,7 @@ func ConstantTokenChar2(c1, c2 byte) *Token {
 func (t *Token) DebugString() string {
 	return t.Type().String() + ":" + strconv.Quote(t.Literal())
 }
+
+func NumTokens() int {
+	return tokensCount
+}

token/token_test.go

@@ -79,7 +79,7 @@ func TestMultiCharTokens(t *testing.T) {
 		{"--", DECR},
 	}
 	for _, tt := range tests {
-		tok := &Token{tokenType: tt.expected, literal: tt.input}
+		tok := &Token{tokenType: tt.expected, literal: tt.input, charCode: tToCode[tt.expected]}
 		tok2 := InternToken(tok)
 		if tok == tok2 {
 			t.Errorf("Intern[%s] was unexpectedly created", tt.input)
@@ -138,3 +138,13 @@ func TestColonEqualAlias(t *testing.T) {
 		t.Errorf("ConstantTokenStr[:=] returned %v, expected '='", tok.Literal())
 	}
 }
+
+func TestNumTokens(t *testing.T) {
+	Init()
+	n := NumTokens()
+	// 49 tokens in total so far
+	expected := int(EOF) - 8 + 1 // 8 markers to subtract + 1 for ILLEGAL==0.
+	if n != expected {
+		t.Errorf("NumTokens() returned %d, expected %d", n, expected)
+	}
+}