lexer_test.go

package sexpr

import (
	"testing"
)

// Simple tests for tokenizing strings with a single token.
func TestLexerValidTokens(t *testing.T) {
	tests := []struct {
		input       string
		expectedTok *token
	}{
		// end-of-file, left paranthesis, right parantsis and comma tokens
		{"", mkTokenEOF()},
		{"(", mkTokenLpar()},
		{")", mkTokenRpar()},
		{".", mkTokenDot()},

		// `mkTokenQuote()` returns the special `'` notation of `QUOTE`
		{"'", mkTokenQuote()},

		// `mkTokenSymbol("QUOTE")` returns the symbol `QUOTE`
		{"QUOTE", mkTokenSymbol("QUOTE")},
		{"quoTE", mkTokenSymbol("QUOTE")},
		{"qUoTe", mkTokenSymbol("QUOTE")},
		{"quote", mkTokenSymbol("QUOTE")},

		// example of valid number tokens
		{"0", mkTokenNumber("0")},
		{"1", mkTokenNumber("1")},
		{"00001", mkTokenNumber("1")},
		{"+00001", mkTokenNumber("1")},
		{"-00001", mkTokenNumber("-1")},
		{"1234567890", mkTokenNumber("1234567890")},
		{
			"10000000000000000000000000000000000000000000000000",
			mkTokenNumber("10000000000000000000000000000000000000000000000000"),
		},
		{
			"-10000000000000000000000000000000000000000000000000",
			mkTokenNumber("-10000000000000000000000000000000000000000000000000"),
		},

		// example of valid symbol tokens
		{"+", mkTokenSymbol("+")},
		{"*", mkTokenSymbol("*")},
		{"atom", mkTokenSymbol("ATOM")},
		{"zerop", mkTokenSymbol("ZEROP")},

		{"nil", mkTokenSymbol("NIL")},
		{"nIl", mkTokenSymbol("NIL")},
		{"Nil", mkTokenSymbol("NIL")},
		{"NIL", mkTokenSymbol("NIL")},

		{"t", mkTokenSymbol("T")},
		{"T", mkTokenSymbol("T")},

		{"foo", mkTokenSymbol("FOO")},
		{"isValidAtom", mkTokenSymbol("ISVALIDATOM")},
		{"is-valid-atom", mkTokenSymbol("IS-VALID-ATOM")},
		{"random_token_0123_", mkTokenSymbol("RANDOM_TOKEN_0123_")},
		{"X", mkTokenSymbol("X")},
		{"Var1", mkTokenSymbol("VAR1")},
		{"_A_", mkTokenSymbol("_A_")},
		{"___Y__2_", mkTokenSymbol("___Y__2_")},
	}
	for idx, test := range tests {
		func() {
			defer func() {
				if r := recover(); r != nil {
					t.Errorf("\nin test %d (\"%s\") panic: %s", idx, test.input, r)
				}
			}()
			lex := newLexer(test.input)
			tok, err := lex.next()
			if err != nil {
				t.Errorf("\nin test %d (\"%s\"): lexer got an unexpected error %#v when tokenizing a valid input %#v", idx, test.input, err, test.input)
			}
			if !equalToken(tok, test.expectedTok) {
				t.Errorf("\nin test %d (\"%s\"):\n\texpected token %#v\n\tgot token      %#v", idx, test.input, test.expectedTok, tok)
			}
		}()
	}
}

// TestLexerInvalidTokens tests that the lexer does not token invalid strings.
func TestLexerInvalidTokens(t *testing.T) {
	invalidStrings := []string{
		// Example of some invalid symbols in terms
		",",
		"\"",
		"=",
		"#",
		"$",
		"%",
		":",
	}
	for idx, input := range invalidStrings {
		func() {
			defer func() {
				if r := recover(); r != nil {
					t.Errorf("\nin test %d (\"%s\") panic: %s", idx, input, r)
				}
			}()
			if _, err := newLexer(input).next(); err != ErrLexer {
				t.Errorf("\nin test %d (\"%s\"): lexer did not get error %#v when tokenizing an invalid input %#v",
					idx, input, ErrLexer, input)
			}
		}()
	}
}

func TestLexerSequence(t *testing.T) {
	// `newLexer(str)` returns a new lexer with given input string.
	input := " (zerop (  +  +1 '+2 (quote -3))) "
	lex := newLexer(input)
	// The expected sequence of literals when calling lex.next()
	expectedTokens := []*token{
		mkTokenLpar(),
		mkTokenSymbol("ZEROP"),
		mkTokenLpar(),
		mkTokenSymbol("+"),
		mkTokenNumber("1"),
		mkTokenQuote(),
		mkTokenNumber("2"),
		mkTokenLpar(),
		mkTokenSymbol("QUOTE"),
		mkTokenNumber("-3"),
		mkTokenRpar(),
		mkTokenRpar(),
		mkTokenRpar(),
		mkTokenEOF(),
		mkTokenEOF(),
	}
	for idx, expectedToken := range expectedTokens {
		// `lex.next()` consumes the input string, skips spaces and returns the next
		// token.
		token, err := lex.next()
		if err != nil {
			t.Errorf("lexer got an unexpected error %#v when tokenizing a valid input", err)
		}
		if token == nil {
			t.Errorf("lexer returned an unexpected nil token")
		}
		if !equalToken(token, expectedToken) {
			t.Errorf("\nin %d-th token of input \"%s\":\n\texpected token %#v\n\tgot token      %#v", idx, input, expectedToken, token)
		}
	}
}