added NULL token for missing fields like function name, start of Serialize() for several ast types

Author: ldemailly

Diff for: ast/ast.go

@@ -4,6 +4,7 @@
 package ast
 
 import (
+	"fmt"
 	"io"
 	"strconv"
 	"strings"
@@ -24,6 +25,7 @@ type PrintState struct {
 
 func DebugString(n Node) string {
 	ps := NewPrintState()
+	ps.Compact = true
 	n.PrettyPrint(ps)
 	return ps.String()
 }
@@ -68,6 +70,7 @@ func (ps *PrintState) Print(str ...string) *PrintState {
 type Node interface {
 	Value() *token.Token
 	PrettyPrint(ps *PrintState) *PrintState
+	Serialize(out io.Writer)
 }
 
 // Common to all nodes that have a token and avoids repeating the same TokenLiteral() methods.
@@ -79,6 +82,17 @@ func (b Base) Value() *token.Token {
 	return b.Token
 }
 
+func (b Base) Serialize(out io.Writer) {
+	if b.Token == nil {
+		panic(fmt.Sprintf("Serialize called on nil token in base %#v", b))
+	}
+	log.Debugf("Serializing %#v", b.Token)
+	_, _ = out.Write([]byte{b.Code()})
+	if b.HasContent() {
+		_, _ = out.Write([]byte(b.Literal()))
+	}
+}
+
 func (b Base) PrettyPrint(ps *PrintState) *PrintState {
 	// In theory should only be called for literals.
 	// log.Debugf("PrettyPrint on base called for %T", b.Value())
@@ -99,6 +113,13 @@ func (rs ReturnStatement) PrettyPrint(ps *PrintState) *PrintState {
 	return ps
 }
 
+func (rs ReturnStatement) Serialize(out io.Writer) {
+	rs.Base.Serialize(out)
+	if rs.ReturnValue != nil {
+		rs.ReturnValue.Serialize(out)
+	}
+}
+
 type Statements struct {
 	Base
 	Statements []Node
@@ -185,13 +206,14 @@ func (p Statements) PrettyPrint(ps *PrintState) *PrintState {
 	return ps
 }
 
-type Identifier struct {
-	Base
+func (p Statements) Serialize(out io.Writer) {
+	for _, s := range p.Statements {
+		s.Serialize(out)
+	}
 }
 
-func (i Identifier) PrettyPrint(out *PrintState) *PrintState {
-	out.Print(i.Literal())
-	return out
+type Identifier struct {
+	Base
 }
 
 type Comment struct {
@@ -200,11 +222,6 @@ type Comment struct {
 	SameLineAsNext     bool
 }
 
-func (c Comment) PrettyPrint(out *PrintState) *PrintState {
-	out.Print(c.Literal())
-	return out
-}
-
 type IntegerLiteral struct {
 	Base
 	Val int64
@@ -244,6 +261,11 @@ func (p PrefixExpression) PrettyPrint(out *PrintState) *PrintState {
 	return out
 }
 
+func (p PrefixExpression) Serialize(out io.Writer) {
+	p.Base.Serialize(out)
+	p.Right.Serialize(out)
+}
+
 type PostfixExpression struct {
 	Base
 	Prev *token.Token
@@ -291,6 +313,12 @@ func (i InfixExpression) PrettyPrint(out *PrintState) *PrintState {
 	return out
 }
 
+func (i InfixExpression) Serialize(out io.Writer) {
+	i.Base.Serialize(out)
+	i.Left.Serialize(out)
+	i.Right.Serialize(out)
+}
+
 type Boolean struct {
 	Base
 	Val bool
@@ -333,6 +361,16 @@ func (ie IfExpression) PrettyPrint(out *PrintState) *PrintState {
 	return out
 }
 
+func (ie IfExpression) Serialize(out io.Writer) {
+	ie.Base.Serialize(out)
+	ie.Condition.Serialize(out)
+	ie.Consequence.Serialize(out)
+	if ie.Alternative != nil {
+		_, _ = out.Write([]byte{token.ELSET.Code()})
+		ie.Alternative.Serialize(out)
+	}
+}
+
 func PrintList(out *PrintState, list []Node, sep string) {
 	for i, p := range list {
 		if i > 0 {
@@ -356,6 +394,13 @@ func (b Builtin) PrettyPrint(out *PrintState) *PrintState {
 	return out
 }
 
+func (b Builtin) Serialize(out io.Writer) {
+	b.Base.Serialize(out)
+	for _, p := range b.Parameters {
+		p.Serialize(out)
+	}
+}
+
 type FunctionLiteral struct {
 	Base       // The 'func' token
 	Name       *Identifier
@@ -380,6 +425,25 @@ func (fl FunctionLiteral) PrettyPrint(out *PrintState) *PrintState {
 	return out
 }
 
+var nullCode = []byte{token.NULLT.Code()}
+
+func writeNull(out io.Writer) {
+	_, _ = out.Write(nullCode)
+}
+
+func (fl FunctionLiteral) Serialize(out io.Writer) {
+	fl.Base.Serialize(out)
+	if fl.Name != nil {
+		fl.Name.Serialize(out)
+	} else {
+		writeNull(out)
+	}
+	for _, p := range fl.Parameters {
+		p.Serialize(out)
+	}
+	fl.Body.Serialize(out)
+}
+
 type CallExpression struct {
 	Base           // The '(' token
 	Function  Node // Identifier or FunctionLiteral
@@ -397,6 +461,14 @@ func (ce CallExpression) PrettyPrint(out *PrintState) *PrintState {
 	return out
 }
 
+func (ce CallExpression) Serialize(out io.Writer) {
+	ce.Base.Serialize(out)
+	ce.Function.Serialize(out)
+	for _, p := range ce.Arguments {
+		p.Serialize(out)
+	}
+}
+
 type ArrayLiteral struct {
 	Base     // The [ token
 	Elements []Node

Diff for: eval/eval_test.go

@@ -337,7 +337,7 @@ func TestFunctionObject(t *testing.T) {
 	if ast.DebugString(fn.Parameters[0]) != "x" {
 		t.Fatalf("parameter is not 'x'. got=%q", fn.Parameters[0])
 	}
-	expectedBody := "x + 2\n"
+	expectedBody := "x+2"
 	got := ast.DebugString(fn.Body)
 	if got != expectedBody {
 		t.Fatalf("body is not %q. got=%q", expectedBody, got)
@@ -614,15 +614,15 @@ func TestQuote(t *testing.T) {
 		},
 		{
 			`quote(5 + 8)`,
-			`5 + 8`,
+			`5+8`,
 		},
 		{
 			`quote(foobar)`,
 			`foobar`,
 		},
 		{
 			`quote(foobar + barfoo)`,
-			`foobar + barfoo`,
+			`foobar+barfoo`,
 		},
 	}
 
@@ -706,7 +706,7 @@ func TestQuoteUnquote(t *testing.T) {
 			t.Fatalf("quote.Node is nil")
 		}
 
-		got := ast.DebugString(quote.Node)
+		got := quote.Node.PrettyPrint(ast.NewPrintState()).String()
 		if got != tt.expected {
 			t.Errorf("not equal. got=%q, want=%q", got, tt.expected)
 		}

Diff for: eval/macro_expension_test.go

@@ -57,7 +57,7 @@ func TestDefineMacros(t *testing.T) {
 		t.Fatalf("parameter is not 'y'. got=%q", macro.Parameters[1])
 	}
 
-	expectedBody := "x + y\n" // or should have {}?
+	expectedBody := "x+y" // or should have {}?
 	got := ast.DebugString(macro.Body)
 	if got != expectedBody {
 		t.Fatalf("body is not %q. got=%q", expectedBody, got)

Diff for: examples/fib.gr

@@ -1,3 +1,4 @@
+/* fib.gr - compute the 35th Fibonacci number */
 func fib(x) {
 	if x <= 1 {
 		x

Diff for: main.go

@@ -21,6 +21,7 @@ func Main() int {
 	showParse := flag.Bool("parse", false, "show parse tree")
 	format := flag.Bool("format", false, "don't execute, just parse and re format the input")
 	compact := flag.Bool("compact", false, "When printing code, use no indentation and most compact form")
+	save := flag.Bool("save", false, "Emit the serialized AST")
 	showEval := flag.Bool("eval", true, "show eval results")
 	sharedState := flag.Bool("shared-state", false, "All files share same interpreter state (default is new state for each)")
 	cli.ArgsHelp = "*.gr files to interpret or `-` for stdin without prompt or no arguments for stdin repl..."
@@ -32,6 +33,7 @@ func Main() int {
 		ShowEval:   *showEval,
 		FormatOnly: *format,
 		Compact:    *compact,
+		Serialize:  *save,
 	}
 	nArgs := len(flag.Args())
 	if *commandFlag != "" {

Diff for: parser/parser_book_test.go

@@ -344,8 +344,7 @@ func TestOperatorPrecedenceParsing(t *testing.T) {
 		p := New(l)
 		program := p.ParseProgram()
 		checkParserErrors(t, p)
-
-		actual := ast.DebugString(program)
+		actual := program.PrettyPrint(ast.NewPrintState()).String()
 		last := actual[len(actual)-1]
 		if actual[len(actual)-1] != '\n' {
 			t.Errorf("expecting newline at end of program output, not found, got %q", last)
@@ -630,7 +629,7 @@ func TestCallExpressionParameterParsing(t *testing.T) {
 		}
 
 		for i, arg := range tt.expectedArgs {
-			got := ast.DebugString(exp.Arguments[i])
+			got := exp.Arguments[i].PrettyPrint(ast.NewPrintState()).String()
 			if got != arg {
 				t.Errorf("argument %d wrong. want=%q, got=%q", i,
 					arg, got)

Diff for: parser/parser_test.go

@@ -197,7 +197,7 @@ func Test_OperatorPrecedenceParsing(t *testing.T) {
 		program := p.ParseProgram()
 		checkParserErrors(t, tt.input, p)
 
-		actual := ast.DebugString(program)
+		actual := program.PrettyPrint(ast.NewPrintState()).String()
 		last := actual[len(actual)-1]
 		if actual[len(actual)-1] != '\n' {
 			t.Errorf("expecting newline at end of program output, not found, got %q", last)

Diff for: repl/repl.go

@@ -39,6 +39,7 @@ type Options struct {
 	NoColor    bool // color controlled by log package, unless this is set to true.
 	FormatOnly bool
 	Compact    bool
+	Serialize  bool
 }
 
 func EvalAll(s, macroState *eval.State, in io.Reader, out io.Writer, options Options) {
@@ -167,6 +168,10 @@ func EvalOne(s, macroState *eval.State, what string, out io.Writer, options Opti
 	if p.ContinuationNeeded() {
 		return true, nil, what
 	}
+	if options.Serialize {
+		program.Serialize(out)
+		return false, nil, what
+	}
 	printer := ast.NewPrintState()
 	printer.Compact = options.Compact
 	formatted := program.PrettyPrint(printer).String()

Diff for: token/token.go

@@ -25,6 +25,7 @@ type Token struct {
 	_         noCopy
 	tokenType Type
 	charCode  byte // 1 byte serialization printable code.
+	valued    bool // does literal contains the value (for IDENT, INT, FLOAT, STRING, *COMMENT) or is the code enough.
 	literal   string
 }
 
@@ -47,7 +48,7 @@ func Intern(t Type, literal string) *Token {
 	if code == 0 {
 		panic("no code for " + t.String())
 	}
-	return InternToken(&Token{tokenType: t, literal: literal, charCode: code})
+	return InternToken(&Token{tokenType: t, literal: literal, charCode: code, valued: true})
 }
 
 func ResetInterning() {
@@ -133,14 +134,17 @@ const (
 
 	endIdentityTokens
 
+	NULL // used for serializing absent fields like function name.
 	EOF
 )
 
 var (
 	EOLT   = &Token{tokenType: EOL}
 	EOFT   = &Token{tokenType: EOF}
+	NULLT  *Token
 	TRUET  *Token
 	FALSET *Token
+	ELSET  *Token
 )
 
 var (
@@ -158,6 +162,10 @@ func init() {
 	Init()
 }
 
+func assocValued(t Type, code byte) {
+	assocCodeAndToken(t, code)
+}
+
 func assocCodeAndToken(t Type, code byte) {
 	if tToCode[t] != 0 {
 		panic("duplicate code for " + t.String() + " " + string(tToCode[t]) + " vs " + string(code))
@@ -237,6 +245,7 @@ func Init() { //nolint:funlen // we need all this.
 	}
 	TRUET = tToT[TRUE]
 	FALSET = tToT[FALSE]
+	ELSET = tToT[ELSE]
 	// Single character tokens:
 	assoc(ASSIGN, '=')
 	assoc(PLUS, '+')
@@ -291,13 +300,15 @@ func Init() { //nolint:funlen // we need all this.
 	// Special alias for := to be same as ASSIGN.
 	c2Tokens[[2]byte{':', '='}] = cTokens['=']
 	// Valued tokens.:
-	assocCodeAndToken(IDENT, ' ')
-	assocCodeAndToken(INT, '0')
-	assocCodeAndToken(FLOAT, '.')
-	assocCodeAndToken(STRING, 's')
-	assocCodeAndToken(LINECOMMENT, 'c')
-	assocCodeAndToken(BLOCKCOMMENT, 'b')
-
+	assocValued(IDENT, ' ')
+	assocValued(INT, '0')
+	assocValued(FLOAT, '.')
+	assocValued(STRING, 's')
+	assocValued(LINECOMMENT, 'c')
+	assocValued(BLOCKCOMMENT, 'b')
+
+	assocCodeAndToken(NULL, 'Z') // absence of value
+	NULLT = &Token{tokenType: NULL, literal: "", charCode: tToCode[NULL], valued: false}
 	assocCodeAndToken(ILLEGAL, 1) // doesn't need to be printable.
 	assocCodeAndToken(EOL, 10)    // nl but not printed anyway
 	assocCodeAndToken(EOF, 'z')   // also not used/visible
@@ -311,7 +322,7 @@ func LookupIdent(ident string) *Token {
 	if t, ok := keywords[ident]; ok {
 		return t
 	}
-	return InternToken(&Token{tokenType: IDENT, literal: ident, charCode: ' '})
+	return InternToken(&Token{tokenType: IDENT, literal: ident, charCode: ' ', valued: true})
 }
 
 // ByType is the cheapest lookup for all the tokens whose type
@@ -347,6 +358,11 @@ func (t *Token) DebugString() string {
 	return t.Type().String() + ":" + strconv.Quote(t.Literal())
 }
 
+// Is this one of the valued type, where the literal is content and not just the code.
+func (t *Token) HasContent() bool {
+	return t.valued
+}
+
 func NumTokens() int {
 	return tokensCount
 }