feat: add f-string literals (f"…{expr}…") to Starlark

internal/compile/codegen_test.go

@@ -71,6 +71,48 @@ func TestPlusFolding(t *testing.T) {
 		}
 	}
 }
+func TestFstring(t *testing.T) {
+	isPredeclared := func(name string) bool { return name == "x" }
+	isUniversal := func(name string) bool { return false }
+	for i, test := range []struct {
+		src  string // source expression
+		want string // disassembled code
+	}{
+		{
+			`f"hehehe{7}"`,
+			`constant "hehehe{}"; attr<0>; constant 7; call<256>; return`,
+		},
+		{
+			`f"hehehe{7}" + f"hehe"`,
+			`constant "hehehe{}"; attr<0>; constant 7; call<256>; constant "hehe"; plus; return`,
+		},
+		{
+			`f"hehe" + f"hehe{7}"`,
+			`constant "hehe"; constant "hehe{}"; attr<0>; constant 7; call<256>; plus; return`,
+		},
+		{
+			`f"hehe" + f"hehe"`,
+			`constant "hehe"; constant "hehe"; plus; return`,
+		},
+	} {
+		expr, err := syntax.ParseExpr("in.star", test.src, 0)
+		if err != nil {
+			t.Errorf("#%d: %v", i, err)
+			continue
+		}
+		locals, err := resolve.Expr(expr, isPredeclared, isUniversal)
+		if err != nil {
+			t.Errorf("#%d: %v", i, err)
+			continue
+		}
+		got := disassemble(Expr(syntax.LegacyFileOptions(), expr, "<expr>", locals).Toplevel)
+		// t.Errorf("disassemble: %s", got)
+		if test.want != got {
+			t.Errorf("expression <<%s>> generated <<%s>>, want <<%s>>",
+				test.src, got, test.want)
+		}
+	}
+}
 
 // disassemble is a trivial disassembler tailored to the accumulator test.
 func disassemble(f *Funcode) string {

internal/compile/compile.go

@@ -1307,6 +1307,14 @@ func (fcomp *fcomp) expr(e syntax.Expr) {
 		}
 		fcomp.emit1(CONSTANT, fcomp.pcomp.constantIndex(v))
 
+	case *syntax.FStringExpr:
+		body := strings.Join(e.StringParts, "{}")
+		fcomp.emit1(CONSTANT, fcomp.pcomp.constantIndex(body))
+		fcomp.setPos(e.TokenPos)
+		fcomp.emit1(ATTR, fcomp.pcomp.nameIndex("format"))
+		op, arg := fcomp.args(e.Args)
+		fcomp.emit1(op, arg)
+
 	case *syntax.ListExpr:
 		for _, x := range e.List {
 			fcomp.expr(x)
@@ -1659,20 +1667,20 @@ func (fcomp *fcomp) call(call *syntax.CallExpr) {
 
 	// usual case
 	fcomp.expr(call.Fn)
-	op, arg := fcomp.args(call)
+	op, arg := fcomp.args(call.Args)
 	fcomp.setPos(call.Lparen)
 	fcomp.emit1(op, arg)
 }
 
 // args emits code to push a tuple of positional arguments
 // and a tuple of named arguments containing alternating keys and values.
 // Either or both tuples may be empty (TODO(adonovan): optimize).
-func (fcomp *fcomp) args(call *syntax.CallExpr) (op Opcode, arg uint32) {
+func (fcomp *fcomp) args(callargs []syntax.Expr) (op Opcode, arg uint32) {
 	var callmode int
 	// Compute the number of each kind of parameter.
 	var p, n int // number of  positional, named arguments
 	var varargs, kwargs syntax.Expr
-	for _, arg := range call.Args {
+	for _, arg := range callargs {
 		if binary, ok := arg.(*syntax.BinaryExpr); ok && binary.Op == syntax.EQ {
 
 			// named argument (name, value)

resolve/resolve.go

@@ -650,7 +650,16 @@ func (r *resolver) expr(e syntax.Expr) {
 	switch e := e.(type) {
 	case *syntax.Ident:
 		r.use(e)
-
+	case *syntax.FStringExpr:
+		for _, arg := range e.Args {
+			r.expr(arg)
+		}
+		// Fail gracefully if compiler-imposed limit is exceeded.
+		p := len(e.Args)
+		if p >= 256 {
+			pos, _ := e.Span()
+			r.errorf(pos, "%v arguments in fstring, limit is 255", p)
+		}
 	case *syntax.Literal:
 
 	case *syntax.ListExpr:

starlark/eval_test.go

@@ -142,6 +142,7 @@ func TestExecFile(t *testing.T) {
 		"testdata/control.star",
 		"testdata/dict.star",
 		"testdata/float.star",
+		"testdata/fstring.star",
 		"testdata/function.star",
 		"testdata/int.star",
 		"testdata/json.star",

starlark/testdata/fstring.star

@@ -0,0 +1,133 @@
+# Tests of Starlark 'fstring'
+# option:set
+
+# Tests of Starlark f-string literals
+# syntax identical to Python 3.6+ (no '=', no '\{', no nested '{ }')
+load("assert.star", "assert")
+
+# --- basic interpolation ----------------------------------------------------
+
+name = "Starlark"
+version = 1
+assert.eq(f"hello {name}", "hello Starlark")
+assert.eq(f"{{}} {name}", "{} Starlark")
+assert.eq(f"{{{{}}}} {name}", "{{}} Starlark")
+assert.eq(f"{name} {version}", "Starlark 1")
+assert.eq(f"{{ literal }}", "{ literal }")   # doubled braces → literal
+assert.eq(f"start{ {"x":3} }end", "start{\"x\": 3}end")   # dict
+
+# --- conversion flags -------------------------------------------------------
+# todo: future plans
+# pi = 3.14
+# assert.eq(f"{pi!s}", "3.14")   # str()
+# assert.eq(f"{pi!r}", "3.14")   # repr()  (same here because str/repr identical)
+# big = 1000000
+# assert.eq(f"{big:,}", "1,000,000")  # format-spec with ','
+
+# --- field names -----------------------------------------------------------
+
+d = {"x": 10, "y": 20}
+assert.eq(f"{d['x']}", "10")
+assert.eq(f"{d['y']}", "20")
+
+# positional and keyword in .format() style already tested elsewhere;
+# f-strings use **inline** expressions, so we just check they evaluate.
+tpl = (4, 5)
+assert.eq(f"{tpl[0]} and {tpl[1]}", "4 and 5")
+
+# --- padding / alignment / precision ---------------------------------------
+#todo: future plans
+# n = 42
+# assert.eq(f"{n:>5}", "   42")
+# assert.eq(f"{n:<5}", "42   ")
+# assert.eq(f"{n:^5}", " 42  ")
+# assert.eq(f"{n:05}", "00042")
+
+# f = 1.234567
+# assert.eq(f"{f:.2f}", "1.23")
+
+# --- escaping --------------------------------------------------------------
+assert.eq(f"backslash \\ still one", "backslash \\ still one")
+assert.eq(f"quotes ' and \" kept", 'quotes \' and " kept')
+
+# --- empty f-string --------------------------------------------------------
+
+assert.eq(f"", "")
+
+# --- nested quotes (no problem) --------------------------------------------
+
+assert.eq(f'He said "Hello {name}"', 'He said "Hello Starlark"')
+assert.eq(f"it's {version} o'clock", "it's 1 o'clock")
+
+# --- multiline f-string ----------------------------------------------------
+
+msg = f"""
+hello {name}
+version {version}
+""".strip()
+assert.true(msg.startswith("hello Starlark"))
+assert.true(msg.endswith("version 1"))
+
+# --- unicode ---------------------------------------------------------------
+
+α = 2
+assert.eq(f"α = {α}", "α = 2")
+
+# --- errors that must be caught at compile-time ----------------------------
+#todo? more sound errors, now raises with text: `expect "}}" or "{expression}", got single"}"` on almost all errors
+# (Un-comment each block to verify the parser rejects it.)
+
+# 1. single '}' without '{'
+# assert.fails(lambda: f"oops}", "single '}' in format")
+
+# 2. unmatched '{' #now says "unexpected new line in string". is it ok?
+# assert.fails(lambda: f"oops{", "unmatched '{' in format")
+
+# 3. invalid expression inside braces # now fails with "got , want primary expression"
+# assert.fails(lambda: f"{1+}", "invalid syntax")
+
+# 4. unknown conversion #future plans?
+# assert.fails(lambda: f"{pi!z}", "unknown conversion")
+
+# 5. # now fails with "got , want primary expression"
+# assert.fails(lambda: f"nothing{}", "nothing{}")          # todo: assert fails
+
+# --- runtime errors --------------------------------------------------------
+
+# expression raises → propagates
+def raise_error():
+    f"{1/0}"
+
+assert.fails(raise_error, "division by zero")
+
+# --- interaction with other string features -------------------------------
+
+# f-string is still a plain string afterwards
+s = f"{name}"
+assert.eq(s.upper(), "STARLARK")
+assert.eq(s * 2, "StarlarkStarlark")
+assert.eq(len(s), 8)
+
+# concatenation
+assert.eq(f"A" + f"{name}" + f"B", "AStarlarkB")
+
+# raw *and* f is illegal in Python; Starlark should follow
+# (un-comment to check)
+# assert.fails(lambda: rf"{name}", "cannot use both raw and f-string") #todo?
+
+# --- edge cases ------------------------------------------------------------
+
+# only doubled braces
+assert.eq(f"{{}}", "{}")
+
+# mixed
+#todo: fix this or just look into this
+# assert.eq(f"{{ {name} }}", "{ Starlark }")
+
+# zero-width joiner emoji (4-byte UTF-8)
+emoji = "😿"
+assert.eq(f"{emoji}", "😿")
+# assert.eq(f"{emoji!r}", '"😿"') # todo: future plans?
+
+# ---------------------------------------------------------------------------
+# end of f-string tests

syntax/parse.go

@@ -11,7 +11,11 @@ package syntax
 // package.  Verify that error positions are correct using the
 // chunkedfile mechanism.
 
-import "log"
+import (
+	// "fmt"
+	"log"
+	"strings"
+)
 
 // Enable this flag to print the token stream and log.Fatal on the first error.
 const debug = false
@@ -828,6 +832,29 @@ func (p *parser) parsePrimary() Expr {
 		raw := p.tokval.raw
 		pos := p.nextToken()
 		return &Literal{Token: tok, TokenPos: pos, Raw: raw, Value: val}
+	case FSTRING_FULL:
+		val := p.tokval.string
+		raw := p.tokval.raw
+		token := p.tok
+		pos := p.nextToken()
+		response := Literal{Token: token, TokenPos: pos, Raw: raw, Value: val}
+		return &response
+	case FSTRING_PART:
+		resultExpr := FStringExpr{}
+		resultExpr.StringParts = append(resultExpr.StringParts, strings.ReplaceAll(strings.ReplaceAll(p.tokval.string, "}", "}}"), "{", "{{"))
+		resultExpr.RawParts = append(resultExpr.RawParts, p.tokval.raw)
+		toktmp := p.tok
+		pos := p.nextToken()
+		resultExpr.TokenPos = pos
+		for toktmp != FSTRING_END {
+			x := p.parseTest()
+			resultExpr.Args = append(resultExpr.Args, x)
+			resultExpr.StringParts = append(resultExpr.StringParts, strings.ReplaceAll(strings.ReplaceAll(p.tokval.string, "}", "}}"), "{", "{{"))
+			resultExpr.RawParts = append(resultExpr.RawParts, p.tokval.raw)
+			toktmp = p.tok
+			p.nextToken()
+		}
+		return &resultExpr
 
 	case LBRACK:
 		return p.parseList()

syntax/parse_test.go

@@ -132,6 +132,27 @@ func TestExprParseTrees(t *testing.T) {
 	}
 }
 
+func TestStmtParseFstring(t *testing.T) {
+	for _, test := range []struct {
+		input, same string
+	}{
+		{`f"hehe"`, `(ExprStmt X="hehe")`},
+		{`f"hehe{0}"`, `(ExprStmt X=(FStringExpr Raw= RawParts=(f"hehe{ }") StringParts=(hehe ) Args=(0)))`},
+		{`f"hehe{1+1}"`, `(ExprStmt X=(FStringExpr Raw= RawParts=(f"hehe{ }") StringParts=(hehe ) Args=((BinaryExpr X=1 Op=+ Y=1))))`},
+		{`f"hehe{f"haha{7}"}hehe"`, `(ExprStmt X=(FStringExpr Raw= RawParts=(f"hehe{ }hehe") StringParts=(hehe hehe) Args=((FStringExpr Raw= RawParts=(f"haha{ }") StringParts=(haha ) Args=(7)))))`},
+	} {
+		expr, err := syntax.Parse("foo.star", test.input, 0)
+		if err != nil {
+			t.Errorf("parse `%s` failed: %v", test.input, stripPos(err))
+			continue
+		}
+		got := treeString(expr.Stmts[0])
+		if got != test.same {
+			t.Errorf("parse `%s` = %s, want %s", test.input, got, test.same)
+		}
+	}
+}
+
 func TestStmtParseTrees(t *testing.T) {
 	for _, test := range []struct {
 		input, want string
@@ -362,6 +383,8 @@ func writeTree(out *bytes.Buffer, x reflect.Value) {
 		switch v := x.Interface().(type) {
 		case syntax.Literal:
 			switch v.Token {
+			case syntax.FSTRING_FULL:
+				fmt.Fprintf(out, "%q", v.Value)
 			case syntax.STRING:
 				fmt.Fprintf(out, "%q", v.Value)
 			case syntax.BYTES:

syntax/quote.go

@@ -58,6 +58,11 @@ func unquote(quoted string) (s string, triple, isByte bool, err error) {
 		quoted = quoted[1:]
 	}
 
+	if strings.HasPrefix(quoted, "f") {
+		isByte = true
+		quoted = quoted[1:]
+	}
+
 	if len(quoted) < 2 {
 		err = fmt.Errorf("string literal too short")
 		return