Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

encoding/json: improve decoder alloc count #71475

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+48 −40
Open

encoding/json: improve decoder alloc count #71475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19c9367
encoding/json: improve decoder alloc count
oiweiwei Jan 29, 2025
b0c5899
fix: ut
oiweiwei Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions src/encoding/json/decode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,21 +467,21 @@ var unmarshalTests = []struct {
{CaseName: Name(""), in: `{"alphabet": "xyz"}`, ptr: new(U), err: fmt.Errorf("json: unknown field \"alphabet\""), disallowUnknownFields: true},

// syntax errors
{CaseName: Name(""), in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}},
{CaseName: Name(""), in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}},
{CaseName: Name(""), in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true},
{CaseName: Name(""), in: `{"X": "foo", "Y"}`, err: &SyntaxError{invalidChar: '}', invalidCharContext: "after object key", Offset: 17}},
{CaseName: Name(""), in: `[1, 2, 3+]`, err: &SyntaxError{invalidChar: '+', invalidCharContext: "after array element", Offset: 9}},
{CaseName: Name(""), in: `{"X":12x}`, err: &SyntaxError{invalidChar: 'x', invalidCharContext: "after object key:value pair", Offset: 8}, useNumber: true},
{CaseName: Name(""), in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}},
{CaseName: Name(""), in: `{"F3": -}`, ptr: new(V), err: &SyntaxError{msg: "invalid character '}' in numeric literal", Offset: 9}},
{CaseName: Name(""), in: `{"F3": -}`, ptr: new(V), err: &SyntaxError{invalidChar: '}', invalidCharContext: "in numeric literal", Offset: 9}},

// raw value errors
{CaseName: Name(""), in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}},
{CaseName: Name(""), in: " 42 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 5}},
{CaseName: Name(""), in: "\x01 true", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}},
{CaseName: Name(""), in: " false \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 8}},
{CaseName: Name(""), in: "\x01 1.2", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}},
{CaseName: Name(""), in: " 3.4 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 6}},
{CaseName: Name(""), in: "\x01 \"string\"", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}},
{CaseName: Name(""), in: " \"string\" \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 11}},
{CaseName: Name(""), in: "\x01 42", err: &SyntaxError{invalidChar: '\x01', invalidCharContext: "looking for beginning of value", Offset: 1}},
{CaseName: Name(""), in: " 42 \x01", err: &SyntaxError{invalidChar: '\x01', invalidCharContext: "after top-level value", Offset: 5}},
{CaseName: Name(""), in: "\x01 true", err: &SyntaxError{invalidChar: '\x01', invalidCharContext: "looking for beginning of value", Offset: 1}},
{CaseName: Name(""), in: " false \x01", err: &SyntaxError{invalidChar: '\x01', invalidCharContext: "after top-level value", Offset: 8}},
{CaseName: Name(""), in: "\x01 1.2", err: &SyntaxError{invalidChar: '\x01', invalidCharContext: "looking for beginning of value", Offset: 1}},
{CaseName: Name(""), in: " 3.4 \x01", err: &SyntaxError{invalidChar: '\x01', invalidCharContext: "after top-level value", Offset: 6}},
{CaseName: Name(""), in: "\x01 \"string\"", err: &SyntaxError{invalidChar: '\x01', invalidCharContext: "looking for beginning of value", Offset: 1}},
{CaseName: Name(""), in: " \"string\" \x01", err: &SyntaxError{invalidChar: '\x01', invalidCharContext: "after top-level value", Offset: 11}},

// array tests
{CaseName: Name(""), in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}},
Expand Down Expand Up @@ -1096,8 +1096,9 @@ var unmarshalTests = []struct {
in: `invalid`,
ptr: new(Number),
err: &SyntaxError{
msg: "invalid character 'i' looking for beginning of value",
Offset: 1,
invalidChar: 'i',
invalidCharContext: "looking for beginning of value",
Offset: 1,
},
},
{
Expand Down Expand Up @@ -1178,7 +1179,7 @@ var unmarshalTests = []struct {
CaseName: Name(""),
in: `[1,2,true,4,5}`,
ptr: new([]int),
err: &SyntaxError{msg: "invalid character '}' after array element", Offset: 14},
err: &SyntaxError{invalidChar: '}', invalidCharContext: "after array element", Offset: 14},
},
{
CaseName: Name(""),
Expand Down Expand Up @@ -2589,23 +2590,23 @@ func TestUnmarshalErrorAfterMultipleJSON(t *testing.T) {
}{{
CaseName: Name(""),
in: `1 false null :`,
err: &SyntaxError{"invalid character ':' looking for beginning of value", 14},
err: &SyntaxError{invalidChar: ':', invalidCharContext: "looking for beginning of value", Offset: 14},
}, {
CaseName: Name(""),
in: `1 [] [,]`,
err: &SyntaxError{"invalid character ',' looking for beginning of value", 7},
err: &SyntaxError{invalidChar: ',', invalidCharContext: "looking for beginning of value", Offset: 7},
}, {
CaseName: Name(""),
in: `1 [] [true:]`,
err: &SyntaxError{"invalid character ':' after array element", 11},
err: &SyntaxError{invalidChar: ':', invalidCharContext: "after array element", Offset: 11},
}, {
CaseName: Name(""),
in: `1 {} {"x"=}`,
err: &SyntaxError{"invalid character '=' after object key", 14},
err: &SyntaxError{invalidChar: '=', invalidCharContext: "after object key", Offset: 14},
}, {
CaseName: Name(""),
in: `falsetruenul#`,
err: &SyntaxError{"invalid character '#' in literal null (expecting 'l')", 13},
err: &SyntaxError{invalidChar: '#', invalidCharContext: "in literal null (expecting 'l')", Offset: 13},
}}
for _, tt := range tests {
t.Run(tt.Name, func(t *testing.T) {
Expand Down
17 changes: 12 additions & 5 deletions src/encoding/json/scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,18 @@ func checkValid(data []byte, scan *scanner) error {
// A SyntaxError is a description of a JSON syntax error.
// [Unmarshal] will return a SyntaxError if the JSON can't be parsed.
type SyntaxError struct {
msg string // description of error
Offset int64 // error occurred after reading Offset bytes
msg string // description of error
Offset int64 // error occurred after reading Offset bytes
invalidCharContext string // invalid character error context
invalidChar byte // the invalid character
}

func (e *SyntaxError) Error() string { return e.msg }
func (e *SyntaxError) Error() string {
if e.invalidCharContext != "" {
return "invalid character " + quoteChar(e.invalidChar) + " " + e.invalidCharContext
}
return e.msg
}

// A scanner is a JSON scanning state machine.
// Callers call scan.reset and then pass bytes in one at a time
Expand Down Expand Up @@ -168,7 +175,7 @@ func (s *scanner) eof() int {
return scanEnd
}
if s.err == nil {
s.err = &SyntaxError{"unexpected end of JSON input", s.bytes}
s.err = &SyntaxError{"unexpected end of JSON input", s.bytes, "", 0}
}
return scanError
}
Expand Down Expand Up @@ -590,7 +597,7 @@ func stateError(s *scanner, c byte) int {
// error records an error and switches to the error state.
func (s *scanner) error(c byte, context string) int {
s.step = stateError
s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes}
s.err = &SyntaxError{invalidCharContext: context, invalidChar: c, Offset: s.bytes}
return scanError
}

Expand Down
4 changes: 2 additions & 2 deletions src/encoding/json/scanner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,8 @@ func TestIndentErrors(t *testing.T) {
in string
err error
}{
{Name(""), `{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
{Name(""), `{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
{Name(""), `{"X": "foo", "Y"}`, &SyntaxError{invalidChar: '}', invalidCharContext: "after object key", Offset: 17}},
{Name(""), `{"X": "foo" "Y": "bar"}`, &SyntaxError{invalidChar: '"', invalidCharContext: "after object key:value pair", Offset: 13}},
}
for _, tt := range tests {
t.Run(tt.Name, func(t *testing.T) {
Expand Down
18 changes: 9 additions & 9 deletions src/encoding/json/stream.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ func (dec *Decoder) tokenPrepareForDecode() error {
return err
}
if c != ',' {
return &SyntaxError{"expected comma after array element", dec.InputOffset()}
return &SyntaxError{"expected comma after array element", dec.InputOffset(), "", 0}
}
dec.scanp++
dec.tokenState = tokenArrayValue
Expand All @@ -322,7 +322,7 @@ func (dec *Decoder) tokenPrepareForDecode() error {
return err
}
if c != ':' {
return &SyntaxError{"expected colon after object key", dec.InputOffset()}
return &SyntaxError{"expected colon after object key", dec.InputOffset(), "", 0}
}
dec.scanp++
dec.tokenState = tokenObjectValue
Expand Down Expand Up @@ -463,19 +463,19 @@ func (dec *Decoder) tokenError(c byte) (Token, error) {
var context string
switch dec.tokenState {
case tokenTopValue:
context = " looking for beginning of value"
context = "looking for beginning of value"
case tokenArrayStart, tokenArrayValue, tokenObjectValue:
context = " looking for beginning of value"
context = "looking for beginning of value"
case tokenArrayComma:
context = " after array element"
context = "after array element"
case tokenObjectKey:
context = " looking for beginning of object key string"
context = "looking for beginning of object key string"
case tokenObjectColon:
context = " after object key"
context = "after object key"
case tokenObjectComma:
context = " after object key:value pair"
context = "after object key:value pair"
}
return nil, &SyntaxError{"invalid character " + quoteChar(c) + context, dec.InputOffset()}
return nil, &SyntaxError{invalidChar: c, invalidCharContext: context, Offset: dec.InputOffset()}
}

// More reports whether there is another element in the
Expand Down
8 changes: 4 additions & 4 deletions src/encoding/json/stream_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,18 +444,18 @@ func TestDecodeInStream(t *testing.T) {
{CaseName: Name(""), json: ` [{"a": 1} {"a": 2}] `, expTokens: []any{
Delim('['),
decodeThis{map[string]any{"a": float64(1)}},
decodeThis{&SyntaxError{"expected comma after array element", 11}},
decodeThis{&SyntaxError{"expected comma after array element", 11, "", 0}},
}},
{CaseName: Name(""), json: `{ "` + strings.Repeat("a", 513) + `" 1 }`, expTokens: []any{
Delim('{'), strings.Repeat("a", 513),
decodeThis{&SyntaxError{"expected colon after object key", 518}},
decodeThis{&SyntaxError{"expected colon after object key", 518, "", 0}},
}},
{CaseName: Name(""), json: `{ "\a" }`, expTokens: []any{
Delim('{'),
&SyntaxError{"invalid character 'a' in string escape code", 3},
&SyntaxError{invalidChar: 'a', invalidCharContext: "in string escape code", Offset: 3},
}},
{CaseName: Name(""), json: ` \a`, expTokens: []any{
&SyntaxError{"invalid character '\\\\' looking for beginning of value", 1},
&SyntaxError{invalidChar: '\\', invalidCharContext: "looking for beginning of value", Offset: 1},
}},
}
for _, tt := range tests {
Expand Down