Skip to content

Commit 4aa5af6

Browse files
maxatomemaxatome
committed
feat(JSON): accept int_lit & float_lit numbers as defined in go spec
The following numbers are now accepted by JSON, SuperJSONOf and SubJSONOf operators: +42 → 42 4_2 → 42 0b101010 → 42 0b10_1010 → 42 0600 → 384 0_600 → 384 0o600 → 384 0O600 → 384 // second character is capital letter 'O' 0xBadFace → 195951310 0x_Bad_Face → 195951310 .25 → 0.25 1_5. → 15.0 0.15e+0_2 → 15.0 0x1p-2 → 0.25 0x2.p10 → 2048.0 0x1.Fp+0 → 1.9375 0X.8p-0 → 0.5 0X_1FFFP-16 → 0.1249847412109375 Signed-off-by: Maxime Soulé <btik-git@scoubidou.com>
1 parent 4e544bd commit 4aa5af6

4 files changed

Lines changed: 198 additions & 35 deletions

File tree

‎internal/json/lex.go‎

Lines changed: 63 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"bytes"
1111
"errors"
1212
"fmt"
13+
"math/big"
1314
"strconv"
1415
"strings"
1516
"unicode"
@@ -198,7 +199,8 @@ func (j *json) nextToken(lval *yySymType) int {
198199
return FALSE
199200
}
200201

201-
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
202+
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
203+
'+', '.': // '+' & '.' are not normally accepted by JSON spec
202204
n, ok := j.parseNumber()
203205
if !ok {
204206
return 0
@@ -337,21 +339,72 @@ str:
337339
return "", false
338340
}
339341

342+
const (
343+
numInt = 1 << iota
344+
numFloat
345+
numGoExt
346+
)
347+
348+
var numBytes = [...]uint8{
349+
'+': numInt, '-': numInt,
350+
'0': numInt,
351+
'1': numInt,
352+
'2': numInt,
353+
'3': numInt,
354+
'4': numInt,
355+
'5': numInt,
356+
'6': numInt,
357+
'7': numInt,
358+
'8': numInt,
359+
'9': numInt,
360+
'_': numGoExt,
361+
// bases 2, 8, 16
362+
'b': numInt, 'B': numInt, 'o': numInt, 'O': numInt, 'x': numInt, 'X': numInt,
363+
'a': numInt, 'A': numInt,
364+
'c': numInt, 'C': numInt,
365+
'd': numInt, 'D': numInt,
366+
'e': numInt | numFloat, 'E': numInt | numFloat,
367+
'f': numInt, 'F': numInt,
368+
// floats
369+
'.': numFloat, 'p': numFloat, 'P': numFloat,
370+
}
371+
340372
func (j *json) parseNumber() (float64, bool) {
341-
// j.buf[j.pos.bpos] == '[0-9]' → caller responsibility
373+
// j.buf[j.pos.bpos] == '[-+0-9.]' → caller responsibility
342374

375+
numKind := numBytes[j.buf[j.pos.bpos]]
343376
i := j.pos.bpos + 1
344-
l := len(j.buf)
345-
num:
346-
for ; i < l; i++ {
347-
switch j.buf[i] {
348-
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.', 'e', 'E', '+', '-':
349-
default:
350-
break num
377+
for l := len(j.buf); i < l; i++ {
378+
b := int(j.buf[i])
379+
if b >= len(numBytes) || numBytes[b] == 0 {
380+
break
381+
}
382+
numKind |= numBytes[b]
383+
}
384+
385+
s := string(j.buf[j.pos.bpos:i])
386+
387+
var (
388+
f float64
389+
err error
390+
)
391+
// Differentiate float/int parsing to accept old octal notation:
392+
// 0600 → 384 as int64, but 600 as float64
393+
if (numKind & numFloat) != 0 {
394+
// strconv.ParseFloat does not handle "_"
395+
var bf *big.Float
396+
bf, _, err = new(big.Float).Parse(s, 0)
397+
if err == nil {
398+
f, _ = bf.Float64()
399+
}
400+
} else { // numInt and/or numGoExt
401+
var int int64
402+
int, err = strconv.ParseInt(s, 0, 64)
403+
if err == nil {
404+
f = float64(int)
351405
}
352406
}
353407

354-
f, err := strconv.ParseFloat(string(j.buf[j.pos.bpos:i]), 64)
355408
if err != nil {
356409
j.fatal("invalid number")
357410
return 0, false

‎internal/json/parser_go113_test.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright (c) 2022, Maxime Soulé
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
//go:build go1.13
8+
// +build go1.13
9+
10+
package json_test
11+
12+
import (
13+
"testing"
14+
)
15+
16+
func TestJSON_go113(t *testing.T) {
17+
// Extend to golang 1.13 accepted numbers
18+
19+
// as int64
20+
checkJSON(t, `4_2`, `42`)
21+
checkJSON(t, `+4_2`, `42`)
22+
checkJSON(t, `-4_2`, `-42`)
23+
24+
checkJSON(t, `0b101010`, `42`)
25+
checkJSON(t, `-0b101010`, `-42`)
26+
checkJSON(t, `+0b101010`, `42`)
27+
28+
checkJSON(t, `0b10_1010`, `42`)
29+
checkJSON(t, `-0b_10_1010`, `-42`)
30+
checkJSON(t, `+0b10_10_10`, `42`)
31+
32+
checkJSON(t, `0B101010`, `42`)
33+
checkJSON(t, `-0B101010`, `-42`)
34+
checkJSON(t, `+0B101010`, `42`)
35+
36+
checkJSON(t, `0B10_1010`, `42`)
37+
checkJSON(t, `-0B_10_1010`, `-42`)
38+
checkJSON(t, `+0B10_10_10`, `42`)
39+
40+
checkJSON(t, `0_600`, `384`)
41+
checkJSON(t, `-0_600`, `-384`)
42+
checkJSON(t, `+0_600`, `384`)
43+
44+
checkJSON(t, `0o600`, `384`)
45+
checkJSON(t, `0o_600`, `384`)
46+
checkJSON(t, `-0o600`, `-384`)
47+
checkJSON(t, `-0o6_00`, `-384`)
48+
checkJSON(t, `+0o600`, `384`)
49+
checkJSON(t, `+0o60_0`, `384`)
50+
51+
checkJSON(t, `0O600`, `384`)
52+
checkJSON(t, `0O_600`, `384`)
53+
checkJSON(t, `-0O600`, `-384`)
54+
checkJSON(t, `-0O6_00`, `-384`)
55+
checkJSON(t, `+0O600`, `384`)
56+
checkJSON(t, `+0O60_0`, `384`)
57+
58+
checkJSON(t, `0xBad_Face`, `195951310`)
59+
checkJSON(t, `-0x_Bad_Face`, `-195951310`)
60+
checkJSON(t, `+0xBad_Face`, `195951310`)
61+
62+
checkJSON(t, `0XBad_Face`, `195951310`)
63+
checkJSON(t, `-0X_Bad_Face`, `-195951310`)
64+
checkJSON(t, `+0XBad_Face`, `195951310`)
65+
66+
// as float64
67+
checkJSON(t, `0_600.123`, `600.123`) // float64 can not be an octal number
68+
checkJSON(t, `1_5.`, `15`)
69+
checkJSON(t, `0.15e+0_2`, `15`)
70+
checkJSON(t, `0x1p-2`, `0.25`)
71+
checkJSON(t, `0x2.p10`, `2048`)
72+
checkJSON(t, `0x1.Fp+0`, `1.9375`)
73+
checkJSON(t, `0X.8p-0`, `0.5`)
74+
checkJSON(t, `0X_1FFFP-16`, `0.1249847412109375`)
75+
}

‎internal/json/parser_test.go‎

Lines changed: 45 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2020, 2021, Maxime Soulé
1+
// Copyright (c) 2020-2022, Maxime Soulé
22
// All rights reserved.
33
//
44
// This source code is licensed under the BSD-style license found in the
@@ -19,6 +19,28 @@ import (
1919
"github.com/maxatome/go-testdeep/internal/test"
2020
)
2121

22+
func checkJSON(t *testing.T, gotJSON, expectedJSON string) {
23+
t.Helper()
24+
25+
var expected interface{}
26+
err := ejson.Unmarshal([]byte(expectedJSON), &expected)
27+
if err != nil {
28+
t.Fatalf("bad JSON: %s", err)
29+
}
30+
31+
got, err := json.Parse([]byte(gotJSON))
32+
if !test.NoError(t, err, "json.Parse succeeds") {
33+
return
34+
}
35+
if !reflect.DeepEqual(got, expected) {
36+
test.EqualErrorMessage(t,
37+
strings.TrimRight(spew.Sdump(got), "\n"),
38+
strings.TrimRight(spew.Sdump(expected), "\n"),
39+
"got matches expected",
40+
)
41+
}
42+
}
43+
2244
func TestJSON(t *testing.T) {
2345
t.Run("Basics", func(t *testing.T) {
2446
for i, js := range []string{
@@ -83,31 +105,29 @@ func TestJSON(t *testing.T) {
83105
})
84106

85107
t.Run("JSON spec infringements", func(t *testing.T) {
86-
check := func(gotJSON, expectedJSON string) {
87-
t.Helper()
88-
var expected interface{}
89-
err := ejson.Unmarshal([]byte(expectedJSON), &expected)
90-
if err != nil {
91-
t.Fatalf("bad JSON: %s", err)
92-
}
93-
94-
got, err := json.Parse([]byte(gotJSON))
95-
if !test.NoError(t, err, "json.Parse succeeds") {
96-
return
97-
}
98-
if !reflect.DeepEqual(got, expected) {
99-
test.EqualErrorMessage(t,
100-
strings.TrimRight(spew.Sdump(got), "\n"),
101-
strings.TrimRight(spew.Sdump(expected), "\n"),
102-
"got matches expected",
103-
)
104-
}
105-
}
106108
// "," is accepted just before non-empty "}" or "]"
107-
check(`{"foo": "bar", }`, `{"foo":"bar"}`)
108-
check(`{"foo":"bar",}`, `{"foo":"bar"}`)
109-
check(`[ 1, 2, 3, ]`, `[1,2,3]`)
110-
check(`[ 1,2,3,]`, `[1,2,3]`)
109+
checkJSON(t, `{"foo": "bar", }`, `{"foo":"bar"}`)
110+
checkJSON(t, `{"foo":"bar",}`, `{"foo":"bar"}`)
111+
checkJSON(t, `[ 1, 2, 3, ]`, `[1,2,3]`)
112+
checkJSON(t, `[ 1,2,3,]`, `[1,2,3]`)
113+
114+
// Extend to golang accepted numbers
115+
// as int64
116+
checkJSON(t, `+42`, `42`)
117+
118+
checkJSON(t, `0600`, `384`)
119+
checkJSON(t, `-0600`, `-384`)
120+
checkJSON(t, `+0600`, `384`)
121+
122+
checkJSON(t, `0xBadFace`, `195951310`)
123+
checkJSON(t, `-0xBadFace`, `-195951310`)
124+
checkJSON(t, `+0xBadFace`, `195951310`)
125+
126+
// as float64
127+
checkJSON(t, `0600.123`, `600.123`) // float64 can not be an octal number
128+
checkJSON(t, `0600.`, `600`) // float64 can not be an octal number
129+
checkJSON(t, `.25`, `0.25`)
130+
checkJSON(t, `+123.`, `123`)
111131
})
112132

113133
t.Run("Special string cases", func(t *testing.T) {

‎td/td_json.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -581,6 +581,11 @@ func jsonify(ctx ctxerr.Context, got reflect.Value) (interface{}, *ctxerr.Error)
581581
// - multi-lines comments start with the character sequence /* and stop
582582
// with the first subsequent character sequence */.
583583
//
584+
// Other JSON divergences:
585+
// - ',' can precede a '}' or a ']' (as in go);
586+
// - int_lit & float_lit numbers as defined in go spec are accepted;
587+
// - numbers can be prefixed by '+'.
588+
//
584589
// Most operators can be directly embedded in JSON without requiring
585590
// any placeholder.
586591
//
@@ -841,6 +846,11 @@ var _ TestDeep = &tdMapJSON{}
841846
// - multi-lines comments start with the character sequence /* and stop
842847
// with the first subsequent character sequence */.
843848
//
849+
// Other JSON divergences:
850+
// - ',' can precede a '}' or a ']' (as in go);
851+
// - int_lit & float_lit numbers as defined in go spec are accepted;
852+
// - numbers can be prefixed by '+'.
853+
//
844854
// Most operators can be directly embedded in SubJSONOf without requiring
845855
// any placeholder.
846856
//
@@ -1055,6 +1065,11 @@ func SubJSONOf(expectedJSON interface{}, params ...interface{}) TestDeep {
10551065
// - multi-lines comments start with the character sequence /* and stop
10561066
// with the first subsequent character sequence */.
10571067
//
1068+
// Other JSON divergences:
1069+
// - ',' can precede a '}' or a ']' (as in go);
1070+
// - int_lit & float_lit numbers as defined in go spec are accepted;
1071+
// - numbers can be prefixed by '+'.
1072+
//
10581073
// Most operators can be directly embedded in SuperJSONOf without requiring
10591074
// any placeholder.
10601075
//

0 commit comments

Comments
 (0)