Skip to content

Commit e45c636

Browse files
brendarbrendar
authored
Merge pull request #169 from Shopify/v15.0.3-shopify-12-candidate
Backport: VStreamer: improve representation of integers in json data types (2b43fd7)
2 parents 84ea974 + d0b5367 commit e45c636

File tree

7 files changed

+249
-79
lines changed

7 files changed

+249
-79
lines changed

‎go.mod

+3-1
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ require (
7171
github.com/spf13/cobra v1.4.0
7272
github.com/spf13/pflag v1.0.5
7373
github.com/spf13/viper v1.8.1
74-
github.com/spyzhov/ajson v0.4.2
7574
github.com/stretchr/testify v1.7.1
7675
github.com/tchap/go-patricia v2.2.6+incompatible
7776
github.com/tebeka/selenium v0.9.9
@@ -118,6 +117,7 @@ require (
118117
require (
119118
github.com/bndr/gotabulate v1.1.2
120119
github.com/hashicorp/go-version v1.6.0
120+
github.com/spyzhov/ajson v0.8.0
121121
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
122122
modernc.org/sqlite v1.20.3
123123
)
@@ -202,3 +202,5 @@ require (
202202
modernc.org/token v1.0.1 // indirect
203203
sigs.k8s.io/structured-merge-diff/v4 v4.0.3 // indirect
204204
)
205+
206+
replace github.com/spyzhov/ajson v0.8.0 => github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
650650
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
651651
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
652652
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
653+
github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c h1:Y/4qcogoZA2WUtLWMk/yXfJSpaIG3mK3r9Lw4kaARL4=
654+
github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzyqMuVA=
653655
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
654656
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
655657
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -695,8 +697,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
695697
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
696698
github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
697699
github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
698-
github.com/spyzhov/ajson v0.4.2 h1:JMByd/jZApPKDvNsmO90X2WWGbmT2ahDFp73QhZbg3s=
699-
github.com/spyzhov/ajson v0.4.2/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzyqMuVA=
700700
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
701701
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
702702
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=

‎go/mysql/binlog_event_json.go

+147-49
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ func (jh *BinlogJSON) register(typ jsonDataType, Plugin jsonPlugin) {
153153
func (jh *BinlogJSON) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
154154
Plugin := jh.plugins[typ]
155155
if Plugin == nil {
156-
return nil, fmt.Errorf("Plugin not found for type %d", typ)
156+
return nil, fmt.Errorf("plugin not found for type %d", typ)
157157
}
158158
return Plugin.getNode(typ, data, pos)
159159
}
@@ -316,59 +316,157 @@ type intPlugin struct {
316316

317317
var _ jsonPlugin = (*intPlugin)(nil)
318318

319-
func (ih intPlugin) getVal(typ jsonDataType, data []byte, pos int) (value float64) {
319+
func (ipl intPlugin) getVal(typ jsonDataType, data []byte, pos int) (value int64) {
320320
var val uint64
321-
var val2 float64
322-
size := ih.sizes[typ]
321+
var val2 int64
322+
size := ipl.sizes[typ]
323323
for i := 0; i < size; i++ {
324324
val = val + uint64(data[pos+i])<<(8*i)
325325
}
326326
switch typ {
327327
case jsonInt16:
328-
val2 = float64(int16(val))
329-
case jsonUint16:
330-
val2 = float64(uint16(val))
328+
val2 = int64(int16(val))
331329
case jsonInt32:
332-
val2 = float64(int32(val))
333-
case jsonUint32:
334-
val2 = float64(uint32(val))
330+
val2 = int64(int32(val))
335331
case jsonInt64:
336-
val2 = float64(int64(val))
337-
case jsonUint64:
338-
val2 = float64(val)
339-
case jsonDouble:
340-
val2 = math.Float64frombits(val)
332+
val2 = int64(val)
341333
}
342334
return val2
343335
}
344336

345-
func (ih intPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
346-
val := ih.getVal(typ, data, pos)
347-
node = ajson.NumericNode("", val)
337+
func (ipl intPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
338+
val := ipl.getVal(typ, data, pos)
339+
node = ajson.IntegerNode("", val)
348340
return node, nil
349341
}
350342

351343
func newIntPlugin() *intPlugin {
352-
ih := &intPlugin{
344+
ipl := &intPlugin{
353345
info: &jsonPluginInfo{
354346
name: "Int",
355-
types: []jsonDataType{jsonInt64, jsonInt32, jsonInt16, jsonUint16, jsonUint32, jsonUint64, jsonDouble},
347+
types: []jsonDataType{jsonInt64, jsonInt32, jsonInt16},
348+
},
349+
sizes: make(map[jsonDataType]int),
350+
}
351+
ipl.sizes = map[jsonDataType]int{
352+
jsonInt64: 8,
353+
jsonInt32: 4,
354+
jsonInt16: 2,
355+
}
356+
for _, typ := range ipl.info.types {
357+
binlogJSON.register(typ, ipl)
358+
}
359+
return ipl
360+
}
361+
362+
//endregion
363+
364+
//region uint plugin
365+
366+
func init() {
367+
newUintPlugin()
368+
}
369+
370+
type uintPlugin struct {
371+
info *jsonPluginInfo
372+
sizes map[jsonDataType]int
373+
}
374+
375+
var _ jsonPlugin = (*uintPlugin)(nil)
376+
377+
func (upl uintPlugin) getVal(typ jsonDataType, data []byte, pos int) (value uint64) {
378+
var val uint64
379+
var val2 uint64
380+
size := upl.sizes[typ]
381+
for i := 0; i < size; i++ {
382+
val = val + uint64(data[pos+i])<<(8*i)
383+
}
384+
switch typ {
385+
case jsonUint16:
386+
val2 = uint64(uint16(val))
387+
case jsonUint32:
388+
val2 = uint64(uint32(val))
389+
case jsonUint64:
390+
val2 = val
391+
}
392+
return val2
393+
}
394+
395+
func (upl uintPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
396+
val := upl.getVal(typ, data, pos)
397+
node = ajson.UnsignedIntegerNode("", val)
398+
return node, nil
399+
}
400+
401+
func newUintPlugin() *uintPlugin {
402+
upl := &uintPlugin{
403+
info: &jsonPluginInfo{
404+
name: "Uint",
405+
types: []jsonDataType{jsonUint16, jsonUint32, jsonUint64},
356406
},
357407
sizes: make(map[jsonDataType]int),
358408
}
359-
ih.sizes = map[jsonDataType]int{
409+
upl.sizes = map[jsonDataType]int{
360410
jsonUint64: 8,
361-
jsonInt64: 8,
362411
jsonUint32: 4,
363-
jsonInt32: 4,
364412
jsonUint16: 2,
365-
jsonInt16: 2,
413+
}
414+
for _, typ := range upl.info.types {
415+
binlogJSON.register(typ, upl)
416+
}
417+
return upl
418+
}
419+
420+
//endregion
421+
422+
//region float plugin
423+
424+
func init() {
425+
newFloatPlugin()
426+
}
427+
428+
type floatPlugin struct {
429+
info *jsonPluginInfo
430+
sizes map[jsonDataType]int
431+
}
432+
433+
var _ jsonPlugin = (*floatPlugin)(nil)
434+
435+
func (flp floatPlugin) getVal(typ jsonDataType, data []byte, pos int) (value float64) {
436+
var val uint64
437+
var val2 float64
438+
size := flp.sizes[typ]
439+
for i := 0; i < size; i++ {
440+
val = val + uint64(data[pos+i])<<(8*i)
441+
}
442+
switch typ {
443+
case jsonDouble:
444+
val2 = math.Float64frombits(val)
445+
}
446+
return val2
447+
}
448+
449+
func (flp floatPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
450+
val := flp.getVal(typ, data, pos)
451+
node = ajson.NumericNode("", val)
452+
return node, nil
453+
}
454+
455+
func newFloatPlugin() *floatPlugin {
456+
fp := &floatPlugin{
457+
info: &jsonPluginInfo{
458+
name: "Float",
459+
types: []jsonDataType{jsonDouble},
460+
},
461+
sizes: make(map[jsonDataType]int),
462+
}
463+
fp.sizes = map[jsonDataType]int{
366464
jsonDouble: 8,
367465
}
368-
for _, typ := range ih.info.types {
369-
binlogJSON.register(typ, ih)
466+
for _, typ := range fp.info.types {
467+
binlogJSON.register(typ, fp)
370468
}
371-
return ih
469+
return fp
372470
}
373471

374472
//endregion
@@ -385,7 +483,7 @@ type literalPlugin struct {
385483

386484
var _ jsonPlugin = (*literalPlugin)(nil)
387485

388-
func (lh literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
486+
func (lpl literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
389487
val := jsonDataLiteral(data[pos])
390488
switch val {
391489
case jsonNullLiteral:
@@ -401,14 +499,14 @@ func (lh literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *a
401499
}
402500

403501
func newLiteralPlugin() *literalPlugin {
404-
lh := &literalPlugin{
502+
lpl := &literalPlugin{
405503
info: &jsonPluginInfo{
406504
name: "Literal",
407505
types: []jsonDataType{jsonLiteral},
408506
},
409507
}
410-
binlogJSON.register(jsonLiteral, lh)
411-
return lh
508+
binlogJSON.register(jsonLiteral, lpl)
509+
return lpl
412510
}
413511

414512
//endregion
@@ -427,7 +525,7 @@ var _ jsonPlugin = (*opaquePlugin)(nil)
427525

428526
// other types are stored as catch-all opaque types: documentation on these is scarce.
429527
// we currently know about (and support) date/time/datetime/decimal.
430-
func (oh opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
528+
func (opl opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
431529
dataType := data[pos]
432530
start := 3 // account for length of stored value
433531
end := start + 8 // all currently supported opaque data types are 8 bytes in size
@@ -484,14 +582,14 @@ func (oh opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *aj
484582
}
485583

486584
func newOpaquePlugin() *opaquePlugin {
487-
oh := &opaquePlugin{
585+
opl := &opaquePlugin{
488586
info: &jsonPluginInfo{
489587
name: "Opaque",
490588
types: []jsonDataType{jsonOpaque},
491589
},
492590
}
493-
binlogJSON.register(jsonOpaque, oh)
494-
return oh
591+
binlogJSON.register(jsonOpaque, opl)
592+
return opl
495593
}
496594

497595
//endregion
@@ -508,22 +606,22 @@ type stringPlugin struct {
508606

509607
var _ jsonPlugin = (*stringPlugin)(nil)
510608

511-
func (sh stringPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
609+
func (spl stringPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
512610
size, pos := readVariableLength(data, pos)
513611
node = ajson.StringNode("", string(data[pos:pos+size]))
514612

515613
return node, nil
516614
}
517615

518616
func newStringPlugin() *stringPlugin {
519-
sh := &stringPlugin{
617+
spl := &stringPlugin{
520618
info: &jsonPluginInfo{
521619
name: "String",
522620
types: []jsonDataType{jsonString},
523621
},
524622
}
525-
binlogJSON.register(jsonString, sh)
526-
return sh
623+
binlogJSON.register(jsonString, spl)
624+
return spl
527625
}
528626

529627
//endregion
@@ -542,7 +640,7 @@ var _ jsonPlugin = (*arrayPlugin)(nil)
542640

543641
// arrays are stored thus:
544642
// | type_identifier(one of [2,3]) | elem count | obj size | list of offsets+lengths of values | actual values |
545-
func (ah arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
643+
func (apl arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
546644
jlog("JSON Array %s, len %d", jsonDataTypeToString(uint(typ)), len(data))
547645
var nodes []*ajson.Node
548646
var elem *ajson.Node
@@ -565,15 +663,15 @@ func (ah arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajs
565663
}
566664

567665
func newArrayPlugin() *arrayPlugin {
568-
ah := &arrayPlugin{
666+
apl := &arrayPlugin{
569667
info: &jsonPluginInfo{
570668
name: "Array",
571669
types: []jsonDataType{jsonSmallArray, jsonLargeArray},
572670
},
573671
}
574-
binlogJSON.register(jsonSmallArray, ah)
575-
binlogJSON.register(jsonLargeArray, ah)
576-
return ah
672+
binlogJSON.register(jsonSmallArray, apl)
673+
binlogJSON.register(jsonLargeArray, apl)
674+
return apl
577675
}
578676

579677
//endregion
@@ -592,7 +690,7 @@ var _ jsonPlugin = (*objectPlugin)(nil)
592690

593691
// objects are stored thus:
594692
// | type_identifier(0/1) | elem count | obj size | list of offsets+lengths of keys | list of offsets+lengths of values | actual keys | actual values |
595-
func (oh objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
693+
func (opl objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) {
596694
jlog("JSON Type is %s, len %d", jsonDataTypeToString(uint(typ)), len(data))
597695

598696
// "large" decides number of bytes used to specify element count and total object size: 4 bytes for large, 2 for small
@@ -640,15 +738,15 @@ func (oh objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *aj
640738
}
641739

642740
func newObjectPlugin() *objectPlugin {
643-
oh := &objectPlugin{
741+
opl := &objectPlugin{
644742
info: &jsonPluginInfo{
645743
name: "Object",
646744
types: []jsonDataType{jsonSmallObject, jsonLargeObject},
647745
},
648746
}
649-
binlogJSON.register(jsonSmallObject, oh)
650-
binlogJSON.register(jsonLargeObject, oh)
651-
return oh
747+
binlogJSON.register(jsonSmallObject, opl)
748+
binlogJSON.register(jsonLargeObject, opl)
749+
return opl
652750
}
653751

654752
//endregion

0 commit comments

Comments
 (0)