Skip to content

go: optimize XDR decoding performance and reduce allocations #225

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tamirms wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

go: optimize XDR decoding performance and reduce allocations #225

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
560 changes: 430 additions & 130 deletions lib/xdrgen/generators/go.rb
View file
Open in desktop

Large diffs are not rendered by default.

48 changes: 20 additions & 28 deletions spec/output/generator_spec_go/block_comments.x/MyXDR_generated.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,29 +23,22 @@ var XdrFilesSHA256 = map[string]string{
"spec/fixtures/generator/block_comments.x": "e13131bc4134f38da17b9d5e9f67d2695a69ef98e3ef272833f4c18d0cc88a30",
}

var ErrMaxDecodingDepthReached = errors.New("maximum decoding depth reached")

type xdrType interface {
xdrType()
}

type decoderFrom interface {
DecodeFrom(d *xdr.Decoder, maxDepth uint) (int, error)
}

// Unmarshal reads an xdr element from `r` into `v`.
func Unmarshal(r io.Reader, v interface{}) (int, error) {
return UnmarshalWithOptions(r, v, xdr.DefaultDecodeOptions)
}
// ErrMaxDecodingDepthReached is returned when the maximum decoding depth is
// exceeded. This prevents stack overflow from deeply nested structures.
var ErrMaxDecodingDepthReached = errors.New("maximum decoding depth reached")

// UnmarshalWithOptions works like Unmarshal but uses decoding options.
func UnmarshalWithOptions(r io.Reader, v interface{}, options xdr.DecodeOptions) (int, error) {
if decodable, ok := v.(decoderFrom); ok {
d := xdr.NewDecoderWithOptions(r, options)
return decodable.DecodeFrom(d, options.MaxDepth)
// Unmarshal reads an xdr element from `data` into `v`.
func Unmarshal(data []byte, v interface{}) (int, error) {
if decodable, ok := v.(xdr.DecoderFrom); ok {
d := xdr.NewDecoder(data)
return decodable.DecodeFrom(d, d.MaxDepth())
}
// delegate to xdr package's Unmarshal
return xdr.UnmarshalWithOptions(r, v, options)
return xdr.Unmarshal(data, v)
}

// Marshal writes an xdr element `v` into `w`.
Expand Down Expand Up @@ -74,15 +67,18 @@ type AccountFlags int32
const (
AccountFlagsAuthRequiredFlag AccountFlags = 1
)
const (
_AccountFlags_Min int32 = 1
_AccountFlags_Max int32 = 1
)
var accountFlagsMap = map[int32]string{
1: "AccountFlagsAuthRequiredFlag",
}

// ValidEnum validates a proposed value for this enum. Implements
// the Enum interface for AccountFlags
func (e AccountFlags) ValidEnum(v int32) bool {
_, ok := accountFlagsMap[v]
return ok
return v >= _AccountFlags_Min && v <= _AccountFlags_Max
}
// String returns the name of `e`
func (e AccountFlags) String() string {
Expand All @@ -92,24 +88,23 @@ func (e AccountFlags) String() string {

// EncodeTo encodes this value using the Encoder.
func (e AccountFlags) EncodeTo(enc *xdr.Encoder) error {
if _, ok := accountFlagsMap[int32(e)]; !ok {
if int32(e) < _AccountFlags_Min || int32(e) > _AccountFlags_Max {
return fmt.Errorf("'%d' is not a valid AccountFlags enum value", e)
}
_, err := enc.EncodeInt(int32(e))
return err
}
var _ decoderFrom = (*AccountFlags)(nil)
// DecodeFrom decodes this value using the Decoder.
var _ xdr.DecoderFrom = (*AccountFlags)(nil)
// DecodeFrom decodes this value from the given decoder.
func (e *AccountFlags) DecodeFrom(d *xdr.Decoder, maxDepth uint) (int, error) {
if maxDepth == 0 {
return 0, fmt.Errorf("decoding AccountFlags: %w", ErrMaxDecodingDepthReached)
}
maxDepth -= 1
v, n, err := d.DecodeInt()
if err != nil {
return n, fmt.Errorf("decoding AccountFlags: %w", err)
}
if _, ok := accountFlagsMap[v]; !ok {
if v < _AccountFlags_Min || v > _AccountFlags_Max {
return n, fmt.Errorf("'%d' is not a valid AccountFlags enum value", v)
}
*e = AccountFlags(v)
Expand All @@ -125,11 +120,8 @@ func (s AccountFlags) MarshalBinary() ([]byte, error) {

// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (s *AccountFlags) UnmarshalBinary(inp []byte) error {
r := bytes.NewReader(inp)
o := xdr.DefaultDecodeOptions
o.MaxInputLen = len(inp)
d := xdr.NewDecoderWithOptions(r, o)
_, err := s.DecodeFrom(d, o.MaxDepth)
d := xdr.NewDecoder(inp)
_, err := s.DecodeFrom(d, d.MaxDepth())
return err
}

Expand Down
87 changes: 53 additions & 34 deletions spec/output/generator_spec_go/const.x/MyXDR_generated.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,29 +23,22 @@ var XdrFilesSHA256 = map[string]string{
"spec/fixtures/generator/const.x": "0bff3b37592fcc16cad2fe10b9a72f5d39d033a114917c24e86a9ebd9cda9c37",
}

var ErrMaxDecodingDepthReached = errors.New("maximum decoding depth reached")

type xdrType interface {
xdrType()
}

type decoderFrom interface {
DecodeFrom(d *xdr.Decoder, maxDepth uint) (int, error)
}

// Unmarshal reads an xdr element from `r` into `v`.
func Unmarshal(r io.Reader, v interface{}) (int, error) {
return UnmarshalWithOptions(r, v, xdr.DefaultDecodeOptions)
}
// ErrMaxDecodingDepthReached is returned when the maximum decoding depth is
// exceeded. This prevents stack overflow from deeply nested structures.
var ErrMaxDecodingDepthReached = errors.New("maximum decoding depth reached")

// UnmarshalWithOptions works like Unmarshal but uses decoding options.
func UnmarshalWithOptions(r io.Reader, v interface{}, options xdr.DecodeOptions) (int, error) {
if decodable, ok := v.(decoderFrom); ok {
d := xdr.NewDecoderWithOptions(r, options)
return decodable.DecodeFrom(d, options.MaxDepth)
// Unmarshal reads an xdr element from `data` into `v`.
func Unmarshal(data []byte, v interface{}) (int, error) {
if decodable, ok := v.(xdr.DecoderFrom); ok {
d := xdr.NewDecoder(data)
return decodable.DecodeFrom(d, d.MaxDepth())
}
// delegate to xdr package's Unmarshal
return xdr.UnmarshalWithOptions(r, v, options)
return xdr.Unmarshal(data, v)
}

// Marshal writes an xdr element `v` into `w`.
Expand Down Expand Up @@ -77,14 +70,16 @@ type TestArray [Foo]int32
// EncodeTo encodes this value using the Encoder.
func (s *TestArray) EncodeTo(e *xdr.Encoder) error {
var err error
if _, err = e.EncodeInt(int32(s)); err != nil {
for i := 0; i < len(s); i++ {
if _, err = e.EncodeInt(int32(s[i])); err != nil {
return err
}
}
return nil
}

var _ decoderFrom = (*TestArray)(nil)
// DecodeFrom decodes this value using the Decoder.
var _ xdr.DecoderFrom = (*TestArray)(nil)
// DecodeFrom decodes this value from the given decoder.
func (s *TestArray) DecodeFrom(d *xdr.Decoder, maxDepth uint) (int, error) {
if maxDepth == 0 {
return 0, fmt.Errorf("decoding TestArray: %w", ErrMaxDecodingDepthReached)
Expand All @@ -93,11 +88,13 @@ func (s *TestArray) DecodeFrom(d *xdr.Decoder, maxDepth uint) (int, error) {
var err error
var n, nTmp int
var v [Foo]int32
v, nTmp, err = d.DecodeInt()
for i := 0; i < len(v); i++ {
v[i], nTmp, err = d.DecodeInt()
n += nTmp
if err != nil {
return n, fmt.Errorf("decoding Int: %w", err)
}
}
*s = TestArray(v)
return n, nil
}
Expand All @@ -112,11 +109,8 @@ func (s TestArray) MarshalBinary() ([]byte, error) {

// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (s *TestArray) UnmarshalBinary(inp []byte) error {
r := bytes.NewReader(inp)
o := xdr.DefaultDecodeOptions
o.MaxInputLen = len(inp)
d := xdr.NewDecoderWithOptions(r, o)
_, err := s.DecodeFrom(d, o.MaxDepth)
d := xdr.NewDecoder(inp)
_, err := s.DecodeFrom(d, d.MaxDepth())
return err
}

Expand All @@ -142,14 +136,19 @@ func (e TestArray2) XDRMaxSize() int {
// EncodeTo encodes this value using the Encoder.
func (s TestArray2) EncodeTo(e *xdr.Encoder) error {
var err error
if _, err = e.EncodeInt(int32(s)); err != nil {
if _, err = e.EncodeUint(uint32(len(s))); err != nil {
return err
}
for i := 0; i < len(s); i++ {
if _, err = e.EncodeInt(int32(s[i])); err != nil {
return err
}
}
return nil
}

var _ decoderFrom = (*TestArray2)(nil)
// DecodeFrom decodes this value using the Decoder.
var _ xdr.DecoderFrom = (*TestArray2)(nil)
// DecodeFrom decodes this value from the given decoder.
func (s *TestArray2) DecodeFrom(d *xdr.Decoder, maxDepth uint) (int, error) {
if maxDepth == 0 {
return 0, fmt.Errorf("decoding TestArray2: %w", ErrMaxDecodingDepthReached)
Expand All @@ -158,10 +157,33 @@ func (s *TestArray2) DecodeFrom(d *xdr.Decoder, maxDepth uint) (int, error) {
var err error
var n, nTmp int
var v []int32
v, nTmp, err = d.DecodeInt()
var l uint32
l, nTmp, err = d.DecodeUint()
n += nTmp
if err != nil {
return n, fmt.Errorf("decoding Int: %w", err)
}
if l > 1 {
return n, fmt.Errorf("decoding int32: data size (%d) exceeds size limit (1)", l)
}
if l == 0 {
v = v[:0]
} else {
if uint(d.Remaining()) < uint(l) {
return n, fmt.Errorf("decoding int32: length (%d) exceeds remaining input length (%d)", l, d.Remaining())
}
if cap(v) >= int(l) {
v = v[:l]
} else {
v = make([]int32, l)
}
for i := uint32(0); i < l; i++ {
v[i], nTmp, err = d.DecodeInt()
n += nTmp
if err != nil {
return n, fmt.Errorf("decoding Int: %w", err)
}
}
}
*s = TestArray2(v)
return n, nil
Expand All @@ -177,11 +199,8 @@ func (s TestArray2) MarshalBinary() ([]byte, error) {

// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (s *TestArray2) UnmarshalBinary(inp []byte) error {
r := bytes.NewReader(inp)
o := xdr.DefaultDecodeOptions
o.MaxInputLen = len(inp)
d := xdr.NewDecoderWithOptions(r, o)
_, err := s.DecodeFrom(d, o.MaxDepth)
d := xdr.NewDecoder(inp)
_, err := s.DecodeFrom(d, d.MaxDepth())
return err
}

Expand Down
Loading