Proto encoding of registered types

Makefile

@@ -35,7 +35,7 @@ update_tools:
 ### Testing
 
 test:
-	go test $(shell go list ./... | grep -v vendor)
+	go test -tags extensive_tests $(shell go list ./... | grep -v vendor)
 
 gofuzz_binary:
 	rm -rf tests/fuzz/binary/corpus/

amino.go

@@ -227,24 +227,50 @@ func (cdc *Codec) MarshalBinaryBare(o interface{}) ([]byte, error) {
 		}
 		bz = buf.Bytes()
 	}
-	// If registered concrete, prepend prefix bytes.
+	// If registered concrete, prepend prefix bytes by wrapping
+	// message in RegisteredAny:
 	if info.Registered {
-		// TODO: https://github.com/tendermint/go-amino/issues/267
-		//return MarshalBinaryBare(RegisteredAny{
-		//	AminoPreOrDisfix: info.Prefix.Bytes(),
-		//	Value: bz,
-		//})
-		pb := info.Prefix.Bytes()
-		bz = append(pb, bz...)
+		return MarshalBinaryBare(RegisteredAny{
+			AminoPreOrDisfix: info.Prefix.Bytes(),
+			Value:            bz,
+		})
 	}
 
 	return bz, nil
 }
 
-//type RegisteredAny struct {
-//	AminoPreOrDisfix []byte
-//	Value []byte
+// TODO: extensively document this!
+//
+// This will be compatible to the following proto3 message:
+//
+// message AminoRegisteredAny {
+//    // Prefix or Disfix (Prefix + Disamb) bytes
+//    bytes AminoPreOrDisfix = 1;
+//    // Must be a valid serialized protocol buffer with the above specified amino pre-/disfix.
+//    bytes Value = 2;
 //}
+//
+// This is comparable to proto.Any but instead of a URL scheme there, we use amino's
+// disfix bytes instead.
+type RegisteredAny struct {
+	// Amino Prefix or Disfix (Prefix + Disamb) bytes derived from
+	// the name while registering the type.
+	// You can get this values by using the NameToDisfix helper
+	// method.
+	// In most common cases you won't need the disambiguation bytes.
+	//
+	// The prefix bytes should be 4 bytes. In case disambiguation bytes are
+	// necessary, this field will hold 7 Disfix bytes
+	// (3 disambiguation and 4 prefix bytes).
+	AminoPreOrDisfix []byte
+	// Must be a valid serialized protocol buffer that was registered
+	// with a name resulting in the above specified amino pre-/disfix bytes.
+	//
+	// Users directly using protobuf in their application need to switch
+	// over the different prefix (disfix) bytes to know which concrete
+	// type they are receiving (or encoding).
+	Value []byte
+}
 
 // Panics if error.
 func (cdc *Codec) MustMarshalBinaryBare(o interface{}) []byte {
@@ -361,14 +387,17 @@ func (cdc *Codec) UnmarshalBinaryBare(bz []byte, ptr interface{}) error {
 
 	// If registered concrete, consume and verify prefix bytes.
 	if info.Registered {
-		// TODO: https://github.com/tendermint/go-amino/issues/267
+		aminoAny := &RegisteredAny{}
+		if err := cdc.UnmarshalBinaryBare(bz, aminoAny); err != nil {
+			return err
+		}
 		pb := info.Prefix.Bytes()
-		if len(bz) < 4 {
-			return fmt.Errorf("UnmarshalBinaryBare expected to read prefix bytes %X (since it is registered concrete) but got %X", pb, bz)
-		} else if !bytes.Equal(bz[:4], pb) {
-			return fmt.Errorf("UnmarshalBinaryBare expected to read prefix bytes %X (since it is registered concrete) but got %X...", pb, bz[:4])
+		if len(aminoAny.AminoPreOrDisfix) < 4 {
+			return fmt.Errorf("UnmarshalBinaryBare expected to read prefix bytes %X (since it is registered concrete) but got %X", pb, aminoAny.AminoPreOrDisfix)
+		} else if !bytes.Equal(aminoAny.AminoPreOrDisfix[:4], pb) {
+			return fmt.Errorf("UnmarshalBinaryBare expected to read prefix bytes %X (since it is registered concrete) but got %X...", pb, aminoAny.AminoPreOrDisfix[:4])
 		}
-		bz = bz[4:]
+		bz = aminoAny.Value
 	}
 	// Only add length prefix if we have another typ3 then Typ3_ByteLength.
 	// Default is non-length prefixed:

binary-decode.go

@@ -338,11 +338,23 @@ func (cdc *Codec) decodeReflectBinaryInterface(bz []byte, iinfo *TypeInfo, rv re
 		bz = buf
 	}
 
-	// Consume disambiguation / prefix bytes.
-	disamb, hasDisamb, prefix, hasPrefix, _n, err := DecodeDisambPrefixBytes(bz)
+	// Consume disambiguation / prefix bytes by reading into a RegisteredAny message:
+	aminoAny := &RegisteredAny{}
+	if err = cdc.UnmarshalBinaryBare(bz, aminoAny); err != nil {
+		return
+	}
+	disamb, hasDisamb, prefix, hasPrefix, _n, err := DecodeDisambPrefixBytes(aminoAny.AminoPreOrDisfix)
 	if slide(&bz, &n, _n) && err != nil {
 		return
 	}
+	overhead := len(bz) - len(aminoAny.Value)
+	if overhead < 0 {
+		// we could as well panic here
+		err = errors.New("encoded value in RegisteredAny larger than original buffer")
+		return
+	}
+	slide(&bz, &n, overhead)
+	bz = aminoAny.Value
 
 	// Get concrete type info from disfix/prefix.
 	var cinfo *TypeInfo
@@ -351,7 +363,7 @@ func (cdc *Codec) decodeReflectBinaryInterface(bz []byte, iinfo *TypeInfo, rv re
 	} else if hasPrefix {
 		cinfo, err = cdc.getTypeInfoFromPrefix_rlock(iinfo, prefix)
 	} else {
-		err = errors.New("Expected disambiguation or prefix bytes.")
+		err = errors.New("expected disambiguation or prefix bytes")
 	}
 	if err != nil {
 		return

binary-encode.go

@@ -217,25 +217,25 @@ func (cdc *Codec) encodeReflectBinaryInterface(w io.Writer, iinfo *TypeInfo, rv
 			return
 		}
 	}
-
-	// Write prefix bytes.
-	_, err = buf.Write(cinfo.Prefix.Bytes())
+	aminoAny := &RegisteredAny{AminoPreOrDisfix: cinfo.Prefix.Bytes()}
+	// Write actual concrete value.
+	buf.Reset()
+	err = cdc.encodeReflectBinary(buf, cinfo, crv, fopts, true)
 	if err != nil {
 		return
 	}
-
-	// Write actual concrete value.
-	err = cdc.encodeReflectBinary(buf, cinfo, crv, fopts, true)
+	aminoAny.Value = buf.Bytes()
+	bz, err := cdc.MarshalBinaryBare(aminoAny)
 	if err != nil {
 		return
 	}
 
 	if bare {
 		// Write byteslice without byte-length prefixing.
-		_, err = w.Write(buf.Bytes())
+		_, err = w.Write(bz)
 	} else {
 		// Write byte-length prefixed byteslice.
-		err = EncodeByteSlice(w, buf.Bytes())
+		err = EncodeByteSlice(w, bz)
 	}
 	return
 }

example_test.go

@@ -65,7 +65,7 @@ func Example() {
 	fmt.Printf("Decoded successfully: %v\n", *bm == *bm2)
 
 	// Output:
-	// Encoded: 0B740613650A034142431064 (err: <nil>)
+	// Encoded: 0F0A047406136512070A034142431064 (err: <nil>)
 	// Decoded: &{ABC 100} (err: <nil>)
 	// Decoded successfully: true
 }

go.mod

@@ -4,6 +4,7 @@ go 1.12
 
 require (
 	github.com/davecgh/go-spew v1.1.1
+	github.com/gogo/protobuf v1.2.1
 	github.com/golang/protobuf v1.3.0
 	github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf
 	github.com/pkg/errors v0.8.1

go.sum

@@ -2,10 +2,14 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -17,4 +21,5 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

reflect_test.go

@@ -168,7 +168,7 @@ func TestCodecMarshalBinaryBarePassesOnRegistered(t *testing.T) {
 
 	bz, err := cdc.MarshalBinaryBare(struct{ tests.Interface1 }{tests.Concrete1{}})
 	assert.NoError(t, err, "correctly registered")
-	assert.Equal(t, []byte{0xa, 0x4, 0xe3, 0xda, 0xb8, 0x33}, bz,
+	assert.Equal(t, []byte{0xa, 0x6, 0xa, 0x4, 0xe3, 0xda, 0xb8, 0x33}, bz,
 		"prefix bytes did not match")
 }
 
@@ -179,7 +179,7 @@ func TestCodecMarshalBinaryBareOnRegisteredMatches(t *testing.T) {
 
 	bz, err := cdc.MarshalBinaryBare(struct{ tests.Interface1 }{tests.Concrete1{}})
 	assert.NoError(t, err, "correctly registered")
-	assert.Equal(t, []byte{0xa, 0x4, 0xe3, 0xda, 0xb8, 0x33}, bz,
+	assert.Equal(t, []byte{0xa, 0x6, 0xa, 0x4, 0xe3, 0xda, 0xb8, 0x33}, bz,
 		"prefix bytes did not match")
 }
 
@@ -192,7 +192,7 @@ func TestCodecMarhsalBinaryBareRegisteredAndDisamb(t *testing.T) {
 
 	bz, err := cdc.MarshalBinaryBare(struct{ tests.Interface1 }{tests.Concrete1{}})
 	assert.NoError(t, err, "correctly registered")
-	assert.Equal(t, []byte{0xa, 0x8, 0x0, 0x12, 0xb5, 0x86, 0xe3, 0xda, 0xb8, 0x33}, bz,
+	assert.Equal(t, []byte{0xa, 0x6, 0xa, 0x4, 0xe3, 0xda, 0xb8, 0x33}, bz,
 		"prefix bytes did not match")
 }
 
@@ -215,14 +215,14 @@ func TestCodecRegisterAndMarshalMultipleConcrete(t *testing.T) {
 	{ // test tests.Concrete1, no conflict.
 		bz, err := cdc.MarshalBinaryBare(struct{ tests.Interface1 }{tests.Concrete1{}})
 		assert.NoError(t, err, "correctly registered")
-		assert.Equal(t, []byte{0xa, 0x4, 0xe3, 0xda, 0xb8, 0x33}, bz,
+		assert.Equal(t, []byte{0xa, 0x6, 0xa, 0x4, 0xe3, 0xda, 0xb8, 0x33}, bz,
 			"disfix bytes did not match")
 	}
 
 	{ // test tests.Concrete2, no conflict
 		bz, err := cdc.MarshalBinaryBare(struct{ tests.Interface1 }{tests.Concrete2{}})
 		assert.NoError(t, err, "correctly registered")
-		assert.Equal(t, []byte{0xa, 0x4, 0x6a, 0x9, 0xca, 0x1}, bz,
+		assert.Equal(t, []byte{0xa, 0x6, 0xa, 0x4, 0x6a, 0x9, 0xca, 0x1}, bz,
 			"disfix bytes did not match")
 	}
 }
@@ -242,7 +242,7 @@ func TestCodecRoundtripNonNilRegisteredTypeDef(t *testing.T) {
 
 	bz, err := cdc.MarshalBinaryBare(c3)
 	assert.Nil(t, err)
-	assert.Equal(t, []byte{0x11, 0x1e, 0x93, 0x82, 0xa, 0x4, 0x30, 0x31, 0x32, 0x33}, bz,
+	assert.Equal(t, []byte{0xa, 0x4, 0x11, 0x1e, 0x93, 0x82, 0x12, 0x6, 0xa, 0x4, 0x30, 0x31, 0x32, 0x33}, bz,
 		"ConcreteTypeDef incorrectly serialized")
 
 	var i1 tests.Interface1
@@ -266,7 +266,7 @@ func TestCodecRoundtripNonNilRegisteredWrappedValue(t *testing.T) {
 
 	bz, err := cdc.MarshalBinaryBare(c3)
 	assert.Nil(t, err)
-	assert.Equal(t, []byte{0x49, 0x3f, 0xa0, 0x4b, 0xa, 0x4, 0x30, 0x31, 0x32, 0x33}, bz,
+	assert.Equal(t, []byte{0xa, 0x4, 0x49, 0x3f, 0xa0, 0x4b, 0x12, 0x6, 0xa, 0x4, 0x30, 0x31, 0x32, 0x33}, bz,
 		"ConcreteWrappedBytes incorrectly serialized")
 
 	var i1 tests.Interface1
@@ -312,7 +312,7 @@ func TestCodecRoundtripMarshalOnConcreteNonNilRegisteredTypeDef(t *testing.T) {
 
 	bz, err := cdc.MarshalBinaryBare(c3)
 	assert.Nil(t, err)
-	assert.Equal(t, []byte{0x11, 0x1e, 0x93, 0x82, 0xa, 0x4, 0x30, 0x31, 0x32, 0x33}, bz,
+	assert.Equal(t, []byte{0xa, 0x4, 0x11, 0x1e, 0x93, 0x82, 0x12, 0x6, 0xa, 0x4, 0x30, 0x31, 0x32, 0x33}, bz,
 		"ConcreteTypeDef incorrectly serialized")
 
 	var i1 tests.Interface1
@@ -332,12 +332,12 @@ func TestCodecRoundtripUnarshalOnConcreteNonNilRegisteredTypeDef(t *testing.T) {
 
 	bz, err := cdc.MarshalBinaryBare(c3a)
 	assert.Nil(t, err)
-	assert.Equal(t, []byte{0x11, 0x1e, 0x93, 0x82, 0xa, 0x4, 0x30, 0x31, 0x32, 0x33}, bz,
+	assert.Equal(t, []byte{0xa, 0x4, 0x11, 0x1e, 0x93, 0x82, 0x12, 0x6, 0xa, 0x4, 0x30, 0x31, 0x32, 0x33}, bz,
 		"ConcreteTypeDef incorrectly serialized")
 
 	var c3b tests.ConcreteTypeDef
 	err = cdc.UnmarshalBinaryBare(bz, &c3b)
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.Equal(t, c3a, c3b)
 }
 
@@ -347,10 +347,10 @@ func TestCodecBinaryStructFieldNilInterface(t *testing.T) {
 	cdc.RegisterConcrete((*tests.InterfaceFieldsStruct)(nil), "interfaceFields", nil)
 
 	i1 := &tests.InterfaceFieldsStruct{F1: new(tests.InterfaceFieldsStruct), F2: nil}
-	bz, err := cdc.MarshalBinaryLengthPrefixed(i1)
+	bz, err := cdc.MarshalBinaryBare(i1)
 
 	i2 := new(tests.InterfaceFieldsStruct)
-	err = cdc.UnmarshalBinaryLengthPrefixed(bz, i2)
+	err = cdc.UnmarshalBinaryBare(bz, i2)
 
 	assert.NoError(t, err)
 	require.Equal(t, i1, i2, "i1 and i2 should be the same after decoding")