Merge pull request #12 from loopholelabs/staging

Release v0.5.0

Author: ShivanshVij

.github/workflows/version.yaml

@@ -3,7 +3,7 @@ name: publish
 on:
   push:
     branches:
-      - main
+      - staging
 
 jobs:
   build:

buffer_test.go

@@ -431,5 +431,5 @@ func TestLen(t *testing.T) {
 	p := NewBuffer()
 	Encoder(p).String("Hello World")
 
-	assert.Equal(t, 17, p.Len())
+	assert.Equal(t, 14, p.Len())
 }

decode.go

@@ -22,7 +22,11 @@ import (
 )
 
 const (
-	emptyString = ""
+	emptyString  = ""
+	VarIntLen16  = 3
+	VarIntLen32  = 5
+	VarIntLen64  = 10
+	continuation = 0x80
 )
 
 var (
@@ -162,48 +166,121 @@ func decodeUint8(b []byte) ([]byte, uint8, error) {
 	return b, 0, InvalidUint8
 }
 
+// Variable integer encoding with the same format as binary.varint
+//(https://developers.google.com/protocol-buffers/docs/encoding#varints)
 func decodeUint16(b []byte) ([]byte, uint16, error) {
-	if len(b) > 2 {
-		if b[0] == Uint16Kind[0] {
-			return b[3:], uint16(b[2]) | uint16(b[1])<<8, nil
+	if len(b) > 1 && b[0] == Uint16Kind[0] {
+		var x uint16
+		var s uint
+		for i := 1; i < VarIntLen16+1; i++ {
+			cb := b[i]
+			// Check if msb is set signifying a continuation byte
+			if cb < continuation {
+				if i > VarIntLen16 && cb > 1 {
+					return b, 0, InvalidUint32
+				}
+				// End of varint, add the last bits and advance the buffer
+				return b[i+1:], x | uint16(cb)<<s, nil
+			}
+			x |= uint16(cb&(continuation-1)) << s
+			s += 7
 		}
 	}
 	return b, 0, InvalidUint16
 }
 
 func decodeUint32(b []byte) ([]byte, uint32, error) {
-	if len(b) > 4 {
-		if b[0] == Uint32Kind[0] {
-			return b[5:], uint32(b[4]) | uint32(b[3])<<8 | uint32(b[2])<<16 | uint32(b[1])<<24, nil
+	if len(b) > 1 && b[0] == Uint32Kind[0] {
+		var x uint32
+		var s uint
+		for i := 1; i < VarIntLen32+1; i++ {
+			cb := b[i]
+			// Check if msb is set signifying a continuation byte
+			if cb < continuation {
+				if i > VarIntLen32 && cb > 1 {
+					return b, 0, InvalidUint32
+				}
+				// End of varint, add the last bits and advance the buffer
+				return b[i+1:], x | uint32(cb)<<s, nil
+			}
+			// Add the lower 7 bits to the result and continue to the next byte
+			x |= uint32(cb&(continuation-1)) << s
+			s += 7
 		}
 	}
 	return b, 0, InvalidUint32
 }
 
 func decodeUint64(b []byte) ([]byte, uint64, error) {
-	if len(b) > 8 {
-		if b[0] == Uint64Kind[0] {
-			return b[9:], uint64(b[8]) | uint64(b[7])<<8 | uint64(b[6])<<16 | uint64(b[5])<<24 |
-				uint64(b[4])<<32 | uint64(b[3])<<40 | uint64(b[2])<<48 | uint64(b[1])<<56, nil
+	if len(b) > 1 && b[0] == Uint64Kind[0] {
+		var x uint64
+		var s uint
+		for i := 1; i < VarIntLen64+1; i++ {
+			cb := b[i]
+			// Check if msb is set signifying a continuation byte
+			if cb < continuation {
+				// Check for overflow
+				if i > VarIntLen64 && cb > 1 {
+					return b, 0, InvalidUint64
+				}
+				// End of varint, add the last bits and advance the buffer
+				return b[i+1:], x | uint64(cb)<<s, nil
+			}
+			// Add the lower 7 bits to the result and continue to the next byte
+			x |= uint64(cb&(continuation-1)) << s
+			s += 7
 		}
 	}
 	return b, 0, InvalidUint64
 }
 
 func decodeInt32(b []byte) ([]byte, int32, error) {
-	if len(b) > 4 {
-		if b[0] == Int32Kind[0] {
-			return b[5:], int32(uint32(b[4]) | uint32(b[3])<<8 | uint32(b[2])<<16 | uint32(b[1])<<24), nil
+	if len(b) > 1 && b[0] == Int32Kind[0] {
+		var ux uint32
+		var s uint
+		for i := 1; i < VarIntLen32+1; i++ {
+			cb := b[i]
+			// Check if msb is set signifying a continuation byte
+			if cb < continuation {
+				if i > VarIntLen32 && cb > 1 {
+					return b, 0, InvalidInt32
+				}
+				// End of varint, add the last bits and cast to signed integer
+				x := int32((ux | uint32(cb)<<s) >> 1)
+				// Flip the bits if the sign bit is set
+				if ux&1 != 0 {
+					x = ^x
+				}
+				return b[i+1:], x, nil
+			}
+			ux |= uint32(cb&(continuation-1)) << s
+			s += 7
 		}
 	}
 	return b, 0, InvalidInt32
 }
 
 func decodeInt64(b []byte) ([]byte, int64, error) {
-	if len(b) > 8 {
-		if b[0] == Int64Kind[0] {
-			return b[9:], int64(uint64(b[8]) | uint64(b[7])<<8 | uint64(b[6])<<16 | uint64(b[5])<<24 |
-				uint64(b[4])<<32 | uint64(b[3])<<40 | uint64(b[2])<<48 | uint64(b[1])<<56), nil
+	if len(b) > 1 && b[0] == Int64Kind[0] {
+		var ux uint64
+		var s uint
+		for i := 1; i < VarIntLen64+1; i++ {
+			cb := b[i]
+			// Check if msb is set signifying a continuation byte
+			if cb < continuation {
+				if i > VarIntLen64 && cb > 1 {
+					return b, 0, InvalidInt64
+				}
+				// End of varint, add the last bits and cast to signed integer
+				x := int64((ux | uint64(cb)<<s) >> 1)
+				// Flip the bits if the sign bit is set
+				if ux&1 != 0 {
+					x = ^x
+				}
+				return b[i+1:], x, nil
+			}
+			ux |= uint64(cb&(continuation-1)) << s
+			s += 7
 		}
 	}
 	return b, 0, InvalidInt64

encode.go

@@ -114,31 +114,66 @@ func encodeUint8(b *Buffer, value uint8) {
 	*b = append(*b, value)
 }
 
+// Variable integer encoding with the same format as binary.varint
+//(https://developers.google.com/protocol-buffers/docs/encoding#varints)
 func encodeUint16(b *Buffer, value uint16) {
 	b.Write(Uint16Kind)
-	*b = append(*b, byte(value>>8), byte(value))
+	for value >= continuation {
+		// Append the lower 7 bits of the value, then shift the value to the right by 7 bits.
+		*b = append(*b, byte(value)|continuation)
+		value >>= 7
+	}
+	*b = append(*b, byte(value))
 }
 
 func encodeUint32(b *Buffer, value uint32) {
 	b.Write(Uint32Kind)
-	*b = append(*b, byte(value>>24), byte(value>>16), byte(value>>8), byte(value))
+	for value >= continuation {
+		// Append the lower 7 bits of the value, then shift the value to the right by 7 bits.
+		*b = append(*b, byte(value)|continuation)
+		value >>= 7
+	}
+	*b = append(*b, byte(value))
 }
 
 func encodeUint64(b *Buffer, value uint64) {
 	b.Write(Uint64Kind)
-	*b = append(*b, byte(value>>56), byte(value>>48), byte(value>>40), byte(value>>32), byte(value>>24), byte(value>>16), byte(value>>8), byte(value))
+	for value >= continuation {
+		// Append the lower 7 bits of the value, then shift the value to the right by 7 bits.
+		*b = append(*b, byte(value)|continuation)
+		value >>= 7
+	}
+	*b = append(*b, byte(value))
 }
 
 func encodeInt32(b *Buffer, value int32) {
 	b.Write(Int32Kind)
-	castValue := uint32(value)
-	*b = append(*b, byte(castValue>>24), byte(castValue>>16), byte(castValue>>8), byte(castValue))
+	// Shift the value to the left by 1 bit, then flip the bits if the value is negative.
+	castValue := uint32(value) << 1
+	if value < 0 {
+		castValue = ^castValue
+	}
+	for castValue >= continuation {
+		// Append the lower 7 bits of the value, then shift the value to the right by 7 bits.
+		*b = append(*b, byte(castValue)|continuation)
+		castValue >>= 7
+	}
+	*b = append(*b, byte(castValue))
 }
 
 func encodeInt64(b *Buffer, value int64) {
 	b.Write(Int64Kind)
-	castValue := uint64(value)
-	*b = append(*b, byte(castValue>>56), byte(castValue>>48), byte(castValue>>40), byte(castValue>>32), byte(castValue>>24), byte(castValue>>16), byte(castValue>>8), byte(castValue))
+	// Shift the value to the left by 1 bit, then flip the bits if the value is negative.
+	castValue := uint64(value) << 1
+	if value < 0 {
+		castValue = ^castValue
+	}
+	for castValue >= continuation {
+		// Append the lower 7 bits of the value, then shift the value to the right by 7 bits.
+		*b = append(*b, byte(castValue)|continuation)
+		castValue >>= 7
+	}
+	*b = append(*b, byte(castValue))
 }
 
 func encodeFloat32(b *Buffer, value float32) {

encode_test.go

@@ -45,7 +45,7 @@ func TestEncodeMap(t *testing.T) {
 	p := NewBuffer()
 	encodeMap(p, 32, StringKind, Uint32Kind)
 
-	assert.Equal(t, 1+1+1+1+4, len(p.Bytes()))
+	assert.Equal(t, 1+1+1+1+1, len(p.Bytes()))
 	assert.Equal(t, MapKind, Kind((p.Bytes())[0:1]))
 	assert.Equal(t, StringKind, Kind((p.Bytes())[1:2]))
 	assert.Equal(t, Uint32Kind, Kind((p.Bytes())[2:3]))
@@ -64,7 +64,7 @@ func TestEncodeSlice(t *testing.T) {
 	p := NewBuffer()
 	encodeSlice(p, 32, StringKind)
 
-	assert.Equal(t, 1+1+1+4, len(p.Bytes()))
+	assert.Equal(t, 1+1+1+1, len(p.Bytes()))
 	assert.Equal(t, SliceKind, Kind((p.Bytes())[0:1]))
 	assert.Equal(t, StringKind, Kind((p.Bytes())[1:2]))
 	assert.Equal(t, Uint32Kind, Kind((p.Bytes())[2:3]))
@@ -84,8 +84,8 @@ func TestEncodeBytes(t *testing.T) {
 
 	encodeBytes(p, v)
 
-	assert.Equal(t, 1+1+4+len(v), len(p.Bytes()))
-	assert.Equal(t, v, (p.Bytes())[1+1+4:])
+	assert.Equal(t, 1+1+1+len(v), len(p.Bytes()))
+	assert.Equal(t, v, (p.Bytes())[1+1+1:])
 
 	p.Reset()
 	n := testing.AllocsPerRun(100, func() {
@@ -104,8 +104,8 @@ func TestEncodeString(t *testing.T) {
 
 	encodeString(p, v)
 
-	assert.Equal(t, 1+1+4+len(e), len(p.Bytes()))
-	assert.Equal(t, e, (p.Bytes())[1+1+4:])
+	assert.Equal(t, 1+1+1+len(e), len(p.Bytes()))
+	assert.Equal(t, e, (p.Bytes())[1+1+1:])
 
 	p.Reset()
 	n := testing.AllocsPerRun(100, func() {
@@ -124,8 +124,8 @@ func TestEncodeError(t *testing.T) {
 
 	encodeError(p, v)
 
-	assert.Equal(t, 1+1+1+4+len(e), len(p.Bytes()))
-	assert.Equal(t, e, (p.Bytes())[1+1+1+4:])
+	assert.Equal(t, 1+1+1+1+len(e), len(p.Bytes()))
+	assert.Equal(t, e, (p.Bytes())[1+1+1+1:])
 
 	p.Reset()
 	n := testing.AllocsPerRun(100, func() {
@@ -179,7 +179,7 @@ func TestEncodeUint16(t *testing.T) {
 
 	p := NewBuffer()
 	v := uint16(1024)
-	e := []byte{byte(v >> 8), byte(v)}
+	e := []byte{128, 8}
 
 	encodeUint16(p, v)
 
@@ -199,7 +199,7 @@ func TestEncodeUint32(t *testing.T) {
 
 	p := NewBuffer()
 	v := uint32(4294967290)
-	e := []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}
+	e := []byte{250, 255, 255, 255, 15}
 
 	encodeUint32(p, v)
 
@@ -219,11 +219,10 @@ func TestEncodeUint64(t *testing.T) {
 
 	p := NewBuffer()
 	v := uint64(18446744073709551610)
-	e := []byte{byte(v >> 56), byte(v >> 48), byte(v >> 40), byte(v >> 32), byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}
+	e := []byte{250, 255, 255, 255, 255, 255, 255, 255, 255, 1}
 
 	encodeUint64(p, v)
 
-	assert.Equal(t, 1+len(e), len(p.Bytes()))
 	assert.Equal(t, e, (p.Bytes())[1:])
 
 	p.Reset()
@@ -239,7 +238,7 @@ func TestEncodeInt32(t *testing.T) {
 
 	p := NewBuffer()
 	v := int32(-2147483648)
-	e := []byte{byte(uint32(v) >> 24), byte(uint32(v) >> 16), byte(uint32(v) >> 8), byte(uint32(v))}
+	e := []byte{255, 255, 255, 255, 15}
 
 	encodeInt32(p, v)
 
@@ -259,7 +258,7 @@ func TestEncodeInt64(t *testing.T) {
 
 	p := NewBuffer()
 	v := int64(-9223372036854775808)
-	e := []byte{byte(uint64(v) >> 56), byte(uint64(v) >> 48), byte(uint64(v) >> 40), byte(uint64(v) >> 32), byte(uint64(v) >> 24), byte(uint64(v) >> 16), byte(uint64(v) >> 8), byte(uint64(v))}
+	e := []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 1}
 
 	encodeInt64(p, v)