Merge pull request #292 from hashicorp/vault-19781/msgpack-upgrade

Upgrade to go-msgpack/v2 2.1.1

Author: Christopher Swenson

.github/workflows/check.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        GO_VERSION: [ "1.16","1.17","1.18" ]
+        GO_VERSION: [ "1.20","1.21" ]
     steps:
       - name: "Fetch source code"
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
@@ -45,7 +45,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        GO_VERSION: [ "1.16","1.17","1.18" ]
+        GO_VERSION: [ "1.20","1.21" ]
     steps:
       - name: "Fetch source code"
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
@@ -76,7 +76,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        GO_VERSION: [ "1.16","1.17","1.18" ]
+        GO_VERSION: [ "1.20","1.21" ]
     steps:
       - name: "Fetch source code"
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
@@ -107,7 +107,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        GO_VERSION: [ "1.16","1.17","1.18" ]
+        GO_VERSION: [ "1.20","1.21" ]
     steps:
       - name: "Fetch source code"
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0

broadcast.go

@@ -58,7 +58,7 @@ func (m *Memberlist) encodeAndBroadcast(node string, msgType messageType, msg in
 // and notifies the given channel when transmission is finished. Fails
 // silently if there is an encoding error.
 func (m *Memberlist) encodeBroadcastNotify(node string, msgType messageType, msg interface{}, notify chan struct{}) {
-	buf, err := encode(msgType, msg)
+	buf, err := encode(msgType, msg, m.config.MsgpackUseNewTimeFormat)
 	if err != nil {
 		m.logger.Printf("[ERR] memberlist: Failed to encode message for broadcast: %s", err)
 	} else {

config.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/armon/go-metrics"
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-multierror"
 )
 
 type Config struct {
@@ -260,6 +260,12 @@ type Config struct {
 	// QueueCheckInterval is the interval at which we check the message
 	// queue to apply the warning and max depth.
 	QueueCheckInterval time.Duration
+
+	// MsgpackUseNewTimeFormat when set to true, force the underlying msgpack
+	// codec to use the new format of time.Time when encoding (used in
+	// go-msgpack v1.1.5 by default). Decoding is not affected, as all
+	// go-msgpack v2.1.0+ decoders know how to decode both formats.
+	MsgpackUseNewTimeFormat bool
 }
 
 // ParseCIDRs return a possible empty list of all Network that have been parsed

go.mod

@@ -1,19 +1,26 @@
 module github.com/hashicorp/memberlist
 
-go 1.12
+go 1.20
 
 require (
 	github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da
-	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c
-	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
-	github.com/hashicorp/go-msgpack v0.5.3
+	github.com/hashicorp/go-msgpack/v2 v2.1.1
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/go-sockaddr v1.0.0
 	github.com/miekg/dns v1.1.26
-	github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529
 	github.com/stretchr/testify v1.2.2
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.0 // indirect
+	github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/net v0.16.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 )

go.sum

@@ -8,8 +8,8 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-msgpack/v2 v2.1.1 h1:xQEY9yB2wnHitoSzk/B9UjXWRQ67QKu5AOm8aFp8N3I=
+github.com/hashicorp/go-msgpack/v2 v2.1.1/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
@@ -29,19 +29,21 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
+golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

memberlist.go

@@ -31,8 +31,8 @@ import (
 	"time"
 
 	"github.com/armon/go-metrics"
-	multierror "github.com/hashicorp/go-multierror"
-	sockaddr "github.com/hashicorp/go-sockaddr"
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/go-sockaddr"
 	"github.com/miekg/dns"
 )
 

net.go

@@ -15,8 +15,8 @@ import (
 	"sync/atomic"
 	"time"
 
-	metrics "github.com/armon/go-metrics"
-	"github.com/hashicorp/go-msgpack/codec"
+	"github.com/armon/go-metrics"
+	"github.com/hashicorp/go-msgpack/v2/codec"
 )
 
 // This is the minimum and maximum protocol version that we can
@@ -271,7 +271,7 @@ func (m *Memberlist) handleConn(conn net.Conn) {
 			m.logger.Printf("[ERR] memberlist: failed to receive: %s %s", err, LogConn(conn))
 
 			resp := errResp{err.Error()}
-			out, err := encode(errMsg, &resp)
+			out, err := encode(errMsg, &resp, m.config.MsgpackUseNewTimeFormat)
 			if err != nil {
 				m.logger.Printf("[ERR] memberlist: Failed to encode error response: %s", err)
 				return
@@ -330,7 +330,7 @@ func (m *Memberlist) handleConn(conn net.Conn) {
 		}
 
 		ack := ackResp{p.SeqNo, nil}
-		out, err := encode(ackRespMsg, &ack)
+		out, err := encode(ackRespMsg, &ack, m.config.MsgpackUseNewTimeFormat)
 		if err != nil {
 			m.logger.Printf("[ERR] memberlist: Failed to encode ack: %s", err)
 			return
@@ -773,7 +773,7 @@ func (m *Memberlist) handleCompressed(buf []byte, from net.Addr, timestamp time.
 
 // encodeAndSendMsg is used to combine the encoding and sending steps
 func (m *Memberlist) encodeAndSendMsg(a Address, msgType messageType, msg interface{}) error {
-	out, err := encode(msgType, msg)
+	out, err := encode(msgType, msg, m.config.MsgpackUseNewTimeFormat)
 	if err != nil {
 		return err
 	}
@@ -819,7 +819,7 @@ func (m *Memberlist) rawSendMsgPacket(a Address, node *Node, msg []byte) error {
 
 	// Check if we have compression enabled
 	if m.config.EnableCompression {
-		buf, err := compressPayload(msg)
+		buf, err := compressPayload(msg, m.config.MsgpackUseNewTimeFormat)
 		if err != nil {
 			m.logger.Printf("[WARN] memberlist: Failed to compress payload: %v", err)
 		} else {
@@ -882,7 +882,7 @@ func (m *Memberlist) rawSendMsgPacket(a Address, node *Node, msg []byte) error {
 func (m *Memberlist) rawSendMsgStream(conn net.Conn, sendBuf []byte, streamLabel string) error {
 	// Check if compression is enabled
 	if m.config.EnableCompression {
-		compBuf, err := compressPayload(sendBuf)
+		compBuf, err := compressPayload(sendBuf, m.config.MsgpackUseNewTimeFormat)
 		if err != nil {
 			m.logger.Printf("[ERROR] memberlist: Failed to compress payload: %v", err)
 		} else {
@@ -930,7 +930,11 @@ func (m *Memberlist) sendUserMsg(a Address, sendBuf []byte) error {
 	}
 
 	header := userMsgHeader{UserMsgLen: len(sendBuf)}
-	hd := codec.MsgpackHandle{}
+	hd := codec.MsgpackHandle{
+		BasicHandle: codec.BasicHandle{
+			TimeNotBuiltin: !m.config.MsgpackUseNewTimeFormat,
+		},
+	}
 	enc := codec.NewEncoder(bufConn, &hd)
 	if err := enc.Encode(&header); err != nil {
 		return err
@@ -1340,7 +1344,7 @@ func (m *Memberlist) sendPingAndWaitForAck(a Address, ping ping, deadline time.T
 	defer conn.Close()
 	conn.SetDeadline(deadline)
 
-	out, err := encode(pingMsg, &ping)
+	out, err := encode(pingMsg, &ping, m.config.MsgpackUseNewTimeFormat)
 	if err != nil {
 		return false, err
 	}

net_test.go

@@ -16,7 +16,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/hashicorp/go-msgpack/codec"
+	"github.com/hashicorp/go-msgpack/v2/codec"
 	"github.com/stretchr/testify/require"
 )
 
@@ -43,7 +43,7 @@ func TestHandleCompoundPing(t *testing.T) {
 		SourcePort: uint16(udpAddr.Port),
 		SourceNode: "test",
 	}
-	buf, err := encode(pingMsg, ping)
+	buf, err := encode(pingMsg, ping, m.config.MsgpackUseNewTimeFormat)
 	if err != nil {
 		t.Fatalf("unexpected err %s", err)
 	}
@@ -112,7 +112,7 @@ func TestHandlePing(t *testing.T) {
 		SourcePort: uint16(udpAddr.Port),
 		SourceNode: "test",
 	}
-	buf, err := encode(pingMsg, ping)
+	buf, err := encode(pingMsg, ping, m.config.MsgpackUseNewTimeFormat)
 	if err != nil {
 		t.Fatalf("unexpected err %s", err)
 	}
@@ -177,7 +177,7 @@ func TestHandlePing_WrongNode(t *testing.T) {
 		SourcePort: uint16(udpAddr.Port),
 		SourceNode: "test",
 	}
-	buf, err := encode(pingMsg, ping)
+	buf, err := encode(pingMsg, ping, m.config.MsgpackUseNewTimeFormat)
 	if err != nil {
 		t.Fatalf("unexpected err %s", err)
 	}
@@ -221,7 +221,7 @@ func TestHandleIndirectPing(t *testing.T) {
 		SourcePort: uint16(udpAddr.Port),
 		SourceNode: "test",
 	}
-	buf, err := encode(indirectPingMsg, &ind)
+	buf, err := encode(indirectPingMsg, &ind, m.config.MsgpackUseNewTimeFormat)
 	if err != nil {
 		t.Fatalf("unexpected err %s", err)
 	}
@@ -333,7 +333,7 @@ func TestTCPPing(t *testing.T) {
 		}
 
 		ack := ackResp{pingIn.SeqNo, nil}
-		out, err := encode(ackRespMsg, &ack)
+		out, err := encode(ackRespMsg, &ack, m.config.MsgpackUseNewTimeFormat)
 		if err != nil {
 			pingErrCh <- fmt.Errorf("failed to encode ack: %s", err)
 			return
@@ -381,7 +381,7 @@ func TestTCPPing(t *testing.T) {
 		}
 
 		ack := ackResp{pingIn.SeqNo + 1, nil}
-		out, err := encode(ackRespMsg, &ack)
+		out, err := encode(ackRespMsg, &ack, m.config.MsgpackUseNewTimeFormat)
 		if err != nil {
 			pingErrCh <- fmt.Errorf("failed to encode ack: %s", err)
 			return
@@ -423,7 +423,7 @@ func TestTCPPing(t *testing.T) {
 		}
 
 		bogus := indirectPingReq{}
-		out, err := encode(indirectPingMsg, &bogus)
+		out, err := encode(indirectPingMsg, &bogus, m.config.MsgpackUseNewTimeFormat)
 		if err != nil {
 			pingErrCh <- fmt.Errorf("failed to encode bogus msg: %s", err)
 			return
@@ -507,7 +507,11 @@ func TestTCPPushPull(t *testing.T) {
 
 	// Send our node state
 	header := pushPullHeader{Nodes: 3}
-	hd := codec.MsgpackHandle{}
+	hd := codec.MsgpackHandle{
+		BasicHandle: codec.BasicHandle{
+			TimeNotBuiltin: !m.config.MsgpackUseNewTimeFormat,
+		},
+	}
 	enc := codec.NewEncoder(conn, &hd)
 
 	// Send the push/pull indicator
@@ -529,7 +533,11 @@ func TestTCPPushPull(t *testing.T) {
 	}
 
 	var bufConn io.Reader = conn
-	msghd := codec.MsgpackHandle{}
+	msghd := codec.MsgpackHandle{
+		BasicHandle: codec.BasicHandle{
+			TimeNotBuiltin: !m.config.MsgpackUseNewTimeFormat,
+		},
+	}
 	dec := codec.NewDecoder(bufConn, &msghd)
 
 	// Check if we have a compressed message
@@ -620,7 +628,7 @@ func TestSendMsg_Piggyback(t *testing.T) {
 		SourcePort: uint16(udpAddr.Port),
 		SourceNode: "test",
 	}
-	buf, err := encode(pingMsg, ping)
+	buf, err := encode(pingMsg, ping, m.config.MsgpackUseNewTimeFormat)
 	if err != nil {
 		t.Fatalf("unexpected err %s", err)
 	}

state.go

@@ -13,7 +13,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	metrics "github.com/armon/go-metrics"
+	"github.com/armon/go-metrics"
 )
 
 type NodeStateType int
@@ -356,14 +356,14 @@ func (m *Memberlist) probeNode(node *nodeState) {
 		}
 	} else {
 		var msgs [][]byte
-		if buf, err := encode(pingMsg, &ping); err != nil {
+		if buf, err := encode(pingMsg, &ping, m.config.MsgpackUseNewTimeFormat); err != nil {
 			m.logger.Printf("[ERR] memberlist: Failed to encode UDP ping message: %s", err)
 			return
 		} else {
 			msgs = append(msgs, buf.Bytes())
 		}
 		s := suspect{Incarnation: node.Incarnation, Node: node.Name, From: m.config.Name}
-		if buf, err := encode(suspectMsg, &s); err != nil {
+		if buf, err := encode(suspectMsg, &s, m.config.MsgpackUseNewTimeFormat); err != nil {
 			m.logger.Printf("[ERR] memberlist: Failed to encode suspect message: %s", err)
 			return
 		} else {