kvserver: skip test on failed re-listen

A test can't listen() on a random port, then close the listener, and expect the
same port to be available for a subsequent listen(). Skip the test if the port
is not available the second time around; this is very rare but can happen.

See #146175.

Epic: none

Author: tbg

pkg/kv/kvserver/raft_transport_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/rpc/nodedialer"
 	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
@@ -111,6 +112,8 @@ type raftTransportTestContext struct {
 	nodeRPCContext *rpc.Context
 	gossip         *gossip.Gossip
 	st             *cluster.Settings
+
+	skipOnListenErr bool // if true, calls Skip on error from net.Listen
 }
 
 // clockWithManualSource is a pair of clocks: a manual clock and a clock that
@@ -202,8 +205,12 @@ func (rttc *raftTransportTestContext) AddNodeWithoutGossip(
 		knobs,
 	)
 	rttc.transports[nodeID] = transport
-	ln, err := netutil.ListenAndServeGRPC(stopper, grpcServer, addr)
+	ln, err := net.Listen(addr.Network(), addr.String())
+	if err != nil && rttc.skipOnListenErr {
+		skip.IgnoreLintf(rttc.t, "skipping test due to listen error: %s", err)
+	}
 	require.NoError(rttc.t, err)
+	require.NoError(rttc.t, netutil.ServeGRPC(stopper, grpcServer, ln))
 	return transport, ln.Addr()
 }
 
@@ -577,6 +584,11 @@ func TestReopenConnection(t *testing.T) {
 		StoreID:   2,
 		ReplicaID: 2,
 	}
+
+	// We're re-listening on an old address here, but the port may be
+	// in use. In the very rare case of this happening, skip the test.
+	// See: https://github.com/cockroachdb/cockroach/issues/146175.
+	rttc.skipOnListenErr = true
 	serverTransport, serverAddr :=
 		rttc.AddNodeWithoutGossip(
 			serverReplica.NodeID,

pkg/util/netutil/net.go

@@ -34,7 +34,13 @@ func ListenAndServeGRPC(
 	if err != nil {
 		return ln, err
 	}
+	if err := ServeGRPC(stopper, server, ln); err != nil {
+		return nil, err
+	}
+	return ln, nil
+}
 
+func ServeGRPC(stopper *stop.Stopper, server *grpc.Server, ln net.Listener) error {
 	ctx := context.TODO()
 
 	stopper.AddCloser(stop.CloserFn(server.Stop))
@@ -44,15 +50,12 @@ func ListenAndServeGRPC(
 	}
 	if err := stopper.RunAsyncTask(ctx, "listen-quiesce", waitQuiesce); err != nil {
 		waitQuiesce(ctx)
-		return nil, err
+		return err
 	}
 
-	if err := stopper.RunAsyncTask(ctx, "serve", func(context.Context) {
+	return stopper.RunAsyncTask(ctx, "serve", func(context.Context) {
 		FatalIfUnexpected(server.Serve(ln))
-	}); err != nil {
-		return nil, err
-	}
-	return ln, nil
+	})
 }
 
 var httpLogger = log.NewStdLogger(severity.ERROR, "net/http")