fix mongo transient tls error

Author: jgao54

flow/alerting/classifier.go

@@ -45,6 +45,7 @@ const (
 	MongoShutdownInProgress              = "(ShutdownInProgress) The server is in quiesce mode and will shut down"
 	MongoInterruptedDueToReplStateChange = "(InterruptedDueToReplStateChange) operation was interrupted"
 	MongoIncompleteReadOfMessageHeader   = "incomplete read of message header"
+	MongoTLSInvalidServerCertSignature   = "tls: invalid signature by the server certificate"
 
 	// mysqlGeometryLinearRingNotClosedError is the specific WKB parse failure raised by the
 	// go-geos library when a LinearRing's points do not close. Used to give a more specific code
@@ -817,6 +818,12 @@ func GetErrorClass(ctx context.Context, err error) (ErrorClass, ErrorInfo) {
 			return ErrorNotifyConnectivity, mongoErrorInfo
 		}
 
+		// TLS handshake failures during connection checkout is typically retryable. We've observed
+		// Atlas briefly serving a mismatched cert/key pair during rolling cert rotation that auto-recovers
+		if strings.Contains(err.Error(), MongoTLSInvalidServerCertSignature) {
+			return ErrorRetryRecoverable, mongoErrorInfo
+		}
+
 		// https://www.mongodb.com/docs/manual/reference/error-codes/
 		switch mongoCmdErr.Code {
 		case 6: // HostUnreachable

flow/alerting/classifier_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.mongodb.org/mongo-driver/v2/mongo"
 	"go.mongodb.org/mongo-driver/v2/x/mongo/driver"
+	"go.mongodb.org/mongo-driver/v2/x/mongo/driver/topology"
 	"go.temporal.io/sdk/temporal"
 	"google.golang.org/api/googleapi"
 
@@ -846,6 +847,29 @@ func TestMongoCursorErrors(t *testing.T) {
 	}, errInfo)
 }
 
+func TestMongoTLSInvalidServerCertSignatureShouldBeRecoverable(t *testing.T) {
+	de := driver.Error{
+		Labels: []string{driver.NetworkError},
+		Wrapped: topology.ConnectionError{
+			Wrapped: errors.New(
+				"failed to configure TLS for pl-1-us-east-1.xxxx.mongodb.net:1234: " +
+					"tls: invalid signature by the server certificate: crypto/rsa: verification error"),
+		},
+	}
+	err := mongo.CommandError{
+		Code:    de.Code,
+		Message: de.Message,
+		Labels:  de.Labels,
+		Wrapped: de,
+	}
+	errorClass, errInfo := GetErrorClass(t.Context(), fmt.Errorf("failed to create change stream: %w", err))
+	assert.Equal(t, ErrorRetryRecoverable, errorClass)
+	assert.Equal(t, ErrorInfo{
+		Source: ErrorSourceMongoDB,
+		Code:   "0",
+	}, errInfo)
+}
+
 func TestAuroraMySQLZeroDowntimePatchErrorShouldBeRecoverable(t *testing.T) {
 	// Simulate Aurora MySQL Zero Downtime Patch error
 	mysqlErr := &mysql.MyError{