Fix OpenSSL Interop for RSA SignatureHashAlgos

Author: adrianosela

e2e/e2e_openssl_test.go

@@ -345,3 +345,21 @@ func TestPionOpenSSLE2ESimpleECDSAClientCert(t *testing.T) {
 		testPionE2ESimpleECDSAClientCert(t, serverPion, clientOpenSSL)
 	})
 }
+
+func TestPionOpenSSLE2ESimpleRSA(t *testing.T) {
+	t.Run("OpenSSLServer", func(t *testing.T) {
+		testPionE2ESimpleRSA(t, serverOpenSSL, clientPion)
+	})
+	t.Run("OpenSSLClient", func(t *testing.T) {
+		testPionE2ESimpleRSA(t, serverPion, clientOpenSSL)
+	})
+}
+
+func TestPionOpenSSLE2ESimpleRSAClientCert(t *testing.T) {
+	t.Run("OpenSSLServer", func(t *testing.T) {
+		testPionE2ESimpleRSAClientCert(t, serverOpenSSL, clientPion)
+	})
+	t.Run("OpenSSLClient", func(t *testing.T) {
+		testPionE2ESimpleRSAClientCert(t, serverPion, clientOpenSSL)
+	})
+}

e2e/e2e_test.go

@@ -335,6 +335,8 @@ func withConnectionIDGenerator(g func() []byte) dtlsTestOpts {
 //   - Assert that you can send messages both ways
 //   - Assert that Close() on both ends work
 //   - Assert that no Goroutines are leaked
+//
+//nolint:dupl
 func testPionE2ESimple(t *testing.T, server, client func(*comm), opts ...dtlsTestOpts) {
 	t.Helper()
 	lim := test.TimeOut(time.Second * 30)
@@ -382,6 +384,56 @@ func testPionE2ESimple(t *testing.T, server, client func(*comm), opts ...dtlsTes
 	}
 }
 
+//nolint:dupl
+func testPionE2ESimpleRSA(t *testing.T, server, client func(*comm), opts ...dtlsTestOpts) {
+	t.Helper()
+	lim := test.TimeOut(time.Second * 30)
+	defer lim.Stop()
+
+	report := test.CheckRoutines(t)
+	defer report()
+
+	for _, cipherSuite := range []dtls.CipherSuiteID{
+		dtls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+		dtls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+		dtls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+	} {
+		cipherSuite := cipherSuite
+		t.Run(cipherSuite.String(), func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+
+			priv, err := rsa.GenerateKey(rand.Reader, 2048)
+			assert.NoError(t, err)
+			cert, err := selfsign.SelfSign(priv)
+			assert.NoError(t, err)
+
+			clientOpts := []dtls.ClientOption{
+				dtls.WithCertificates(cert),
+				dtls.WithCipherSuites(cipherSuite),
+				dtls.WithInsecureSkipVerify(true),
+			}
+			serverOpts := []dtls.ServerOption{
+				dtls.WithCertificates(cert),
+				dtls.WithCipherSuites(cipherSuite),
+				dtls.WithInsecureSkipVerify(true),
+			}
+			for _, o := range opts {
+				clientOpts = append(clientOpts, o.clientOpts...)
+				serverOpts = append(serverOpts, o.serverOpts...)
+			}
+			serverPort := randomPort(t)
+			comm := newComm(ctx, clientOpts, serverOpts, serverPort, server, client)
+			comm.setOpenSSLInfo(
+				[]dtls.CipherSuiteID{cipherSuite}, []dtls.CipherSuiteID{cipherSuite},
+				[]tls.Certificate{cert}, []tls.Certificate{cert},
+				nil, nil, nil, nil, true)
+			defer comm.cleanup(t)
+			comm.assert(t)
+		})
+	}
+}
+
 func testPionE2ESimplePSK(t *testing.T, server, client func(*comm), opts ...dtlsTestOpts) {
 	t.Helper()
 
@@ -812,6 +864,10 @@ func TestPionE2ESimple(t *testing.T) {
 	testPionE2ESimple(t, serverPion, clientPion)
 }
 
+func TestPionE2ESimpleRSA(t *testing.T) {
+	testPionE2ESimpleRSA(t, serverPion, clientPion)
+}
+
 func TestPionE2ESimplePSK(t *testing.T) {
 	testPionE2ESimplePSK(t, serverPion, clientPion)
 }

pkg/crypto/signaturehash/signaturehash.go

@@ -26,18 +26,15 @@ type Algorithm struct {
 	Signature signature.Algorithm
 }
 
-// Algorithms are all the known SignatureHash Algorithms.
+// Algorithms returns signature algorithms compatible with DTLS 1.2 / TLS 1.2.
+// This excludes TLS 1.3-specific schemes like RSA-PSS to ensure compatibility
+// with implementations like OpenSSL that don't recognize TLS 1.3 signature
+// algorithm IDs in DTLS 1.2 handshakes.
 //
-// IMPORTANT: order in this slice determines priority used
-// by SelectSignatureScheme and SelectSignatureScheme13.
+// IMPORTANT: order in this slice determines priority used by SelectSignatureScheme.
 //
-// Order follows industry standard preference (ECDSA-first) as used
-// by: OpenSSL, BoringSSL, Firefox, Chrome, and other major TLS 1.3
-// implementations. This prioritizes elliptic curve crypto for better
-// performance (smaller keys, faster computations, less bandwidth).
-//
-// Note that this is different than Go's TLS implementation order of
-// preference.
+// Order follows industry standard preference (ECDSA-first) as used by OpenSSL,
+// BoringSSL, Firefox, Chrome, and other major TLS implementations.
 func Algorithms() []Algorithm {
 	return []Algorithm{
 		// ECDSA schemes (modern, efficient - industry standard preference)
@@ -48,21 +45,6 @@ func Algorithms() []Algorithm {
 		// Ed25519
 		{hash.Ed25519, signature.Ed25519},
 
-		// RSA-PSS RSAE schemes (DTLS 1.3 compatible with standard RSA certs)
-		// Note: We only offer RSA_PSS_RSAE variants (0x0804-0x0806), not RSA_PSS_PSS
-		// (0x0809-0x080b). RSA-PSS certificates with OID id-RSASSA-PSS are virtually
-		// unused in the real world and are not allowed by the CA/Browser Forum Baseline
-		// Requirements for WebPKI. We avoid unnecessary complexity for certificates that
-		// don't exist in practice, following the pragmatic approach of Go's crypto/tls
-		// and BoringSSL: target real-world WebPKI use cases rather than RFC completeness.
-		// RSA_PSS_PSS schemes are parsed for wire-format compatibility but never negotiated.
-		{hash.SHA256, signature.RSA_PSS_RSAE_SHA256},
-		{hash.SHA384, signature.RSA_PSS_RSAE_SHA384},
-		{hash.SHA512, signature.RSA_PSS_RSAE_SHA512},
-		// {hash.SHA256, signature.RSA_PSS_PSS_SHA256},
-		// {hash.SHA384, signature.RSA_PSS_PSS_SHA384},
-		// {hash.SHA512, signature.RSA_PSS_PSS_SHA512},
-
 		// RSA PKCS#1 v1.5 schemes (legacy, DTLS 1.2)
 		{hash.SHA256, signature.RSA},
 		{hash.SHA384, signature.RSA},
@@ -98,6 +80,10 @@ func (a *Algorithm) isCompatible(signer crypto.Signer) bool {
 // ParseSignatureSchemes translates []tls.SignatureScheme to []signatureHashAlgorithm.
 // It returns default signature scheme list if no SignatureScheme is passed.
 // This function handles both TLS 1.2 byte-split encoding and TLS 1.3 PSS full uint16 schemes.
+//
+// For DTLS 1.2 / TLS 1.2, this returns Algorithms() which excludes TLS 1.3-specific
+// schemes like RSA-PSS for compatibility with implementations like OpenSSL.
+// When DTLS 1.3 is implemented, use Algorithms13() or create ParseSignatureSchemes13().
 func ParseSignatureSchemes(sigs []tls.SignatureScheme, insecureHashes bool) ([]Algorithm, error) {
 	if len(sigs) == 0 {
 		return Algorithms(), nil

pkg/crypto/signaturehash/signaturehash_13.go

@@ -0,0 +1,48 @@
+// SPDX-FileCopyrightText: 2026 The Pion community <https://pion.ly>
+// SPDX-License-Identifier: MIT
+
+package signaturehash
+
+import (
+	"github.com/pion/dtls/v3/pkg/crypto/hash"
+	"github.com/pion/dtls/v3/pkg/crypto/signature"
+)
+
+// Algorithms13 returns signature algorithms compatible with DTLS 1.3. This.
+// includes DTLS 1.3-specific schemes like RSA-PSS in addition to DTLS 1.2 schemes.
+//
+// IMPORTANT: order in this slice determines priority used by SelectSignatureScheme13.
+//
+// Order follows industry standard preference (ECDSA-first) as used by OpenSSL,
+// BoringSSL, Firefox, Chrome, and other major TLS 1.3 implementations.
+func Algorithms13() []Algorithm {
+	return []Algorithm{
+		// ECDSA schemes (modern, efficient - industry standard preference)
+		{hash.SHA256, signature.ECDSA},
+		{hash.SHA384, signature.ECDSA},
+		{hash.SHA512, signature.ECDSA},
+
+		// Ed25519
+		{hash.Ed25519, signature.Ed25519},
+
+		// RSA-PSS RSAE schemes (TLS 1.3 / DTLS 1.3 compatible with standard RSA certs)
+		// Note: We only offer RSA_PSS_RSAE variants (0x0804-0x0806), not RSA_PSS_PSS
+		// (0x0809-0x080b). RSA-PSS certificates with OID id-RSASSA-PSS are virtually
+		// unused in the real world and are not allowed by the CA/Browser Forum Baseline
+		// Requirements for WebPKI. We avoid unnecessary complexity for certificates that
+		// don't exist in practice, following the pragmatic approach of Go's crypto/tls
+		// and BoringSSL: target real-world WebPKI use cases rather than RFC completeness.
+		// RSA_PSS_PSS schemes are parsed for wire-format compatibility but never negotiated.
+		{hash.SHA256, signature.RSA_PSS_RSAE_SHA256},
+		{hash.SHA384, signature.RSA_PSS_RSAE_SHA384},
+		{hash.SHA512, signature.RSA_PSS_RSAE_SHA512},
+		// {hash.SHA256, signature.RSA_PSS_PSS_SHA256},
+		// {hash.SHA384, signature.RSA_PSS_PSS_SHA384},
+		// {hash.SHA512, signature.RSA_PSS_PSS_SHA512},
+
+		// RSA PKCS#1 v1.5 schemes (backward compatibility with DTLS 1.2)
+		{hash.SHA256, signature.RSA},
+		{hash.SHA384, signature.RSA},
+		{hash.SHA512, signature.RSA},
+	}
+}

pkg/crypto/signaturehash/signaturehash_13_test.go

@@ -0,0 +1,108 @@
+// SPDX-FileCopyrightText: 2026 The Pion community <https://pion.ly>
+// SPDX-License-Identifier: MIT
+
+package signaturehash
+
+import (
+	"testing"
+
+	"github.com/pion/dtls/v3/pkg/crypto/hash"
+	"github.com/pion/dtls/v3/pkg/crypto/signature"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAlgorithms13(t *testing.T) {
+	algos := Algorithms13()
+
+	// Verify we got expected number of algorithms
+	// ECDSA (3) + Ed25519 (1) + RSA-PSS (3) + RSA PKCS#1 (3) = 10
+	assert.Len(t, algos, 10, "Algorithms13 should return 10 signature schemes")
+
+	// Verify ECDSA schemes come first (industry standard preference)
+	assert.Equal(t, Algorithm{hash.SHA256, signature.ECDSA}, algos[0])
+	assert.Equal(t, Algorithm{hash.SHA384, signature.ECDSA}, algos[1])
+	assert.Equal(t, Algorithm{hash.SHA512, signature.ECDSA}, algos[2])
+
+	// Verify Ed25519
+	assert.Equal(t, Algorithm{hash.Ed25519, signature.Ed25519}, algos[3])
+
+	// Verify RSA-PSS schemes (TLS 1.3 preference for RSA)
+	assert.Equal(t, Algorithm{hash.SHA256, signature.RSA_PSS_RSAE_SHA256}, algos[4])
+	assert.Equal(t, Algorithm{hash.SHA384, signature.RSA_PSS_RSAE_SHA384}, algos[5])
+	assert.Equal(t, Algorithm{hash.SHA512, signature.RSA_PSS_RSAE_SHA512}, algos[6])
+
+	// Verify RSA PKCS#1 v1.5 schemes come last (TLS 1.2 compatibility)
+	assert.Equal(t, Algorithm{hash.SHA256, signature.RSA}, algos[7])
+	assert.Equal(t, Algorithm{hash.SHA384, signature.RSA}, algos[8])
+	assert.Equal(t, Algorithm{hash.SHA512, signature.RSA}, algos[9])
+}
+
+func TestAlgorithms_DTLS12_ExcludesRSAPSS(t *testing.T) {
+	algos := Algorithms()
+
+	// Verify DTLS 1.2 algorithms exclude RSA-PSS schemes
+	for _, algo := range algos {
+		assert.False(t, algo.Signature.IsPSS(),
+			"Algorithms() for DTLS 1.2 should not include RSA-PSS schemes (found %v)", algo)
+	}
+
+	// Verify we still have RSA PKCS#1 v1.5
+	hasRSA := false
+	for _, algo := range algos {
+		if algo.Signature == signature.RSA {
+			hasRSA = true
+
+			break
+		}
+	}
+	assert.True(t, hasRSA, "Algorithms() should include RSA PKCS#1 v1.5 schemes")
+}
+
+func TestAlgorithms13_IncludesRSAPSS(t *testing.T) {
+	algos := Algorithms13()
+
+	// Verify DTLS 1.3 algorithms include RSA-PSS schemes
+	hasRSAPSS := false
+	for _, algo := range algos {
+		if algo.Signature.IsPSS() {
+			hasRSAPSS = true
+
+			break
+		}
+	}
+	assert.True(t, hasRSAPSS, "Algorithms13() should include RSA-PSS schemes")
+
+	// Verify we still have RSA PKCS#1 v1.5 for backward compatibility
+	hasRSA := false
+	for _, algo := range algos {
+		if algo.Signature == signature.RSA {
+			hasRSA = true
+
+			break
+		}
+	}
+	assert.True(t, hasRSA, "Algorithms13() should include RSA PKCS#1 v1.5 for backward compatibility")
+}
+
+func TestAlgorithms13_RSAPSSBeforePKCS1(t *testing.T) {
+	algos := Algorithms13()
+
+	// Find positions of first RSA-PSS and first RSA PKCS#1 schemes
+	firstRSAPSS := -1
+	firstRSA := -1
+
+	for i, algo := range algos {
+		if firstRSAPSS == -1 && algo.Signature.IsPSS() {
+			firstRSAPSS = i
+		}
+		if firstRSA == -1 && algo.Signature == signature.RSA {
+			firstRSA = i
+		}
+	}
+
+	// In TLS 1.3, RSA-PSS should be preferred over RSA PKCS#1 v1.5
+	assert.NotEqual(t, -1, firstRSAPSS, "Should find RSA-PSS schemes")
+	assert.NotEqual(t, -1, firstRSA, "Should find RSA PKCS#1 schemes")
+	assert.Less(t, firstRSAPSS, firstRSA,
+		"RSA-PSS schemes should come before RSA PKCS#1 in Algorithms13() for TLS 1.3 preference")
+}