ML-DSA support as a TLS 1.3 signature scheme (#3251)

Wire up draft-ietf-tls-mldsa (IANA code points 0x0904/0x0905/0x0906 for
MLDSA44/65/87) through AWS-LC's libssl, backed by the existing PQDSA
EVP_PKEY support in libcrypto. ML-DSA is TLS 1.3 only and is advertised
in the default signing and verification sigalg lists; the per-sigalg
gate in pkey_supports_algorithm keeps these entries from being selected
on earlier protocol versions.

- Add SSL_SIGN_MLDSA44/65/87 to the public header.
- Add a new SSL_PKEY_PQDSA certificate slot.
- Extend SSL_SIGNATURE_ALGORITHM with param_nid (replacing the EC-only
curve field) so sigalg rows can constrain both EC curves and ML-DSA
parameter sets.
- Gate PQDSA to TLS 1.3 in pkey_supports_algorithm, and require the
sigalg's param_nid to match EVP_PKEY_pqdsa_get_type(pkey).
- Route PQDSA through the existing EVP_DigestSign/Verify path, which
dispatches to sign_message/verify_message in pure mode.
- Add MLDSA44/65/87 to kVerifySignatureAlgorithms and
kSignSignatureAlgorithms so they're negotiated by default.

Tests:
- Add MLDSA test fixtures (cert + seed private key) from the IETF LAMPS
WG examples accompanying draft-ietf-lamps-dilithium-certificates, and
parameterized handshake tests covering success, TLS 1.2 rejection, and
cross-variant sigalg mismatch.
- Regenerate the golden ClientHello byte vectors in ssl_test.cc and bump
kKeyShare1Offset by the corresponding 6 bytes.
- Relax SSLTest.Padding to skip version/session combinations whose
baseline ClientHello is now pushed past the 0xff boundary by the larger
sigalg list.

Author: dkostic

include/openssl/ssl.h

@@ -1149,6 +1149,9 @@ OPENSSL_EXPORT int SSL_set_ocsp_response(SSL *ssl, const uint8_t *response,
 #define SSL_SIGN_RSA_PSS_RSAE_SHA384 0x0805
 #define SSL_SIGN_RSA_PSS_RSAE_SHA512 0x0806
 #define SSL_SIGN_ED25519 0x0807
+#define SSL_SIGN_MLDSA44 0x0904
+#define SSL_SIGN_MLDSA65 0x0905
+#define SSL_SIGN_MLDSA87 0x0906
 
 // SSL_SIGN_RSA_PKCS1_MD5_SHA1 is an internal signature algorithm used to
 // specify raw RSASSA-PKCS1-v1_5 with an MD5/SHA-1 concatenation, as used in TLS

ssl/extensions.cc

@@ -299,6 +299,14 @@ static const uint16_t kVerifySignatureAlgorithms[] = {
     SSL_SIGN_RSA_PSS_RSAE_SHA512,
     SSL_SIGN_RSA_PKCS1_SHA512,
 
+    // ML-DSA. The codepoints are advertised in all TLS versions, but the
+    // sigalgs are only selectable in TLS 1.3 (filtered by
+    // |pkey_supports_algorithm|, and rejected at peer-sigalg-check time by
+    // |tls12_check_peer_sigalg| to satisfy draft-ietf-tls-mldsa §3.3).
+    SSL_SIGN_MLDSA44,
+    SSL_SIGN_MLDSA65,
+    SSL_SIGN_MLDSA87,
+
     // For now, SHA-1 is still accepted but least preferable.
     SSL_SIGN_RSA_PKCS1_SHA1,
 };
@@ -323,6 +331,12 @@ static const uint16_t kSignSignatureAlgorithms[] = {
     SSL_SIGN_RSA_PSS_RSAE_SHA512,
     SSL_SIGN_RSA_PKCS1_SHA512,
 
+    // ML-DSA. Selectable for signing only in TLS 1.3; the version gate lives
+    // in |pkey_supports_algorithm|.
+    SSL_SIGN_MLDSA44,
+    SSL_SIGN_MLDSA65,
+    SSL_SIGN_MLDSA87,
+
     // If the peer supports nothing else, sign with SHA-1.
     SSL_SIGN_ECDSA_SHA1,
     SSL_SIGN_RSA_PKCS1_SHA1,
@@ -344,8 +358,32 @@ bool tls12_add_verify_sigalgs(const SSL_HANDSHAKE *hs, CBB *out) {
   return true;
 }
 
+// sigalg_valid_for_protocol_version returns whether |sigalg| is permitted at
+// the negotiated TLS protocol version. ML-DSA (draft-ietf-tls-mldsa §3.3) is
+// defined only for TLS 1.3; a peer that receives ML-DSA in a TLS 1.2
+// ServerKeyExchange or CertificateVerify MUST abort with illegal_parameter.
+static bool sigalg_valid_for_protocol_version(const SSL *ssl, uint16_t sigalg) {
+  switch (sigalg) {
+    case SSL_SIGN_MLDSA44:
+    case SSL_SIGN_MLDSA65:
+    case SSL_SIGN_MLDSA87:
+      return ssl_protocol_version(ssl) >= TLS1_3_VERSION;
+    default:
+      return true;
+  }
+}
+
 bool tls12_check_peer_sigalg(const SSL_HANDSHAKE *hs, uint8_t *out_alert,
                              uint16_t sigalg) {
+  // Reject sigalgs that are not defined for the negotiated protocol version,
+  // even if they appear in the local verify list (we still advertise them so
+  // a higher version can negotiate them).
+  if (!sigalg_valid_for_protocol_version(hs->ssl, sigalg)) {
+    OPENSSL_PUT_ERROR(SSL, SSL_R_WRONG_SIGNATURE_TYPE);
+    *out_alert = SSL_AD_ILLEGAL_PARAMETER;
+    return false;
+  }
+
   for (uint16_t verify_sigalg : tls12_get_verify_sigalgs(hs)) {
     if (verify_sigalg == sigalg) {
       return true;

ssl/internal.h

@@ -1842,8 +1842,8 @@ struct SSL_HANDSHAKE_HINTS {
 };
 
 struct SSL_HANDSHAKE {
-  explicit SSL_HANDSHAKE(SSL *ssl);
-  ~SSL_HANDSHAKE();
+  OPENSSL_EXPORT explicit SSL_HANDSHAKE(SSL *ssl);
+  OPENSSL_EXPORT ~SSL_HANDSHAKE();
   static constexpr bool kAllowUniquePtr = true;
 
   // ssl is a non-owning pointer to the parent |SSL| object.
@@ -2203,7 +2203,7 @@ struct SSL_HANDSHAKE {
 // so many tickets.
 constexpr size_t kMaxTickets = 16;
 
-UniquePtr<SSL_HANDSHAKE> ssl_handshake_new(SSL *ssl);
+OPENSSL_EXPORT UniquePtr<SSL_HANDSHAKE> ssl_handshake_new(SSL *ssl);
 
 // ssl_check_message_type checks if |msg| has type |type|. If so it returns
 // one. Otherwise, it sends an alert and returns zero.
@@ -2486,8 +2486,9 @@ bool tls12_add_verify_sigalgs(const SSL_HANDSHAKE *hs, CBB *out);
 // tls12_check_peer_sigalg checks if |sigalg| is acceptable for the peer
 // signature. It returns true on success and false on error, setting
 // |*out_alert| to an alert to send.
-bool tls12_check_peer_sigalg(const SSL_HANDSHAKE *hs, uint8_t *out_alert,
-                             uint16_t sigalg);
+OPENSSL_EXPORT bool tls12_check_peer_sigalg(const SSL_HANDSHAKE *hs,
+                                            uint8_t *out_alert,
+                                            uint16_t sigalg);
 
 
 // Underdocumented functions.
@@ -2504,7 +2505,8 @@ bool tls12_check_peer_sigalg(const SSL_HANDSHAKE *hs, uint8_t *out_alert,
 #define SSL_PKEY_RSA 0
 #define SSL_PKEY_ECC 1
 #define SSL_PKEY_ED25519 2
-#define SSL_PKEY_SIZE 3
+#define SSL_PKEY_PQDSA 3
+#define SSL_PKEY_SIZE 4
 
 struct CERT_PKEY {
   UniquePtr<EVP_PKEY> privatekey;

ssl/ssl_cipher.cc

@@ -1328,6 +1328,8 @@ int ssl_get_certificate_slot_index(const EVP_PKEY *pkey) {
       return SSL_PKEY_ECC;
     case EVP_PKEY_ED25519:
       return SSL_PKEY_ED25519;
+    case EVP_PKEY_PQDSA:
+      return SSL_PKEY_PQDSA;
     default:
       return -1;
   }
@@ -1341,6 +1343,11 @@ uint32_t ssl_cipher_auth_mask_for_key(const EVP_PKEY *key) {
     case EVP_PKEY_ED25519:
       // Ed25519 keys in TLS 1.2 repurpose the ECDSA ciphers.
       return SSL_aECDSA;
+    case EVP_PKEY_PQDSA:
+      // ML-DSA is TLS 1.3 only and is not used for TLS <= 1.2 cipher-auth
+      // selection. TLS 1.3 paths gate on |SSL_aGENERIC| in
+      // |tls12_pkey_supports_cipher_auth| instead.
+      return 0;
     default:
       return 0;
   }