ssl: Skip undecodable certificate_authorities names

The TLS CertificateRequest certificate_authorities list is only a hint for client certificate selection. Some real servers advertise CA names that violate PKIX type constraints, for example countryName encoded as UTF8String.

Previously ssl_handshake decoded every advertised CA name with public_key:pkix_normalize_name/1. One malformed advisory name could therefore abort the whole handshake with a fatal decode_error, even when the client had no certificate configured.

Catch normalization failures and drop only the bad CA name, keeping any valid names in the list. Add a regression test covering a malformed DN from GH-11338 alongside a valid CA name.

Fixes #11338

Author: ausimian

lib/ssl/src/ssl_handshake.erl

@@ -3480,7 +3480,13 @@ decode_psk_binders(<<?BYTE(Len), Binder:Len/binary, Rest/binary>>, Acc) ->
 decode_cert_auths(<<>>, Acc) ->
     lists:reverse(Acc);
 decode_cert_auths(<<?UINT16(Len), Auth:Len/binary, Rest/binary>>, Acc) ->
-    decode_cert_auths(Rest, [public_key:pkix_normalize_name(Auth) | Acc]).
+    try public_key:pkix_normalize_name(Auth) of
+        CertAuth ->
+            decode_cert_auths(Rest, [CertAuth | Acc])
+    catch
+        _:_ ->
+            decode_cert_auths(Rest, Acc)
+    end.
 
 %% encode/decode stream of certificate data to/from list of certificate data
 certs_to_list(ASN1Certs) ->

lib/ssl/test/ssl_handshake_SUITE.erl

@@ -55,7 +55,8 @@
          select_proper_tls_1_2_rsa_default_hashsign/1,
          ignore_hassign_extension_pre_tls_1_2/1,
          signature_algorithms/1,
-         drop_unassigned_signature_algorithms/1]).
+         drop_unassigned_signature_algorithms/1,
+         drop_undecodable_certificate_authorities/1]).
 
 %%--------------------------------------------------------------------
 %% Common Test interface functions -----------------------------------
@@ -70,7 +71,8 @@ all() -> [decode_hello_handshake,
 	  select_proper_tls_1_2_rsa_default_hashsign,
 	  ignore_hassign_extension_pre_tls_1_2,
 	  signature_algorithms,
-      drop_unassigned_signature_algorithms].
+          drop_unassigned_signature_algorithms,
+          drop_undecodable_certificate_authorities].
 
 %%--------------------------------------------------------------------
 init_per_suite(Config) ->
@@ -294,6 +296,26 @@ drop_unassigned_signature_algorithms(_Config) ->
         end,
         SigAlgs).
 
+drop_undecodable_certificate_authorities(_Config) ->
+    GoodAuth0 = {rdnSequence,
+                 [[{'AttributeTypeAndValue', ?'id-at-commonName',
+                    {utf8String, <<"Good CA">>}}]]},
+    GoodDN = public_key:pkix_encode('Name', GoodAuth0, otp),
+    BadDN = base64:decode(<<"MHAxCzAJBgNVBAYMAkJSMRMwEQYDVQQKDApJQ1AtQnJhc2lsMTQwMgY",
+                            "DVQQLDCtBdXRvcmlkYWRlIENlcnRpZmljYWRvcmEgUmFpeiBCcmFz",
+                            "aWxlaXJhIHY1MRYwFAYDVQQDDA1BQyBTeW5ndWxhcklE">>),
+    CertAuths = cert_auths_vector([BadDN, GoodDN]),
+    HashSigns = <<(ssl_cipher:signature_scheme({sha256, rsa})):16>>,
+    HashSignsLen = byte_size(HashSigns),
+    CertAuthsLen = byte_size(CertAuths),
+    CertReq = <<?BYTE(1), ?BYTE(?RSA_SIGN),
+                ?UINT16(HashSignsLen), HashSigns/binary,
+                ?UINT16(CertAuthsLen), CertAuths/binary>>,
+    GoodAuth = public_key:pkix_normalize_name(GoodAuth0),
+
+    #certificate_request{certificate_authorities = [GoodAuth]} =
+        ssl_handshake:decode_handshake(?TLS_1_2, ?CERTIFICATE_REQUEST, CertReq).
+
 
 %%--------------------------------------------------------------------
 %% Internal functions ------------------------------------------------
@@ -302,3 +324,9 @@ drop_unassigned_signature_algorithms(_Config) ->
 is_supported(Hash) ->
     Hashs = crypto:supports(hashs),
     lists:member(Hash, Hashs).
+
+cert_auths_vector(Auths) ->
+    list_to_binary([begin
+                        Len = byte_size(Auth),
+                        <<?UINT16(Len), Auth/binary>>
+                    end || Auth <- Auths]).