ssl: For TLS-1.2 cert signature must be compatible with cipher suite

In TLS-1.3 this is negotiated separtly

Author: IngelaAndin

lib/ssl/src/ssl_handshake.erl

@@ -1146,38 +1146,71 @@ server_select_cert_key_pair_and_params(CipherSuites, [#{private_key := NoKey, ce
     CipherSuite0 = select_cipher_suite(CipherSuites, Suites, HonorCipherOrder),
     CurveAndSuite = cert_curve(undefined, ECCCurve0, CipherSuite0),
     {NoCerts, NoKey, CurveAndSuite};
-server_select_cert_key_pair_and_params(CipherSuites, [#{private_key := Key, certs := [Cert | _] = Certs}], HashSigns, ECCCurve0,
+server_select_cert_key_pair_and_params(CipherSuites0, [#{private_key := Key, certs := [Cert | _] = Certs}], HashSigns, ECCCurve0,
                                 #{ciphers := UserSuites, honor_cipher_order := HonorCipherOrder}, Version) ->
     Suites = available_suites(Cert, UserSuites, Version, HashSigns, ECCCurve0),
+    CipherSuites = [ Suite || Suite <- CipherSuites0,
+                              is_acceptable_cert(Suite, Cert, HashSigns, ssl:tls_version(Version))],
     CipherSuite0 = select_cipher_suite(CipherSuites, Suites, HonorCipherOrder),
     CurveAndSuite = cert_curve(Cert, ECCCurve0, CipherSuite0),
     {Certs, Key, CurveAndSuite};
-server_select_cert_key_pair_and_params(CipherSuites, [#{private_key := Key, certs := [Cert | _] = Certs} | Rest], HashSigns, ECCCurve0,
+server_select_cert_key_pair_and_params(CipherSuites0, [#{private_key := Key, certs := [Cert | _] = Certs} | Rest], HashSigns, ECCCurve0,
                  #{ciphers := UserSuites, honor_cipher_order := HonorCipherOrder} = Opts, Version) ->
     Suites = available_suites(Cert, UserSuites, Version, HashSigns, ECCCurve0),
+    CipherSuites = [ Suite || Suite <- CipherSuites0, is_acceptable_cert(Suite, Cert, HashSigns, ssl:tls_version(Version))],
     case select_cipher_suite(CipherSuites, Suites, HonorCipherOrder) of
         no_suite ->
-            server_select_cert_key_pair_and_params(CipherSuites, Rest, HashSigns, ECCCurve0, Opts, Version);
+            server_select_cert_key_pair_and_params(CipherSuites0, Rest, HashSigns, ECCCurve0, Opts, Version);
         CipherSuite0 ->
-            case is_acceptable_cert(Cert, HashSigns, ssl:tls_version(Version)) of
-                true ->
-                    CurveAndSuite = cert_curve(Cert, ECCCurve0, CipherSuite0),
-                    {Certs, Key, CurveAndSuite};
-                false ->
-                    server_select_cert_key_pair_and_params(CipherSuites, Rest, HashSigns, ECCCurve0, Opts, Version)
-            end
+            CurveAndSuite = cert_curve(Cert, ECCCurve0, CipherSuite0),
+            {Certs, Key, CurveAndSuite}
     end.
 
-is_acceptable_cert(Cert, HashSigns, Version)
+is_acceptable_cert(CipherSuite, Cert, HashSigns, Version)
   when ?TLS_1_X(Version),
        ?TLS_GTE(Version, ?TLS_1_2) ->
     {SignAlgo0, Param, _, _, _} = get_cert_params(Cert),
     SignAlgo = sign_algo(SignAlgo0, Param),
-    is_acceptable_hash_sign(SignAlgo, HashSigns);
-is_acceptable_cert(_,_,_) ->
+    is_acceptable_hash_sign(SignAlgo, acceptable_hash_signs(CipherSuite, HashSigns));
+is_acceptable_cert(_, _,_,_) ->
     %% Not negotiable pre TLS-1.2. So if cert is available for version it is acceptable
     true.
 
+acceptable_hash_signs(CipherSuite, HashSigns) ->
+    #{key_exchange := Kex} = ssl_cipher_format:suite_bin_to_map(CipherSuite),
+    Filter = fun({_, rsa_pss_pss}) when Kex == ecdh_rsa;
+                                        Kex == ecdhe_rsa;
+                                        Kex == dhe_rsa;
+                                        Kex == srp_rsa;
+                                        Kex == rsa_psk ->
+
+                     true;
+                ({_, rsa_pss_rsae}) when Kex == ecdh_rsa;
+                                         Kex == ecdhe_rsa;
+                                         Kex == dhe_rsa;
+                                         Kex == srp_rsa;
+                                         Kex == rsa_psk ->
+                     true;
+                ({_, rsa}) when Kex == ecdh_rsa;
+                                Kex == ecdhe_rsa;
+                                Kex == dhe_rsa;
+                                Kex == srp_rsa;
+                                Kex == rsa_psk;
+                                Kex == rsa->
+                     true;
+                ({_, ecdsa}) when Kex == ecdh_ecdsa;
+                                  Kex == ecdhe_ecdsa ->
+                     true;
+                ({_, dsa}) when Kex == dhe_dss;
+                                Kex == srp_dss ->
+                     true;
+                (_) when Kex == any ->
+                     true;
+                (_) ->
+                     false
+             end,
+    [HS || HS <- HashSigns, Filter(HS)].
+
 premaster_secret(OtherPublicDhKey, MyPrivateKey, #'DHParameter'{} = Params) ->
     try
 	public_key:compute_key(OtherPublicDhKey, MyPrivateKey, Params)
@@ -3658,8 +3691,11 @@ handle_srp_extension(undefined, Session) ->
 handle_srp_extension(#srp{username = Username}, Session) ->
     Session#session{srp_username = Username}.
 
+is_acceptable_hash_sign({SHA, rsa} = Algos, SupportedHashSigns) ->
+    lists:member({SHA, rsa_pss_rsae}, SupportedHashSigns) orelse
+        lists:member(Algos, SupportedHashSigns);
 is_acceptable_hash_sign(Algos, SupportedHashSigns) ->
-    lists:member(Algos, SupportedHashSigns).
+        lists:member(Algos, SupportedHashSigns).
 
 is_acceptable_cert_type(Sign, Types) ->
     lists:member(sign_type(Sign), binary_to_list(Types)).

lib/ssl/src/tls_handshake.erl

@@ -351,7 +351,7 @@ handle_client_hello(Version,
                                                AvailableHashSigns,
 					       SessIdTracker, Session0#session{ecc = ECCCurve},
                                                Version, SslOpts, CertKeyPairs),
-	    case CipherSuite of
+            case CipherSuite of
 		no_suite ->
                     throw(?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_ciphers));
 		_ ->