Merge 'sverker/crypto/dh-max-key-length' into maint

OTP-19872

Author: sverker

lib/crypto/c_src/bn.c

@@ -209,13 +209,14 @@ int get_ossl_octet_string_param_from_bin(ErlNifEnv* env, const char* key, ERL_NI
 }
 
 
-int get_ossl_BN_param_from_bin(ErlNifEnv* env, char* key, ERL_NIF_TERM bin, OSSL_PARAM *dest)
+int get_ossl_BN_param_from_bin_x(ErlNifEnv* env, char* key, ERL_NIF_TERM bin,
+                                 OSSL_PARAM *dest, BIGNUM** bn_out)
 {
-    return get_ossl_BN_param_from_bin_sz(env, key, bin, dest, NULL);
+    return get_ossl_BN_param_from_bin_sz_x(env, key, bin, dest, NULL, bn_out);
 }
 
-int get_ossl_BN_param_from_bin_sz(ErlNifEnv* env, char* key, ERL_NIF_TERM bin,
-                                  OSSL_PARAM *dest, size_t *size)
+int get_ossl_BN_param_from_bin_sz_x(ErlNifEnv* env, char* key, ERL_NIF_TERM bin,
+                                    OSSL_PARAM *dest, size_t *size, BIGNUM** bn_out)
 {
     BIGNUM *bn = NULL;
     int ok = 0;
@@ -224,7 +225,11 @@ int get_ossl_BN_param_from_bin_sz(ErlNifEnv* env, char* key, ERL_NIF_TERM bin,
         return 0;
 
     ok = get_ossl_BN_param_from_bn(env, key, bn, dest);
-    BN_free(bn);
+    if (ok && bn_out) {
+        *bn_out = bn;
+    } else {
+        BN_free(bn);
+    }
     return ok;
 }
 
@@ -246,13 +251,15 @@ int get_ossl_BN_param_from_bn(ErlNifEnv* env, char* key, const BIGNUM* bn,
 
 
 
-int get_ossl_param_from_bin_in_list(ErlNifEnv* env, char* key, ERL_NIF_TERM *listcell, OSSL_PARAM *dest)
+int get_ossl_param_from_bin_in_list_x(ErlNifEnv* env, char* key,
+                                      ERL_NIF_TERM *listcell, OSSL_PARAM *dest,
+                                      BIGNUM** bn_out)
 {
     ERL_NIF_TERM head;
 
     return
         enif_get_list_cell(env, *listcell, &head, listcell) &&
-        get_ossl_BN_param_from_bin(env, key, head, dest);
+        get_ossl_BN_param_from_bin_x(env, key, head, dest, bn_out);
 }
 
 #endif

lib/crypto/c_src/bn.h

@@ -38,11 +38,17 @@ int get_bn_from_bin_sz(ErlNifEnv* env, ERL_NIF_TERM term, BIGNUM** bnp, size_t*
 
 #ifdef HAS_3_0_API
 int get_ossl_octet_string_param_from_bin(ErlNifEnv* env, const char* key, ERL_NIF_TERM bin, OSSL_PARAM *dest);
-int get_ossl_BN_param_from_bin(ErlNifEnv* env, char* key, ERL_NIF_TERM bin, OSSL_PARAM *dest);
-int get_ossl_BN_param_from_bin_sz(ErlNifEnv* env, char* key, ERL_NIF_TERM bin, OSSL_PARAM *dest, size_t *size);
+int get_ossl_BN_param_from_bin_x(ErlNifEnv* env, char* key, ERL_NIF_TERM bin, OSSL_PARAM *dest, BIGNUM** bn_out);
+#define get_ossl_BN_param_from_bin(ENV,KEY,BIN,DEST) get_ossl_BN_param_from_bin_x(ENV,KEY,BIN,DEST,NULL)
+
+int get_ossl_BN_param_from_bin_sz_x(ErlNifEnv* env, char* key, ERL_NIF_TERM bin, OSSL_PARAM *dest, size_t *size, BIGNUM** bn_out);
+#define get_ossl_BN_param_from_bin_sz(ENV,KEY,BIN,DEST,SIZE) get_ossl_BN_param_from_bin_sz_x(ENV,KEY,BIN,DEST,SIZE,NULL)
+
 int get_ossl_BN_param_from_bn(ErlNifEnv* env, char* key, const BIGNUM* bn, OSSL_PARAM *dest);
 
-int get_ossl_param_from_bin_in_list(ErlNifEnv* env, char* key, ERL_NIF_TERM *listcell, OSSL_PARAM *dest);
+int get_ossl_param_from_bin_in_list_x(ErlNifEnv* env, char* key, ERL_NIF_TERM *listcell, OSSL_PARAM *dest, BIGNUM** bn_out);
+#define get_ossl_param_from_bin_in_list(ENV,KEY,CELL,DEST) get_ossl_param_from_bin_in_list_x(ENV,KEY,CELL,DEST,NULL)
+
 #endif
 
 #endif /* E_BN_H__ */

lib/crypto/c_src/dh.c

@@ -48,6 +48,7 @@ ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
     OSSL_PARAM params[8];
     EVP_PKEY *pkey = NULL, *pkey_gen = NULL;
     EVP_PKEY_CTX *pctx = NULL, *pctx_gen = NULL;
+    BIGNUM *p_bn =  NULL;
     BIGNUM *pub_key_gen = NULL, *priv_key_gen = NULL;
     unsigned char *pub_ptr, *prv_ptr;
     int pub_len, prv_len;
@@ -64,7 +65,7 @@ ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
         ERL_NIF_TERM head, tail;
 
         head = argv[1];
-        if (!get_ossl_param_from_bin_in_list(env, "p",  &head, &params[i++]) ) {
+        if (!get_ossl_param_from_bin_in_list_x(env, "p",  &head, &params[i++], &p_bn) ) {
             ret = EXCP_BADARG_N(env, 1, "Bad value of 'p'");
             goto done;
         }
@@ -91,9 +92,13 @@ ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
         ret = EXCP_BADARG_N(env, 3, "Bad value of length element");
         goto done;
     }
-    else if (len)
+    else if (len) {
+        if (len >= BN_num_bits(p_bn)) {
+            len = BN_num_bits(p_bn) - 1;
+        }
         params[i++] = OSSL_PARAM_construct_uint64("priv_len", &len);
-
+    }
+    
     /* End of parameter fetching */
     params[i++] = OSSL_PARAM_construct_end();
 
@@ -159,6 +164,7 @@ ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
     ret = enif_make_tuple2(env, ret_pub, ret_prv);
 
  done:
+    if (p_bn) BN_free(p_bn);
     if (pub_key_gen)  BN_free(pub_key_gen);
     if (priv_key_gen) BN_free(priv_key_gen);
     if (pkey) EVP_PKEY_free(pkey);
@@ -334,9 +340,9 @@ ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
         if ((bn_len = BN_num_bits(dh_p_shared)) < 0)
             assign_goto(ret, err, EXCP_ERROR(env, "BN_num_bits < 0"));
         dh_p_shared = NULL;  /* dh_params owns the reference */
-        if (len >= (size_t)bn_len)
-            assign_goto(ret, err, EXCP_ERROR_N(env, 3, "Too big length"));
-
+        if (len >= (size_t)bn_len) {
+            len = bn_len - 1;
+        }
         if (!DH_set_length(dh_params, (long)len))
             assign_goto(ret, err, EXCP_ERROR_N(env, 3, "The length is not accepted"));
     }

lib/crypto/src/crypto.erl

@@ -454,11 +454,11 @@ Scrambler is `u` (optional will be generated if not provided) from
 -type dh_private() :: key_integer() .
 -doc """
 ```text
-dh_params() = [P, G] | [P, G, PrivateKeyBitLength]
+dh_params() = [P, G] | [P, G, MaxPrivateKeyBitLength]
 ```
 """.
 -doc(#{group => <<"Diffie-Hellman Keys and parameters">>}).
--type dh_params() :: [key_integer()] . % [P, G] | [P, G, PrivateKeyBitLength]
+-type dh_params() :: [key_integer()] . % [P, G] | [P, G, MaxPrivateKeyBitLength]
 
 -doc(#{group => <<"Diffie-Hellman Keys and parameters">>,
        equiv => ecdh_params()}).
@@ -2963,7 +2963,7 @@ Uses the [3-tuple style](`m:crypto#error_3tup`) for error handling.
 > - and the `Type` is `dh` (diffie-hellman)
 > - and the parameter `P` (in `t:dh_params/0`) is one of the MODP groups (see
 >   [RFC 3526](https://tools.ietf.org/html/rfc3526))
-> - and the optional `PrivateKeyBitLength` parameter (in `t:dh_params/0`) is
+> - and the optional `MaxPrivateKeyBitLength` parameter (in `t:dh_params/0`) is
 >   present,
 >
 > then the optional key length parameter must be at least 224, 256, 302, 352 and