crypto: Fix DH private key length type mismatch

Signed-off-by: ruslandoga <ruslandoga+gh@icloud.com>

Author: ruslandoga

lib/crypto/c_src/dh.c

@@ -44,6 +44,7 @@ ERL_NIF_TERM dh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
 ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {/* (PrivKey|undefined, DHParams=[P,G], 0, Len|0) */
     ErlNifUInt64 len = 0;
+    uint64_t ossl_len = 0;
     int i = 0;
     OSSL_PARAM params[8];
     EVP_PKEY *pkey = NULL, *pkey_gen = NULL;
@@ -96,7 +97,8 @@ ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
         if (len >= BN_num_bits(p_bn)) {
             len = BN_num_bits(p_bn) - 1;
         }
-        params[i++] = OSSL_PARAM_construct_uint64("priv_len", &len);
+        ossl_len = len;
+        params[i++] = OSSL_PARAM_construct_uint64("priv_len", &ossl_len);
     }
 
     /* End of parameter fetching */

lib/crypto/test/crypto_SUITE.erl

@@ -2189,7 +2189,11 @@ do_generate_compute({dh, P, G}) ->
     {UserPub, UserPriv} = crypto:generate_key(dh, [P, G]),
     {HostPub, HostPriv} = crypto:generate_key(dh, [P, G]),
     SharedSecret = crypto:compute_key(dh, HostPub, UserPriv, [P, G]),
-    SharedSecret = crypto:compute_key(dh, UserPub, HostPriv, [P, G]).
+    SharedSecret = crypto:compute_key(dh, UserPub, HostPriv, [P, G]),
+    {UserPubWithLen, UserPrivWithLen} = crypto:generate_key(dh, [P, G, 224]),
+    {HostPubWithLen, HostPrivWithLen} = crypto:generate_key(dh, [P, G, 224]),
+    SharedSecretWithLen = crypto:compute_key(dh, HostPubWithLen, UserPrivWithLen, [P, G]),
+    SharedSecretWithLen = crypto:compute_key(dh, UserPubWithLen, HostPrivWithLen, [P, G]).
 
 do_compute({ecdh = Type, Pub, Priv, Curve, SharedSecret}) ->
     ct:log("~p ~p", [Type,Curve]),