crypto: Fix DH private key length type mismatch

Signed-off-by: ruslandoga <ruslandoga+gh@icloud.com>

Author: ruslandoga

lib/crypto/c_src/dh.c

@@ -42,6 +42,7 @@ ERL_NIF_TERM dh_compute_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
 ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {/* (PrivKey|undefined, DHParams=[P,G], 0, Len|0) */
     ErlNifUInt64 len = 0;
+    uint64_t ossl_len = 0;
     int i = 0;
     OSSL_PARAM params[8];
     EVP_PKEY *pkey = NULL, *pkey_gen = NULL;
@@ -89,8 +90,10 @@ ERL_NIF_TERM dh_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
         ret = EXCP_BADARG_N(env, 3, "Bad value of length element");
         goto done;
     }
-    else if (len)
-        params[i++] = OSSL_PARAM_construct_uint64("priv_len", &len);
+    else if (len) {
+        ossl_len = len;
+        params[i++] = OSSL_PARAM_construct_uint64("priv_len", &ossl_len);
+    }
 
     /* End of parameter fetching */
     params[i++] = OSSL_PARAM_construct_end();

lib/crypto/test/crypto_SUITE.erl

@@ -1895,7 +1895,11 @@ do_generate_compute({dh, P, G}) ->
     {UserPub, UserPriv} = crypto:generate_key(dh, [P, G]),
     {HostPub, HostPriv} = crypto:generate_key(dh, [P, G]),
     SharedSecret = crypto:compute_key(dh, HostPub, UserPriv, [P, G]),
-    SharedSecret = crypto:compute_key(dh, UserPub, HostPriv, [P, G]).
+    SharedSecret = crypto:compute_key(dh, UserPub, HostPriv, [P, G]),
+    {UserPubWithLen, UserPrivWithLen} = crypto:generate_key(dh, [P, G, 224]),
+    {HostPubWithLen, HostPrivWithLen} = crypto:generate_key(dh, [P, G, 224]),
+    SharedSecretWithLen = crypto:compute_key(dh, HostPubWithLen, UserPrivWithLen, [P, G]),
+    SharedSecretWithLen = crypto:compute_key(dh, UserPubWithLen, HostPrivWithLen, [P, G]).
 
 do_compute({ecdh = Type, Pub, Priv, Curve, SharedSecret}) ->
     ct:log("~p ~p", [Type,Curve]),