fix(ext/node): implement ECDH.convertKey and fix related ECDH bugs (#32532)

## Summary
- Implement `ECDH.convertKey()` static method supporting compressed,
uncompressed, and hybrid point formats
- Fix `setPrivateKey()` not decoding string keys with encoding parameter
- Fix `getPublicKey()` not supporting hybrid format
- Fix `op_node_ecdh_compute_public_key` using `to_sec1_bytes()`
(compressed, 33 bytes) instead of `to_encoded_point(false)`
(uncompressed, 65 bytes), which caused a panic on `copy_from_slice`
- Enable `test-crypto-ecdh-convert-key.js` in node_compat config

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: bartlomieju

ext/node/polyfills/internal/crypto/diffiehellman.ts

@@ -15,12 +15,12 @@ import {
   op_node_gen_prime,
 } from "ext:core/ops";
 
-import { notImplemented } from "ext:deno_node/_utils.ts";
 import {
   isAnyArrayBuffer,
   isArrayBufferView,
 } from "ext:deno_node/internal/util/types.ts";
 import {
+  ERR_CRYPTO_ECDH_INVALID_FORMAT,
   ERR_CRYPTO_UNKNOWN_DH_GROUP,
   ERR_INVALID_ARG_TYPE,
   NodeError,
@@ -1230,13 +1230,55 @@ export class ECDHImpl {
   }
 
   static convertKey(
-    _key: BinaryLike,
-    _curve: string,
-    _inputEncoding?: BinaryToTextEncoding,
-    _outputEncoding?: "latin1" | "hex" | "base64" | "base64url",
-    _format?: "uncompressed" | "compressed" | "hybrid",
+    key: BinaryLike,
+    curve: string,
+    inputEncoding?: BinaryToTextEncoding,
+    outputEncoding?: "latin1" | "hex" | "base64" | "base64url",
+    format?: "uncompressed" | "compressed" | "hybrid",
   ): Buffer | string {
-    notImplemented("crypto.ECDH.prototype.convertKey");
+    validateString(curve, "curve");
+    const buf = getArrayBufferOrView(key, "key", inputEncoding);
+
+    let compress: boolean;
+    if (format) {
+      if (format === "compressed") {
+        compress = true;
+      } else if (format === "hybrid" || format === "uncompressed") {
+        compress = false;
+      } else {
+        throw new ERR_CRYPTO_ECDH_INVALID_FORMAT(format);
+      }
+    } else {
+      compress = false;
+    }
+
+    let result;
+    try {
+      result = Buffer.from(
+        op_node_ecdh_encode_pubkey(curve, buf, compress),
+      );
+    } catch (e) {
+      if (e instanceof TypeError && e.message === "Unsupported curve") {
+        throw new TypeError("Invalid EC curve name");
+      }
+      throw new Error("Failed to convert Buffer to EC_POINT");
+    }
+
+    if (format === "hybrid") {
+      // Hybrid format: same as uncompressed but first byte is 06 or 07
+      // Get compressed form to determine parity
+      const compressedBuf = Buffer.from(
+        op_node_ecdh_encode_pubkey(curve, buf, true),
+      );
+      // compressed first byte is 02 (even) or 03 (odd)
+      // hybrid first byte is 06 (even) or 07 (odd)
+      result[0] = compressedBuf[0] + 4;
+    }
+
+    if (outputEncoding && outputEncoding !== "buffer") {
+      return result.toString(outputEncoding);
+    }
+    return result;
   }
 
   computeSecret(otherPublicKey: ArrayBufferView): Buffer;
@@ -1312,8 +1354,16 @@ export class ECDHImpl {
     const pubbuf = Buffer.from(op_node_ecdh_encode_pubkey(
       this.#curve.name,
       this.#pubbuf,
-      format == "compressed",
+      format === "compressed",
     ));
+    if (format === "hybrid") {
+      const compressedBuf = Buffer.from(op_node_ecdh_encode_pubkey(
+        this.#curve.name,
+        this.#pubbuf,
+        true,
+      ));
+      pubbuf[0] = compressedBuf[0] + 4;
+    }
     if (encoding !== undefined) {
       return pubbuf.toString(encoding);
     }
@@ -1326,7 +1376,9 @@ export class ECDHImpl {
     privateKey: ArrayBufferView | string,
     encoding?: BinaryToTextEncoding,
   ): Buffer | string {
-    this.#privbuf = privateKey;
+    this.#privbuf = typeof privateKey === "string"
+      ? Buffer.from(privateKey, encoding)
+      : Buffer.from(privateKey);
     this.#pubbuf = Buffer.alloc(this.#curve.publicKeySize);
 
     op_node_ecdh_compute_public_key(
@@ -1343,6 +1395,7 @@ export class ECDHImpl {
 }
 
 ECDH.prototype = ECDHImpl.prototype;
+ECDH.convertKey = ECDHImpl.convertKey;
 
 export function diffieHellman(options: {
   privateKey: KeyObject;

ext/node_crypto/lib.rs

@@ -1118,35 +1118,35 @@ pub fn op_node_ecdh_compute_public_key(
         elliptic_curve::SecretKey::<k256::Secp256k1>::from_slice(privkey)
           .expect("bad private key");
       let public_key = this_private_key.public_key();
-      pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref());
+      pubkey.copy_from_slice(public_key.to_encoded_point(false).as_ref());
     }
     "prime256v1" | "secp256r1" => {
       let this_private_key =
         elliptic_curve::SecretKey::<NistP256>::from_slice(privkey)
           .expect("bad private key");
       let public_key = this_private_key.public_key();
-      pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref());
+      pubkey.copy_from_slice(public_key.to_encoded_point(false).as_ref());
     }
     "secp384r1" => {
       let this_private_key =
         elliptic_curve::SecretKey::<NistP384>::from_slice(privkey)
           .expect("bad private key");
       let public_key = this_private_key.public_key();
-      pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref());
+      pubkey.copy_from_slice(public_key.to_encoded_point(false).as_ref());
     }
     "secp521r1" => {
       let this_private_key =
         elliptic_curve::SecretKey::<NistP521>::from_slice(privkey)
           .expect("bad private key");
       let public_key = this_private_key.public_key();
-      pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref());
+      pubkey.copy_from_slice(public_key.to_encoded_point(false).as_ref());
     }
     "secp224r1" => {
       let this_private_key =
         elliptic_curve::SecretKey::<NistP224>::from_slice(privkey)
           .expect("bad private key");
       let public_key = this_private_key.public_key();
-      pubkey.copy_from_slice(public_key.to_sec1_bytes().as_ref());
+      pubkey.copy_from_slice(public_key.to_encoded_point(false).as_ref());
     }
     &_ => todo!(),
   }

tests/node_compat/config.jsonc

@@ -262,6 +262,7 @@
     "parallel/test-crypto-dh-shared.js": {},
     "parallel/test-crypto-domain.js": {},
     "parallel/test-crypto-domains.js": {},
+    "parallel/test-crypto-ecdh-convert-key.js": {},
     "parallel/test-crypto-ecb.js": {},
     "parallel/test-crypto-from-binary.js": {},
     "parallel/test-crypto-keygen-async-elliptic-curve-jwk-rsa.js": {},