fix(ext/node): improve crypto.generateKeyPair validation (#32620)

## Summary
- Restructures `generateKeyPair` to call `createJob` synchronously so
validation errors throw synchronously (matching Node.js behavior)
- Adds key encoding validation (format, type, cipher, passphrase) with
proper Node.js error codes (`ERR_INVALID_ARG_VALUE`,
`ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS`, `ERR_CRYPTO_UNKNOWN_CIPHER`)
- Adds DH group name validation (`ERR_CRYPTO_UNKNOWN_DH_GROUP`)
- Adds RSA-PSS hash algorithm validation (`ERR_CRYPTO_INVALID_DIGEST`)
- Fixes RSA key generation to return `Result` instead of panicking on
invalid parameters, and validates public exponents to prevent infinite
loops with even/small exponents
- Fixes EC curve error message to match Node.js
- Supports parsing private key PEM/DER in `publicEncrypt` to extract the
public key (matching Node.js behavior)
- Rejects encrypt/decrypt operations on non-RSA key types (rsa-pss, dsa,
ec, etc.) with "operation not supported for this keytype"
- Rejects PKCS1 padding for RSA-PSS keys in sign/verify with "illegal or
unsupported padding mode"
- Enables 6 new Node compat tests: `test-crypto-keygen`,
`test-crypto-keygen-sync`, `test-crypto-keygen-rsa-pss`,
`test-crypto-keygen-eddsa`,
`test-crypto-keygen-key-object-without-encoding`,
`test-crypto-keygen-promisify`

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: bartlomieju

ext/node/polyfills/internal/crypto/cipher.ts

@@ -594,10 +594,30 @@ function _lazyInitDecipherDecoder(self: any, encoding: string) {
   }
 }
 
+const ENCRYPT_UNSUPPORTED_KEY_TYPES = new Set([
+  "rsa-pss",
+  "dsa",
+  "ec",
+  "ed25519",
+  "ed448",
+  "x25519",
+  "x448",
+]);
+
+function checkUnsupportedKeyType(key) {
+  const keyType = isKeyObject(key)
+    ? key.asymmetricKeyType
+    : key?.key?.asymmetricKeyType;
+  if (keyType && ENCRYPT_UNSUPPORTED_KEY_TYPES.has(keyType)) {
+    throw new Error("operation not supported for this keytype");
+  }
+}
+
 export function privateEncrypt(
   privateKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  checkUnsupportedKeyType(privateKey);
   const { data } = prepareKey(privateKey);
   const padding = privateKey.padding || 1;
 
@@ -609,6 +629,7 @@ export function privateDecrypt(
   privateKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  checkUnsupportedKeyType(privateKey);
   const { data } = prepareKey(privateKey);
   const padding = privateKey.padding || 1;
 
@@ -620,6 +641,7 @@ export function publicEncrypt(
   publicKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  checkUnsupportedKeyType(publicKey);
   const { data } = prepareKey(publicKey);
   const padding = publicKey.padding || 1;
 

ext/node/polyfills/internal/crypto/keygen.ts

@@ -12,10 +12,16 @@ import {
   SecretKeyObject,
 } from "ext:deno_node/internal/crypto/keys.ts";
 import {
+  ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS,
+  ERR_CRYPTO_INVALID_DIGEST,
+  ERR_CRYPTO_UNKNOWN_CIPHER,
+  ERR_CRYPTO_UNKNOWN_DH_GROUP,
   ERR_INCOMPATIBLE_OPTION_PAIR,
   ERR_INVALID_ARG_VALUE,
   ERR_MISSING_OPTION,
 } from "ext:deno_node/internal/errors.ts";
+import { getCiphers } from "ext:deno_node/internal/crypto/util.ts";
+import { getHashes } from "ext:deno_node/internal/crypto/hash.ts";
 import {
   validateBuffer,
   validateFunction,
@@ -578,6 +584,8 @@ export function generateKeyPair(
     callback = options;
     options = undefined;
   }
+  validateFunction(callback, "callback");
+
   _generateKeyPair(type, options)
     .then(
       (res) => callback!(null, res.publicKey, res.privateKey),
@@ -819,9 +827,188 @@ export function generateKeyPairSync(
 const kSync = 0;
 const kAsync = 1;
 
+function parseKeyFormat(
+  formatStr: string | undefined,
+  defaultFormat: string | undefined,
+  optionName: string,
+): string | undefined {
+  if (formatStr === undefined && defaultFormat !== undefined) {
+    return defaultFormat;
+  } else if (formatStr === "pem") {
+    return "pem";
+  } else if (formatStr === "der") {
+    return "der";
+  } else if (formatStr === "jwk") {
+    return "jwk";
+  }
+  throw new ERR_INVALID_ARG_VALUE(optionName, formatStr);
+}
+
+function parseKeyType(
+  typeStr: string | undefined,
+  required: boolean,
+  keyType: string | undefined,
+  isPublic: boolean | undefined,
+  optionName: string,
+): string | undefined {
+  if (typeStr === undefined && !required) {
+    return undefined;
+  } else if (typeStr === "pkcs1") {
+    if (keyType !== undefined && keyType !== "rsa") {
+      throw new ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(
+        typeStr,
+        "can only be used for RSA keys",
+      );
+    }
+    return "pkcs1";
+  } else if (typeStr === "spki" && isPublic !== false) {
+    return "spki";
+  } else if (typeStr === "pkcs8" && isPublic !== true) {
+    return "pkcs8";
+  } else if (typeStr === "sec1" && isPublic !== true) {
+    if (keyType !== undefined && keyType !== "ec") {
+      throw new ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(
+        typeStr,
+        "can only be used for EC keys",
+      );
+    }
+    return "sec1";
+  }
+
+  throw new ERR_INVALID_ARG_VALUE(optionName, typeStr);
+}
+
+function option(name: string, objName?: string): string {
+  return objName === undefined
+    ? `options.${name}`
+    : `options.${objName}.${name}`;
+}
+
+function isStringOrBuffer(val: unknown): boolean {
+  return typeof val === "string" ||
+    ArrayBuffer.isView(val) ||
+    val instanceof ArrayBuffer ||
+    val instanceof SharedArrayBuffer;
+}
+
+function parseKeyFormatAndType(
+  enc: any,
+  keyType: string | undefined,
+  isPublic: boolean | undefined,
+  objName?: string,
+) {
+  const { format: formatStr, type: typeStr } = enc;
+
+  const isInput = keyType === undefined;
+  const format = parseKeyFormat(
+    formatStr,
+    isInput ? "pem" : undefined,
+    option("format", objName),
+  );
+
+  const isRequired = (!isInput || format === "der") && format !== "jwk";
+  const type = parseKeyType(
+    typeStr,
+    isRequired,
+    keyType,
+    isPublic,
+    option("type", objName),
+  );
+  return { format, type };
+}
+
+function parsePublicKeyEncoding(
+  enc: any,
+  keyType: string | undefined,
+  objName: string,
+) {
+  validateObject(enc, "options");
+
+  const { format, type } = parseKeyFormatAndType(
+    enc,
+    keyType,
+    keyType ? true : undefined,
+    objName,
+  );
+
+  return { format, type };
+}
+
+function parsePrivateKeyEncoding(
+  enc: any,
+  keyType: string | undefined,
+  objName: string,
+) {
+  validateObject(enc, "options");
+
+  const { format, type } = parseKeyFormatAndType(enc, keyType, false, objName);
+
+  const { cipher, passphrase } = enc;
+
+  if (cipher != null) {
+    if (typeof cipher !== "string") {
+      throw new ERR_INVALID_ARG_VALUE(option("cipher", objName), cipher);
+    }
+    if (!getCiphers().includes(cipher)) {
+      throw new ERR_CRYPTO_UNKNOWN_CIPHER();
+    }
+    if (
+      format === "der" &&
+      (type === "pkcs1" || type === "sec1")
+    ) {
+      throw new ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS(
+        type,
+        "does not support encryption",
+      );
+    }
+  } else if (passphrase !== undefined) {
+    throw new ERR_INVALID_ARG_VALUE(option("cipher", objName), cipher);
+  }
+
+  if (cipher != null && !isStringOrBuffer(passphrase)) {
+    throw new ERR_INVALID_ARG_VALUE(option("passphrase", objName), passphrase);
+  }
+
+  return { format, type, cipher, passphrase };
+}
+
+function validateKeyEncoding(keyType: string, options: any) {
+  if (options == null || typeof options !== "object") return;
+
+  const { publicKeyEncoding, privateKeyEncoding } = options;
+
+  if (publicKeyEncoding != null) {
+    if (typeof publicKeyEncoding === "object") {
+      parsePublicKeyEncoding(publicKeyEncoding, keyType, "publicKeyEncoding");
+    } else {
+      throw new ERR_INVALID_ARG_VALUE(
+        "options.publicKeyEncoding",
+        publicKeyEncoding,
+      );
+    }
+  }
+
+  if (privateKeyEncoding != null) {
+    if (typeof privateKeyEncoding === "object") {
+      parsePrivateKeyEncoding(
+        privateKeyEncoding,
+        keyType,
+        "privateKeyEncoding",
+      );
+    } else {
+      throw new ERR_INVALID_ARG_VALUE(
+        "options.privateKeyEncoding",
+        privateKeyEncoding,
+      );
+    }
+  }
+}
+
 function createJob(mode, type, options) {
   validateString(type, "type");
 
+  validateKeyEncoding(type, options);
+
   if (options !== undefined) {
     validateObject(options, "options");
   }
@@ -867,9 +1054,15 @@ function createJob(mode, type, options) {
       }
       if (hashAlgorithm !== undefined) {
         validateString(hashAlgorithm, "options.hashAlgorithm");
+        if (!getHashes().includes(hashAlgorithm)) {
+          throw new ERR_CRYPTO_INVALID_DIGEST(hashAlgorithm);
+        }
       }
       if (mgf1HashAlgorithm !== undefined) {
         validateString(mgf1HashAlgorithm, "options.mgf1HashAlgorithm");
+        if (!getHashes().includes(mgf1HashAlgorithm)) {
+          throw new ERR_CRYPTO_INVALID_DIGEST(mgf1HashAlgorithm, "MGF1");
+        }
       }
       if (hash !== undefined) {
         process.emitWarning(
@@ -995,6 +1188,13 @@ function createJob(mode, type, options) {
 
         validateString(group, "options.group");
 
+        if (
+          group !== "modp5" && group !== "modp14" && group !== "modp15" &&
+          group !== "modp16" && group !== "modp17" && group !== "modp18"
+        ) {
+          throw new ERR_CRYPTO_UNKNOWN_DH_GROUP();
+        }
+
         if (mode === kSync) {
           return op_node_generate_dh_group_key(group);
         } else {

ext/node/polyfills/internal/errors.ts

@@ -1000,6 +1000,15 @@ export class ERR_CRYPTO_UNKNOWN_DH_GROUP extends NodeError {
   }
 }
 
+export class ERR_CRYPTO_UNKNOWN_CIPHER extends NodeError {
+  constructor() {
+    super(
+      "ERR_CRYPTO_UNKNOWN_CIPHER",
+      "Unknown cipher",
+    );
+  }
+}
+
 export class ERR_CRYPTO_ENGINE_UNKNOWN extends NodeError {
   constructor(x: string) {
     super("ERR_CRYPTO_ENGINE_UNKNOWN", `Engine "${x}" was not found`);
@@ -1052,8 +1061,11 @@ export class ERR_CRYPTO_INCOMPATIBLE_KEY_OPTIONS extends NodeError {
 }
 
 export class ERR_CRYPTO_INVALID_DIGEST extends NodeTypeError {
-  constructor(x: string) {
-    super("ERR_CRYPTO_INVALID_DIGEST", `Invalid digest: ${x}`);
+  constructor(x: string, prefix?: string) {
+    super(
+      "ERR_CRYPTO_INVALID_DIGEST",
+      prefix ? `Invalid ${prefix} digest: ${x}` : `Invalid digest: ${x}`,
+    );
   }
 }
 

ext/node_crypto/keys.rs

@@ -2536,26 +2536,33 @@ pub fn op_node_get_private_key_from_pair(
 fn generate_rsa(
   modulus_length: usize,
   public_exponent: usize,
-) -> KeyObjectHandlePair {
+) -> Result<KeyObjectHandlePair, JsErrorBox> {
+  if public_exponent <= 1 || public_exponent.is_multiple_of(2) {
+    return Err(JsErrorBox::generic(format!(
+      "invalid RSA public exponent: {}",
+      public_exponent
+    )));
+  }
+
   let private_key = RsaPrivateKey::new_with_exp(
     &mut thread_rng(),
     modulus_length,
     &rsa::BigUint::from_usize(public_exponent).unwrap(),
   )
-  .unwrap();
+  .map_err(|e| JsErrorBox::generic(e.to_string()))?;
 
   let private_key = AsymmetricPrivateKey::Rsa(private_key);
   let public_key = private_key.to_public_key();
 
-  KeyObjectHandlePair::new(private_key, public_key)
+  Ok(KeyObjectHandlePair::new(private_key, public_key))
 }
 
 #[op2]
 #[cppgc]
 pub fn op_node_generate_rsa_key(
   #[smi] modulus_length: usize,
   #[smi] public_exponent: usize,
-) -> KeyObjectHandlePair {
+) -> Result<KeyObjectHandlePair, JsErrorBox> {
   generate_rsa(modulus_length, public_exponent)
 }
 
@@ -2564,7 +2571,7 @@ pub fn op_node_generate_rsa_key(
 pub async fn op_node_generate_rsa_key_async(
   #[smi] modulus_length: usize,
   #[smi] public_exponent: usize,
-) -> KeyObjectHandlePair {
+) -> Result<KeyObjectHandlePair, JsErrorBox> {
   spawn_blocking(move || generate_rsa(modulus_length, public_exponent))
     .await
     .unwrap()
@@ -2747,10 +2754,7 @@ fn ec_generate(named_curve: &str) -> Result<KeyObjectHandlePair, JsErrorBox> {
       AsymmetricPrivateKey::Ec(EcPrivateKey::Secp256k1(key))
     }
     _ => {
-      return Err(JsErrorBox::type_error(format!(
-        "unsupported named curve: {}",
-        named_curve
-      )));
+      return Err(JsErrorBox::type_error("Invalid EC curve name"));
     }
   };
   let public_key = private_key.to_public_key();