Ticket #2004 Fix RC4/RC4Drop hex passphrase parsing and add format validation

src/core/lib/Ciphers.mjs

@@ -13,6 +13,7 @@
 import OperationError from "../errors/OperationError.mjs";
 import Utils from "../Utils.mjs";
 import CryptoJS from "crypto-js";
+import { fromHex, toHexFast } from "./Hex.mjs";
 
 /**
  * Affine Cipher Encode operation.
@@ -86,3 +87,64 @@ export const format = {
     "UTF16BE": CryptoJS.enc.Utf16BE,
     "Latin1":  CryptoJS.enc.Latin1,
 };
+
+
+/**
+ * Validates that a passphrase/key string contains only characters valid for the
+ * given format. Recognised delimiters (spaces, commas, colons, 0x prefix, etc.)
+ * are always permitted in Hex and Binary formats.
+ * Throws an OperationError if genuinely invalid characters are found.
+ *
+ * @param {string} str
+ * @param {string} type - One of "Hex", "Base64", "Binary", "UTF8", "Latin1", etc.
+ * @throws {OperationError}
+ */
+function validateFormatInput(str, type) {
+    if (!str) return;
+    switch (type.toLowerCase()) {
+        case "hex": {
+            const stripped = str.replace(/0x|\\x|%|[\s,;:\n\r]/gi, "");
+            const invalid = stripped.match(/[^0-9a-fA-F]/);
+            if (invalid) throw new OperationError(
+                `Invalid character '${invalid[0]}' in Hex input. ` +
+                `Hex accepts 0-9, a-f, A-F, and delimiters (space, comma, colon, 0x prefix).`
+            );
+            break;
+        }
+        case "base64": {
+            const invalid = str.replace(/[\s]/g, "").match(/[^A-Za-z0-9+/=]/);
+            if (invalid) throw new OperationError(
+                `Invalid character '${invalid[0]}' in Base64 input. ` +
+                `Base64 accepts A-Z, a-z, 0-9, +, /, and = (padding).`
+            );
+            break;
+        }
+        case "binary": {
+            const stripped = str.replace(/[\s,;:\n\r]/g, "");
+            const invalid = stripped.match(/[^01]/);
+            if (invalid) throw new OperationError(
+                `Invalid character '${invalid[0]}' in Binary input. Binary accepts only 0 and 1.`
+            );
+            break;
+        }
+    }
+}
+
+
+/**
+ * Parses a user-entered string in a given CryptoJS format, normalising common
+ * hex delimiter conventions (commas, spaces, 0x prefix, etc.) before parsing.
+ *
+ * Use this instead of format[name].parse() for user-supplied passphrase/key inputs.
+ *
+ * @param {string} str
+ * @param {string} formatName - Key into the format map (e.g. "Hex", "UTF8")
+ * @returns {CryptoJS.lib.WordArray}
+ */
+export function parseFormatString(str, formatName) {
+    validateFormatInput(str, formatName);
+    if (formatName === "Hex") {
+        return CryptoJS.enc.Hex.parse(toHexFast(fromHex(str)));
+    }
+    return format[formatName].parse(str);
+}

src/core/operations/DeriveEVPKey.mjs

@@ -121,27 +121,3 @@ CryptoJS.kdf.OpenSSL.execute = function (password, keySize, ivSize, salt) {
     // Return params
     return CryptoJS.lib.CipherParams.create({ key: key, iv: iv, salt: salt });
 };
-
-
-/**
- * Override for the CryptoJS Hex encoding parser to remove whitespace before attempting to parse
- * the hex string.
- *
- * @param {string} hexStr
- * @returns {CryptoJS.lib.WordArray}
- */
-CryptoJS.enc.Hex.parse = function (hexStr) {
-    // Remove whitespace
-    hexStr = hexStr.replace(/\s/g, "");
-
-    // Shortcut
-    const hexStrLength = hexStr.length;
-
-    // Convert
-    const words = [];
-    for (let i = 0; i < hexStrLength; i += 2) {
-        words[i >>> 3] |= parseInt(hexStr.substr(i, 2), 16) << (24 - (i % 8) * 4);
-    }
-
-    return new CryptoJS.lib.WordArray.init(words, hexStrLength / 2);
-};

src/core/operations/RC4.mjs

@@ -6,7 +6,7 @@
 
 import Operation from "../Operation.mjs";
 import CryptoJS from "crypto-js";
-import { format } from "../lib/Ciphers.mjs";
+import { format, parseFormatString } from "../lib/Ciphers.mjs";
 
 /**
  * RC4 operation
@@ -52,7 +52,7 @@ class RC4 extends Operation {
      */
     run(input, args) {
         const message = format[args[1]].parse(input),
-            passphrase = format[args[0].option].parse(args[0].string),
+            passphrase = parseFormatString(args[0].string, args[0].option),
             encrypted = CryptoJS.RC4.encrypt(message, passphrase);
 
         return encrypted.ciphertext.toString(format[args[2]]);

src/core/operations/RC4Drop.mjs

@@ -5,7 +5,7 @@
  */
 
 import Operation from "../Operation.mjs";
-import { format } from "../lib/Ciphers.mjs";
+import { format, parseFormatString } from "../lib/Ciphers.mjs";
 import CryptoJS from "crypto-js";
 
 /**
@@ -57,7 +57,7 @@ class RC4Drop extends Operation {
      */
     run(input, args) {
         const message = format[args[1]].parse(input),
-            passphrase = format[args[0].option].parse(args[0].string),
+            passphrase = parseFormatString(args[0].string, args[0].option),
             drop = args[3],
             encrypted = CryptoJS.RC4Drop.encrypt(message, passphrase, { drop: drop });
 

tests/operations/tests/RC4.mjs

@@ -0,0 +1,148 @@
+/**
+ * RC4 tests.
+ *
+ * @author Stuart Wilson
+ * @copyright Crown Copyright 2025
+ * @license Apache-2.0
+ */
+import TestRegister from "../../lib/TestRegister.mjs";
+
+TestRegister.addTests([
+    {
+        "name": "RC4: Hex passphrase, clean hex (1f10)",
+        "input": "test",
+        "expectedOutput": "8aee3802",
+        "recipeConfig": [
+            {
+                "op": "RC4",
+                "args": [
+                    {"option": "Hex", "string": "1f10"},
+                    "Latin1",
+                    "Hex"
+                ]
+            }
+        ]
+    },
+    {
+        "name": "RC4: Hex passphrase, comma-delimited (1f,10) equals clean hex",
+        "input": "test",
+        "expectedOutput": "8aee3802",
+        "recipeConfig": [
+            {
+                "op": "RC4",
+                "args": [
+                    {"option": "Hex", "string": "1f,10"},
+                    "Latin1",
+                    "Hex"
+                ]
+            }
+        ]
+    },
+    {
+        "name": "RC4: Hex passphrase, 0x-prefixed (0x1f,0x10) equals clean hex",
+        "input": "test",
+        "expectedOutput": "8aee3802",
+        "recipeConfig": [
+            {
+                "op": "RC4",
+                "args": [
+                    {"option": "Hex", "string": "0x1f,0x10"},
+                    "Latin1",
+                    "Hex"
+                ]
+            }
+        ]
+    },
+    {
+        "name": "RC4: Hex passphrase, space-delimited (1f 10) equals clean hex",
+        "input": "test",
+        "expectedOutput": "8aee3802",
+        "recipeConfig": [
+            {
+                "op": "RC4",
+                "args": [
+                    {"option": "Hex", "string": "1f 10"},
+                    "Latin1",
+                    "Hex"
+                ]
+            }
+        ]
+    },
+    {
+        "name": "RC4: Hex passphrase, colon-delimited uppercase (1F:10) equals clean hex",
+        "input": "test",
+        "expectedOutput": "8aee3802",
+        "recipeConfig": [
+            {
+                "op": "RC4",
+                "args": [
+                    {"option": "Hex", "string": "1F:10"},
+                    "Latin1",
+                    "Hex"
+                ]
+            }
+        ]
+    },
+    {
+        "name": "RC4: invalid hex character in passphrase shows error in output",
+        "input": "test",
+        "expectedOutput": "Invalid character 'G' in Hex input. Hex accepts 0-9, a-f, A-F, and delimiters (space, comma, colon, 0x prefix).",
+        "recipeConfig": [
+            {
+                "op": "RC4",
+                "args": [
+                    {"option": "Hex", "string": "1fG0"},
+                    "Latin1",
+                    "Hex"
+                ]
+            }
+        ]
+    },
+    {
+        "name": "RC4: UTF8 passphrase still works",
+        "input": "test",
+        "expectedOutput": "8b904b7a",
+        "recipeConfig": [
+            {
+                "op": "RC4",
+                "args": [
+                    {"option": "UTF8", "string": "password"},
+                    "Latin1",
+                    "Hex"
+                ]
+            }
+        ]
+    },
+    {
+        "name": "RC4 Drop: Hex passphrase, comma-delimited (1f,10) equals clean hex",
+        "input": "test",
+        "expectedOutput": "47055271",
+        "recipeConfig": [
+            {
+                "op": "RC4 Drop",
+                "args": [
+                    {"option": "Hex", "string": "1f,10"},
+                    "Latin1",
+                    "Hex",
+                    192
+                ]
+            }
+        ]
+    },
+    {
+        "name": "RC4 Drop: Hex passphrase, 0x-prefixed equals clean hex",
+        "input": "test",
+        "expectedOutput": "47055271",
+        "recipeConfig": [
+            {
+                "op": "RC4 Drop",
+                "args": [
+                    {"option": "Hex", "string": "0x1f,0x10"},
+                    "Latin1",
+                    "Hex",
+                    192
+                ]
+            }
+        ]
+    }
+]);