Harden decode prototype parity checks

Author: eldryoth

docs/UNSAFE.md

@@ -720,9 +720,10 @@ Safety argument:
   unaligned intrinsics, so no alignment precondition is required.
 - The SSSE3/SSE4.1 target-feature contract enables every intrinsic used by
   the prototype.
-- The prototype copies only the validated decoded length to caller output and
-  wipes local value, packed, and scalar-validation staging buffers before
-  returning.
+- The prototype copies only the validated decoded length to caller output after
+  an unconditional release-mode equality check proves the vector-packed prefix
+  matches the scalar-validation prefix. It wipes local value, packed, and
+  scalar-validation staging buffers before returning.
 - Runtime decode dispatch does not call this function. Future admission must
   update this inventory, generated-code evidence, fuzzing, backend reporting,
   benchmarks, and release notes in the same commit that makes any decode SIMD
@@ -784,9 +785,10 @@ Safety argument:
   prototype.
 - VBMI compaction indices are constants in `0..=59`, so the permute reads only
   bytes produced by the lane-local decode shuffle.
-- The prototype copies only the validated decoded length to caller output and
-  wipes local value, packed, and scalar-validation staging buffers before
-  returning.
+- The prototype copies only the validated decoded length to caller output after
+  an unconditional release-mode equality check proves the vector-packed prefix
+  matches the scalar-validation prefix. It wipes local value, packed, and
+  scalar-validation staging buffers before returning.
 - Runtime decode dispatch does not call this function. Future admission must
   update this inventory, generated-code evidence, fuzzing, backend reporting,
   benchmarks, and release notes in the same commit that makes any decode SIMD
@@ -846,9 +848,10 @@ Safety argument:
 - AVX2 byte shuffle is lane-local, so the prototype explicitly compacts the
   second 12-byte decoded lane from offsets `16..28` to `12..24` inside local
   staging before copying to caller output.
-- The prototype copies only the validated decoded length to caller output and
-  wipes local value, packed, and scalar-validation staging buffers before
-  returning.
+- The prototype copies only the validated decoded length to caller output after
+  an unconditional release-mode equality check proves the vector-packed prefix
+  matches the scalar-validation prefix. It wipes local value, packed, and
+  scalar-validation staging buffers before returning.
 - Runtime decode dispatch does not call this function. Future admission must
   update this inventory, generated-code evidence, fuzzing, backend reporting,
   benchmarks, and release notes in the same commit that makes any decode SIMD
@@ -977,7 +980,9 @@ Safety argument:
   prototype copies bytes to caller-visible output.
 - The input and output array types provide fixed readable and writable bounds.
 - The vector store writes only to a local 16-byte packed buffer; the caller
-  output receives at most the scalar-validated `written` prefix.
+  output receives at most the scalar-validated `written` prefix, and only after
+  an unconditional release-mode equality check proves the vector-packed prefix
+  matches the scalar-validation prefix.
 - The compaction mask contains only valid source indices or zero lanes.
 - Staging, packed, and scalar-output buffers are wiped before successful return
   or along the error path.

src/simd/neon.rs

@@ -287,11 +287,8 @@ where
         clear_neon_registers_after_vector_block!();
     }
 
-    debug_assert_eq!(&packed[..written], &scalar_output[..written]);
-    output[..written].copy_from_slice(&packed[..written]);
     crate::wipe_bytes(&mut values);
-    crate::wipe_bytes(&mut packed);
-    crate::wipe_bytes(&mut scalar_output);
+    copy_verified_decode_output(&mut packed, &mut scalar_output, output, written)?;
     Ok(written)
 }
 
@@ -422,17 +419,42 @@ where
     vbslq_u8(slash, vdupq_n_u8(slash_char), encoded)
 }
 
+#[cfg(all(target_arch = "aarch64", test))]
+fn copy_verified_decode_output<const PACKED: usize, const SCALAR: usize>(
+    packed: &mut [u8; PACKED],
+    scalar_output: &mut [u8; SCALAR],
+    output: &mut [u8],
+    written: usize,
+) -> Result<(), crate::DecodeError> {
+    if packed[..written] != scalar_output[..written] {
+        crate::wipe_bytes(packed);
+        crate::wipe_bytes(scalar_output);
+        return Err(crate::DecodeError::InvalidInput);
+    }
+
+    output[..written].copy_from_slice(&packed[..written]);
+    crate::wipe_bytes(packed);
+    crate::wipe_bytes(scalar_output);
+    Ok(())
+}
+
 #[cfg(all(target_arch = "aarch64", test))]
 fn fill_decode_values<A, const N: usize>(input: &[u8; N], values: &mut [u8; N])
 where
     A: Alphabet,
 {
     let mut index = 0;
     while index < input.len() {
-        values[index] = if input[index] == b'=' {
-            0
-        } else {
-            A::decode(input[index]).unwrap_or(0)
+        values[index] = match input[index] {
+            b'=' => 0,
+            byte => {
+                if let Some(value) = A::decode(byte) {
+                    value
+                } else {
+                    debug_assert!(false, "fill_decode_values called on unvalidated input");
+                    0
+                }
+            }
         };
         index += 1;
     }

src/simd/x86/decode.rs

@@ -66,11 +66,8 @@ where
         super::cleanup::clear_xmm_registers_after_encode_block();
     }
 
-    debug_assert_eq!(&packed[..written], &scalar_output[..written]);
-    output[..written].copy_from_slice(&packed[..written]);
     crate::wipe_bytes(&mut values);
-    crate::wipe_bytes(&mut packed);
-    crate::wipe_bytes(&mut scalar_output);
+    copy_verified_decode_output(&mut packed, &mut scalar_output, output, written)?;
     Ok(written)
 }
 
@@ -127,11 +124,8 @@ where
     ];
     packed[12..24].copy_from_slice(&upper_lane);
 
-    debug_assert_eq!(&packed[..written], &scalar_output[..written]);
-    output[..written].copy_from_slice(&packed[..written]);
     crate::wipe_bytes(&mut values);
-    crate::wipe_bytes(&mut packed);
-    crate::wipe_bytes(&mut scalar_output);
+    copy_verified_decode_output(&mut packed, &mut scalar_output, output, written)?;
     Ok(written)
 }
 
@@ -192,24 +186,45 @@ where
         super::cleanup::clear_zmm_registers_after_encode_block();
     }
 
-    debug_assert_eq!(&packed[..written], &scalar_output[..written]);
-    output[..written].copy_from_slice(&packed[..written]);
     crate::wipe_bytes(&mut values);
-    crate::wipe_bytes(&mut packed);
-    crate::wipe_bytes(&mut scalar_output);
+    copy_verified_decode_output(&mut packed, &mut scalar_output, output, written)?;
     Ok(written)
 }
 
+fn copy_verified_decode_output<const PACKED: usize, const SCALAR: usize>(
+    packed: &mut [u8; PACKED],
+    scalar_output: &mut [u8; SCALAR],
+    output: &mut [u8],
+    written: usize,
+) -> Result<(), DecodeError> {
+    if packed[..written] != scalar_output[..written] {
+        crate::wipe_bytes(packed);
+        crate::wipe_bytes(scalar_output);
+        return Err(DecodeError::InvalidInput);
+    }
+
+    output[..written].copy_from_slice(&packed[..written]);
+    crate::wipe_bytes(packed);
+    crate::wipe_bytes(scalar_output);
+    Ok(())
+}
+
 fn fill_decode_values<A, const N: usize>(input: &[u8; N], values: &mut [u8; N])
 where
     A: Alphabet,
 {
     let mut index = 0;
     while index < input.len() {
-        values[index] = if input[index] == b'=' {
-            0
-        } else {
-            A::decode(input[index]).unwrap_or(0)
+        values[index] = match input[index] {
+            b'=' => 0,
+            byte => {
+                if let Some(value) = A::decode(byte) {
+                    value
+                } else {
+                    debug_assert!(false, "fill_decode_values called on unvalidated input");
+                    0
+                }
+            }
         };
         index += 1;
     }