Debug more

Author: larseggert

src/aead.rs

@@ -245,7 +245,7 @@ mod recprot {
 mod recprot {
     use std::{
         fmt,
-        os::raw::{c_char, c_int, c_uint},
+        os::raw::{c_char, c_int, c_uint, c_ulong},
         ptr::{null, null_mut},
     };
 
@@ -271,8 +271,24 @@ mod recprot {
         reason = "NONCE_LEN = 12 and TAG_LEN = 16 both fit in u32"
     )]
     const TAG_LEN_C: c_uint = TAG_LEN as c_uint;
-    const NONCE_LEN_UL: u64 = NONCE_LEN as u64;
-    const TAG_BITS_UL: u64 = (TAG_LEN * 8) as u64;
+    // On Windows c_ulong is u32; on other platforms it is u64 (same width as usize),
+    // so cast_possible_truncation only fires on Windows.
+    #[cfg_attr(
+        target_os = "windows",
+        expect(
+            clippy::cast_possible_truncation,
+            reason = "NONCE_LEN = 12 fits in u32"
+        )
+    )]
+    const NONCE_LEN_UL: c_ulong = NONCE_LEN as c_ulong;
+    #[cfg_attr(
+        target_os = "windows",
+        expect(
+            clippy::cast_possible_truncation,
+            reason = "TAG_LEN * 8 = 128 fits in u32"
+        )
+    )]
+    const TAG_BITS_UL: c_ulong = (TAG_LEN * 8) as c_ulong;
     #[expect(
         clippy::cast_possible_truncation,
         reason = "CK_GCM_MESSAGE_PARAMS is a small fixed struct"
@@ -645,10 +661,14 @@ mod recprot {
 
         #[test]
         fn ck_gcm_message_params_size() {
+            use std::os::raw::c_ulong;
             eprintln!(
-                "size_of::<CK_GCM_MESSAGE_PARAMS>() = {}, GCM_PARAMS_LEN_C = {}",
+                "size_of::<CK_GCM_MESSAGE_PARAMS>()={} GCM_PARAMS_LEN_C={} \
+                 size_of::<c_ulong>()={} size_of::<*mut u8>()={}",
                 size_of::<freebl::CK_GCM_MESSAGE_PARAMS>(),
                 super::GCM_PARAMS_LEN_C,
+                size_of::<c_ulong>(),
+                size_of::<*mut u8>(),
             );
         }
 
@@ -688,6 +708,21 @@ mod recprot {
                 pTag: tag.as_mut_ptr(),
                 ulTagBits: super::TAG_BITS_UL,
             };
+            eprintln!(
+                "params: pIv={:?} ulIvLen={} ulIvFixedBits={} ivGenerator={} pTag={:?} ulTagBits={}",
+                params.pIv, params.ulIvLen, params.ulIvFixedBits,
+                params.ivGenerator, params.pTag, params.ulTagBits,
+            );
+            // Show the raw bytes NSS will receive so we can spot any layout mismatch.
+            let params_bytes: &[u8] = unsafe {
+                std::slice::from_raw_parts(
+                    (&raw const params).cast::<u8>(),
+                    size_of::<freebl::CK_GCM_MESSAGE_PARAMS>(),
+                )
+            };
+            eprintln!("params raw bytes: {params_bytes:02x?}");
+            // Clear any stale error so we know exactly what AES_AEAD sets (if anything).
+            unsafe { crate::err::PR_SetError(0, 0) };
             let rv = unsafe {
                 freebl::AES_AEAD(
                     ctx_enc,

src/freebl.rs

@@ -13,7 +13,7 @@
 // self-test gate. This is intentional for neqo (non-FIPS) and saves ~7.6%
 // CPU on the PK11_AEADOp hot path (sftk_SessionFromHandle + mutex overhead).
 
-use std::os::raw::{c_int, c_uchar, c_uint, c_void};
+use std::os::raw::{c_int, c_uchar, c_uint, c_ulong, c_void};
 
 // NSS_AES_GCM = 4 (from blapit.h)
 pub const NSS_AES_GCM: c_int = 4;
@@ -35,27 +35,23 @@ pub type ChaCha20Poly1305Context = ChaCha20Poly1305ContextStr;
 // For CKG_NO_GENERATE (ivGenerator = 0), pIv/ulIvLen supply the full nonce.
 // For encrypt, pTag is the output tag buffer; for decrypt, pTag is the input
 // tag to verify. ulTagBits = TAG_LEN * 8 = 128.
-//
-// CK_ULONG maps to `unsigned long` in C, which is 8 bytes on LP64 (Linux/macOS)
-// and also on the Windows NSS build (whose freebl/gcm libraries are compiled
-// with an LP64 toolchain).  Using u64 here ensures a uniform 48-byte layout
-// on all 64-bit targets and avoids a sizeof mismatch in the AES_AEAD paramLen
-// check on Windows.
 #[repr(C)]
 #[derive(Copy, Clone)]
 #[expect(non_snake_case, reason = "PKCS#11 naming conventions.")]
 pub struct CK_GCM_MESSAGE_PARAMS {
     pub pIv: *mut c_uchar,
-    pub ulIvLen: u64,
-    pub ulIvFixedBits: u64,
-    pub ivGenerator: u64, // CK_GENERATOR_FUNCTION; 0 = CKG_NO_GENERATE
+    pub ulIvLen: c_ulong,
+    pub ulIvFixedBits: c_ulong,
+    pub ivGenerator: c_ulong, // CK_GENERATOR_FUNCTION; 0 = CKG_NO_GENERATE
     pub pTag: *mut c_uchar,
-    pub ulTagBits: u64,
+    pub ulTagBits: c_ulong,
 }
 
-// All 6 fields are 8 bytes wide (two 8-byte pointers and four u64 values),
-// so the struct is 48 bytes on all 64-bit targets — no platform-specific padding.
-const _: () = assert!(size_of::<CK_GCM_MESSAGE_PARAMS>() == 6 * 8);
+// On LP64 (Linux/macOS) pointer == c_ulong == 8 bytes, so all 6 fields are
+// the same width.  On Windows LLP64, c_ulong is 4 bytes but pointers are 8,
+// so the layout differs and the check would be wrong — skip it there.
+#[cfg(not(target_os = "windows"))]
+const _: () = assert!(size_of::<CK_GCM_MESSAGE_PARAMS>() == 6 * size_of::<c_ulong>());
 
 unsafe extern "C" {
     /// Allocate and initialise an AES context.  Pass `NULL` for `iv` when