feat: add blapi feature to bypass PKCS#11 in RecordProtection

Calls freebl AES-GCM and ChaCha20-Poly1305 primitives directly instead
of going through `PK11_AEADOp` → softoken, eliminating the
`sftk_SessionFromHandle` mutex and hash-table overhead (~7.6% CPU on
the hot path).

`RecordProtection::new` takes a `Mode` so callers that know their
direction (send vs. receive) create only the context they need.

NOTE: bypasses softoken's FIPS self-test gate.  Intentional for neqo
(non-FIPS).

Author: larseggert

.github/workflows/check.yml

@@ -83,7 +83,7 @@ jobs:
         with:
           version: ${{ matrix.rust-toolchain }}
           components: ${{ matrix.rust-toolchain == 'stable' && 'llvm-tools' || '' }} ${{ matrix.rust-toolchain == 'nightly' && startsWith(matrix.os, 'ubuntu') && !endsWith(matrix.os, 'arm') && 'rust-src ' || '' }}
-          tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov' || '' }} ${{ matrix.rust-toolchain == 'nightly' && startsWith(matrix.os, 'ubuntu') && !endsWith(matrix.os, 'arm') && 'cargo-careful ' || '' }}
+          tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov' || '' }} ${{ matrix.rust-toolchain == 'nightly' && startsWith(matrix.os, 'ubuntu') && !endsWith(matrix.os, 'arm') && 'cargo-careful ' || '' }} cargo-hack
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - uses: mozilla/actions/nss@27cbe8fb5d338c2861b787e5de10410559065db1 # v1.1.3
@@ -96,6 +96,11 @@ jobs:
           # shellcheck disable=SC2086
           cargo check $BUILD_TYPE --locked --all-targets
 
+      - name: Check feature powerset
+        run: |
+          # shellcheck disable=SC2086
+          cargo hack check $BUILD_TYPE --locked --feature-powerset --no-dev-deps --exclude-features gecko --mutually-exclusive-features blapi,disable-encryption
+
       - name: Run tests and determine coverage
         env:
           RUST_LOG: trace
@@ -119,6 +124,11 @@ jobs:
             cargo $CAREFUL test $BUILD_TYPE --locked $TRIPLE
           fi
 
+      - name: Test with blapi feature
+        run: |
+          # shellcheck disable=SC2086
+          cargo test $BUILD_TYPE --locked --features blapi
+
       - name: CodeCov Windows workaround
         if: ${{ startsWith(matrix.os, 'windows') && matrix.type == 'debug' && matrix.rust-toolchain == 'stable' }}
         run: |

.github/workflows/clippy.yml

@@ -59,7 +59,7 @@ jobs:
       # respective default features only. Can reveal warnings otherwise
       # hidden given that a plain cargo clippy combines all features of the
       # workspace. See e.g. https://github.com/mozilla/neqo/pull/1695.
-      - run: cargo hack clippy --feature-powerset --no-dev-deps --exclude-features gecko -- -D warnings
+      - run: cargo hack clippy --feature-powerset --no-dev-deps --exclude-features gecko --mutually-exclusive-features blapi,disable-encryption -- -D warnings
       - run: cargo clippy --locked --workspace --all-targets -- -D warnings
       - run: cargo doc --locked --workspace --no-deps --document-private-items
         env:

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "nss-rs"
-version = "0.11.0"
+version = "0.12.0"
 authors = ["Martin Thomson <mt@lowentropy.net>", "Andy Leiserson <aleiserson@mozilla.com>", "John M. Schanck <jschanck@mozilla.com>", "Benjamin Beurdouche <beurdouche@mozilla.com>", "Anna Weine <anna.weine@mozilla.com>"]
 categories = ["network-programming", "web-programming"]
 keywords = ["nss", "crypto", "mozilla", "firefox"]
@@ -124,6 +124,9 @@ verbose_file_reads = "warn"
 
 [features]
 bench = ["log/release_max_level_info"]
+# Bypass PKCS#11 session layer for RecordProtection AEAD operations by calling
+# freebl directly. Improves performance but skips softoken's FIPS self-test gate.
+blapi = []
 disable-encryption = []
 disable-random = []
 gecko = ["mozbuild"]

Cargo.toml

@@ -229,6 +229,9 @@ fn dynamic_link() {
     for lib in dynamic_libs {
         println!("cargo:rustc-link-lib=dylib={lib}");
     }
+    if env::var("CARGO_FEATURE_BLAPI").is_ok() {
+        println!("cargo:rustc-link-lib=static=freebl");
+    }
 }
 
 fn static_link() {
@@ -396,19 +399,57 @@ fn pkg_config() -> Result<Vec<String>, Box<dyn Error>> {
     let cfg_str = String::from_utf8(cfg)?;
 
     let mut flags: Vec<String> = Vec::new();
+    let mut lib_dirs: Vec<PathBuf> = Vec::new();
 
     for f in cfg_str.split_whitespace() {
         if f.starts_with("-I") {
             flags.push(String::from(f));
         } else if let Some(path) = f.strip_prefix("-L") {
             println!("cargo:rustc-link-search=native={path}");
+            lib_dirs.push(PathBuf::from(path));
         } else if let Some(lib) = f.strip_prefix("-l") {
             println!("cargo:rustc-link-lib=dylib={lib}");
         } else {
             println!("cargo:warning=Unknown flag from pkg-config: {f}");
         }
     }
 
+    if env::var("CARGO_FEATURE_BLAPI").is_ok() {
+        // Probe for freebl in preference order:
+        //
+        // 1. libfreebl.a  — static archive from a source build (e.g. Homebrew on macOS, or a
+        //    standalone NSS_DIR build).  All internal symbols guaranteed present.
+        //
+        // 2. libfreeblpriv3  — some package managers (e.g. FreeBSD ports) separate the
+        //    private/internal freebl API into this library (it is what libsoftokn3 links against),
+        //    while libfreebl3 only exports the public API.
+        //
+        // 3. libfreebl3  — on other systems (NetBSD, Debian, …) libfreebl3 exports the internal API
+        //    directly and libfreeblpriv3 is absent.
+        let has = |name: &str| lib_dirs.iter().any(|dir| dir.join(name).exists());
+        let dylib_ext = if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") {
+            "dylib"
+        } else {
+            "so"
+        };
+        let has_dylib = |stem: &str| has(&format!("{stem}.{dylib_ext}"));
+        let link = [
+            (has("libfreebl.a"), "static=freebl"),
+            (has_dylib("libfreeblpriv3"), "dylib=freeblpriv3"),
+            (has_dylib("libfreebl3"), "dylib=freebl3"),
+        ]
+        .into_iter()
+        .find_map(|(found, link)| found.then_some(link))
+        .unwrap_or_else(|| {
+            panic!(
+                "blapi feature requires freebl: libfreebl.a, libfreeblpriv3, \
+             and libfreebl3 were all absent from the pkg-config library \
+             paths. Set NSS_DIR to a standalone NSS source build."
+            )
+        });
+        println!("cargo:rustc-link-lib={link}");
+    }
+
     Ok(flags)
 }
 

build.rs

@@ -4,15 +4,21 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#[cfg(not(feature = "disable-encryption"))]
+use std::os::raw::{c_char, c_uint};
+#[cfg(not(feature = "disable-encryption"))]
+use std::ptr::null;
 use std::{os::raw::c_int, ptr::null_mut};
 
 #[cfg(feature = "disable-encryption")]
 pub use recprot::AEAD_NULL_TAG;
 pub use recprot::RecordProtection;
 
+#[cfg(not(feature = "disable-encryption"))]
+use crate::{Cipher, Version, hp::SSL_HkdfExpandLabelWithMech, p11::PK11SymKey};
 use crate::{
     SECItemBorrowed, SymKey,
-    err::{Error, Res},
+    err::{Error, Res, sec::SEC_ERROR_BAD_DATA},
     p11::{
         self, CK_ATTRIBUTE_TYPE, CK_GENERATOR_FUNCTION, CK_MECHANISM_TYPE, CKA_DECRYPT,
         CKA_ENCRYPT, CKA_NSS_MESSAGE, CKG_GENERATE_COUNTER_XOR, CKG_NO_GENERATE, CKM_AES_GCM,
@@ -21,10 +27,98 @@ use crate::{
     secstatus_to_res,
 };
 
+#[cfg(all(feature = "blapi", feature = "disable-encryption"))]
+compile_error!("`blapi` and `disable-encryption` are mutually exclusive features");
+
+/// Shared API contract for all `RecordProtection` backends.
+///
+/// Implemented by each cfg-selected `recprot*.rs` backend so that a
+/// signature change in one backend is caught at compile time across all.
+/// Import this trait to call AEAD methods on `RecordProtection`.
+pub trait RecordProtectionOps {
+    /// Get the expansion size (authentication tag length) for this AEAD.
+    #[must_use]
+    fn expansion(&self) -> usize;
+
+    /// Encrypt plaintext with associated data.
+    ///
+    /// # Errors
+    ///
+    /// Returns `Error` when encryption fails.
+    fn encrypt<'a>(
+        &self,
+        count: u64,
+        aad: &[u8],
+        input: &[u8],
+        output: &'a mut [u8],
+    ) -> Res<&'a [u8]>;
+
+    /// Encrypt plaintext in place with associated data.
+    ///
+    /// # Errors
+    ///
+    /// Returns `Error` when encryption fails.
+    fn encrypt_in_place(&self, count: u64, aad: &[u8], data: &mut [u8]) -> Res<usize>;
+
+    /// Decrypt ciphertext with associated data.
+    ///
+    /// # Errors
+    ///
+    /// Returns `Error` when decryption or authentication fails.
+    fn decrypt<'a>(
+        &self,
+        count: u64,
+        aad: &[u8],
+        input: &[u8],
+        output: &'a mut [u8],
+    ) -> Res<&'a [u8]>;
+
+    /// Decrypt ciphertext in place with associated data.
+    ///
+    /// # Errors
+    ///
+    /// Returns `Error` when decryption or authentication fails.
+    fn decrypt_in_place(&self, count: u64, aad: &[u8], data: &mut [u8]) -> Res<usize>;
+}
+
 #[cfg_attr(feature = "disable-encryption", path = "recprot_null.rs")]
-#[cfg_attr(not(feature = "disable-encryption"), path = "recprot.rs")]
+#[cfg_attr(
+    all(not(feature = "disable-encryption"), feature = "blapi"),
+    path = "recprot_blapi.rs"
+)]
+#[cfg_attr(
+    all(not(feature = "disable-encryption"), not(feature = "blapi")),
+    path = "recprot.rs"
+)]
 mod recprot;
 
+#[cfg(not(feature = "disable-encryption"))]
+fn expand_label(
+    version: Version,
+    cipher: Cipher,
+    secret: &SymKey,
+    label: &str,
+    mech: CK_MECHANISM_TYPE,
+    key_len: c_uint,
+) -> Res<SymKey> {
+    let mut ptr: *mut PK11SymKey = null_mut();
+    unsafe {
+        SSL_HkdfExpandLabelWithMech(
+            version,
+            cipher,
+            **secret,
+            null(),
+            0,
+            label.as_ptr().cast::<c_char>(),
+            c_uint::try_from(label.len())?,
+            mech,
+            key_len,
+            &raw mut ptr,
+        )
+    }?;
+    SymKey::from_ptr(ptr)
+}
+
 /// All the nonces are the same length.  Exploit that.
 pub const NONCE_LEN: usize = 12;
 
@@ -46,6 +140,18 @@ fn xor_nonce(base: &[u8; NONCE_LEN], count: SequenceNumber) -> [u8; NONCE_LEN] {
 /// All of the AEAD functions here have a tag of this length, so use a fixed offset.
 const TAG_LEN: usize = 16;
 
+/// Split `data` into `(ct_len, tag)`, returning `SEC_ERROR_BAD_DATA` if it is
+/// too short to contain a tag.
+fn split_tag(data: &[u8]) -> Res<(usize, [u8; TAG_LEN])> {
+    let ct_len = data
+        .len()
+        .checked_sub(TAG_LEN)
+        .ok_or_else(|| Error::from(SEC_ERROR_BAD_DATA))?;
+    let mut tag = [0u8; TAG_LEN];
+    tag.copy_from_slice(&data[ct_len..]);
+    Ok((ct_len, tag))
+}
+
 pub type SequenceNumber = u64;
 
 /// All the lengths used by `PK11_AEADOp` are signed.  This converts to that.