feat: add blapi feature to bypass PKCS#11 in RecordProtection

Calls freebl AES-GCM and ChaCha20-Poly1305 primitives directly instead
of going through `PK11_AEADOp` → softoken, eliminating the
`sftk_SessionFromHandle` mutex and hash-table overhead (~7.6% CPU on
the hot path).

`RecordProtection::new` takes a `Mode` so callers that know their
direction (send vs. receive) create only the context they need.

NOTE: bypasses softoken's FIPS self-test gate.  Intentional for neqo
(non-FIPS).

Author: larseggert

.github/workflows/check.yml

@@ -83,7 +83,7 @@ jobs:
         with:
           version: ${{ matrix.rust-toolchain }}
           components: ${{ matrix.rust-toolchain == 'stable' && 'llvm-tools' || '' }} ${{ matrix.rust-toolchain == 'nightly' && startsWith(matrix.os, 'ubuntu') && !endsWith(matrix.os, 'arm') && 'rust-src ' || '' }}
-          tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov' || '' }} ${{ matrix.rust-toolchain == 'nightly' && startsWith(matrix.os, 'ubuntu') && !endsWith(matrix.os, 'arm') && 'cargo-careful ' || '' }}
+          tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov' || '' }} ${{ matrix.rust-toolchain == 'nightly' && startsWith(matrix.os, 'ubuntu') && !endsWith(matrix.os, 'arm') && 'cargo-careful ' || '' }} cargo-hack
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - uses: mozilla/actions/nss@27cbe8fb5d338c2861b787e5de10410559065db1 # v1.1.3
@@ -96,6 +96,11 @@ jobs:
           # shellcheck disable=SC2086
           cargo check $BUILD_TYPE --locked --all-targets
 
+      - name: Check feature powerset
+        run: |
+          # shellcheck disable=SC2086
+          cargo hack check $BUILD_TYPE --locked --feature-powerset --no-dev-deps --exclude-features gecko --mutually-exclusive-features blapi,disable-encryption
+
       - name: Run tests and determine coverage
         env:
           RUST_LOG: trace
@@ -119,6 +124,11 @@ jobs:
             cargo $CAREFUL test $BUILD_TYPE --locked $TRIPLE
           fi
 
+      - name: Test with blapi feature
+        run: |
+          # shellcheck disable=SC2086
+          cargo test $BUILD_TYPE --locked --features blapi
+
       - name: CodeCov Windows workaround
         if: ${{ startsWith(matrix.os, 'windows') && matrix.type == 'debug' && matrix.rust-toolchain == 'stable' }}
         run: |

.github/workflows/clippy.yml

@@ -59,7 +59,7 @@ jobs:
       # respective default features only. Can reveal warnings otherwise
       # hidden given that a plain cargo clippy combines all features of the
       # workspace. See e.g. https://github.com/mozilla/neqo/pull/1695.
-      - run: cargo hack clippy --feature-powerset --no-dev-deps --exclude-features gecko -- -D warnings
+      - run: cargo hack clippy --feature-powerset --no-dev-deps --exclude-features gecko --mutually-exclusive-features blapi,disable-encryption -- -D warnings
       - run: cargo clippy --locked --workspace --all-targets -- -D warnings
       - run: cargo doc --locked --workspace --no-deps --document-private-items
         env:

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nss-rs"
-version = "0.10.0"
+version = "0.11.0"
 authors = ["Martin Thomson <mt@lowentropy.net>", "Andy Leiserson <aleiserson@mozilla.com>", "John M. Schanck <jschanck@mozilla.com>", "Benjamin Beurdouche <beurdouche@mozilla.com>", "Anna Weine <anna.weine@mozilla.com>"]
 categories = ["network-programming", "web-programming"]
 keywords = ["nss", "crypto", "mozilla", "firefox"]
@@ -124,6 +124,9 @@ verbose_file_reads = "warn"
 
 [features]
 bench = ["log/release_max_level_info"]
+# Bypass PKCS#11 session layer for RecordProtection AEAD operations by calling
+# freebl directly. Improves performance but skips softoken's FIPS self-test gate.
+blapi = []
 deny-warnings = []
 disable-encryption = []
 disable-random = []

build.rs

@@ -229,6 +229,9 @@ fn dynamic_link() {
     for lib in dynamic_libs {
         println!("cargo:rustc-link-lib=dylib={lib}");
     }
+    if env::var("CARGO_FEATURE_BLAPI").is_ok() {
+        println!("cargo:rustc-link-lib=static=freebl");
+    }
 }
 
 fn static_link() {
@@ -396,19 +399,57 @@ fn pkg_config() -> Result<Vec<String>, Box<dyn Error>> {
     let cfg_str = String::from_utf8(cfg)?;
 
     let mut flags: Vec<String> = Vec::new();
+    let mut lib_dirs: Vec<PathBuf> = Vec::new();
 
     for f in cfg_str.split_whitespace() {
         if f.starts_with("-I") {
             flags.push(String::from(f));
         } else if let Some(path) = f.strip_prefix("-L") {
             println!("cargo:rustc-link-search=native={path}");
+            lib_dirs.push(PathBuf::from(path));
         } else if let Some(lib) = f.strip_prefix("-l") {
             println!("cargo:rustc-link-lib=dylib={lib}");
         } else {
             println!("cargo:warning=Unknown flag from pkg-config: {f}");
         }
     }
 
+    if env::var("CARGO_FEATURE_BLAPI").is_ok() {
+        // Probe for freebl in preference order:
+        //
+        // 1. libfreebl.a  — static archive from a source build (e.g. Homebrew on macOS, or a
+        //    standalone NSS_DIR build).  All internal symbols guaranteed present.
+        //
+        // 2. libfreeblpriv3  — some package managers (e.g. FreeBSD ports) separate the
+        //    private/internal freebl API into this library (it is what libsoftokn3 links against),
+        //    while libfreebl3 only exports the public API.
+        //
+        // 3. libfreebl3  — on other systems (NetBSD, Debian, …) libfreebl3 exports the internal API
+        //    directly and libfreeblpriv3 is absent.
+        let has = |name: &str| lib_dirs.iter().any(|dir| dir.join(name).exists());
+        let dylib_ext = if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") {
+            "dylib"
+        } else {
+            "so"
+        };
+        let has_dylib = |stem: &str| has(&format!("{stem}.{dylib_ext}"));
+        let link = [
+            (has("libfreebl.a"), "static=freebl"),
+            (has_dylib("libfreeblpriv3"), "dylib=freeblpriv3"),
+            (has_dylib("libfreebl3"), "dylib=freebl3"),
+        ]
+        .into_iter()
+        .find_map(|(found, link)| found.then_some(link))
+        .unwrap_or_else(|| {
+            panic!(
+                "blapi feature requires freebl: libfreebl.a, libfreeblpriv3, \
+             and libfreebl3 were all absent from the pkg-config library \
+             paths. Set NSS_DIR to a standalone NSS source build."
+            )
+        });
+        println!("cargo:rustc-link-lib={link}");
+    }
+
     Ok(flags)
 }
 

src/aead/mod.rs

@@ -21,8 +21,18 @@ use crate::{
     secstatus_to_res,
 };
 
+#[cfg(all(feature = "blapi", feature = "disable-encryption"))]
+compile_error!("`blapi` and `disable-encryption` are mutually exclusive features");
+
 #[cfg_attr(feature = "disable-encryption", path = "recprot_null.rs")]
-#[cfg_attr(not(feature = "disable-encryption"), path = "recprot.rs")]
+#[cfg_attr(
+    all(not(feature = "disable-encryption"), feature = "blapi"),
+    path = "recprot_blapi.rs"
+)]
+#[cfg_attr(
+    all(not(feature = "disable-encryption"), not(feature = "blapi")),
+    path = "recprot.rs"
+)]
 mod recprot;
 
 /// All the nonces are the same length.  Exploit that.

src/aead/recprot.rs

@@ -10,7 +10,7 @@ use std::{
     ptr::{null, null_mut},
 };
 
-use super::{COUNTER_LEN, NONCE_LEN, TAG_LEN, c_int_len, xor_nonce};
+use super::{COUNTER_LEN, Mode, NONCE_LEN, TAG_LEN, c_int_len, xor_nonce};
 use crate::{
     Cipher, Error, Res, SECItemBorrowed, SymKey, Version,
     constants::{TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256},
@@ -133,7 +133,13 @@ impl RecordProtection {
     /// # Errors
     ///
     /// Returns `Error` when the underlying crypto operations fail.
-    pub fn new(version: Version, cipher: Cipher, secret: &SymKey, prefix: &str) -> Res<Self> {
+    pub fn new(
+        version: Version,
+        cipher: Cipher,
+        secret: &SymKey,
+        prefix: &str,
+        _mode: Mode,
+    ) -> Res<Self> {
         let (mech, key_len) = cipher_mech_and_key_len(cipher)?;
         let key = expand_label(
             version,
@@ -174,8 +180,8 @@ impl RecordProtection {
 
     /// Get the expansion size (authentication tag length) for this AEAD.
     #[must_use]
-    #[expect(clippy::missing_const_for_fn, clippy::unused_self)]
-    pub fn expansion(&self) -> usize {
+    #[expect(clippy::unused_self)]
+    pub const fn expansion(&self) -> usize {
         TAG_LEN
     }