perf(spider): bloom filter bitmask addressing + inline early-exit, interner zero-clone extend, bump v2.47.53

Bloom: power-of-2 sizing with bitmask (replaces expensive modulo 7x per op),
inline early-exit contains (1-2 bit tests on absent path vs all 7),
MurmurHash3 finalizer for h2 decorrelation. Interner: move instead of
clone in extend_links, read-only get() instead of get_or_intern() for
visited checks, removed double-reference indirection in bloom calls.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: j-mendez

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.47.52"
+version = "2.47.53"
 authors = ["j-mendez <jeff@spider.cloud>"]
 description = "A web crawler and scraper, building blocks for data curation workloads."
 repository = "https://github.com/spider-rs/spider"
@@ -121,11 +121,11 @@ features = ["serde", "headers", "dynamic-versions"]
 
 [dependencies.spider_agent_types]
 path = "../spider_agent_types"
-version = "2.47.52"
+version = "2.47.53"
 
 [dependencies.spider_agent]
 path = "../spider_agent"
-version = "2.47.52"
+version = "2.47.53"
 optional = true
 default-features = false
 

spider/Cargo.toml

@@ -24,13 +24,16 @@ const TARGET_FP: f64 = 0.01;
 /// k = -ln(p) / ln(2) ≈ 6.64 → 7
 const NUM_HASHES: u32 = 7;
 
-/// Compute optimal bit count for `n` elements at `fp` false-positive rate.
-/// m = -n * ln(p) / (ln2)^2
+/// Compute optimal bit count for `n` elements at `fp` false-positive rate,
+/// rounded up to the next **power of two** so modulo can be replaced with
+/// a bitmask (`& mask`).
+///
+/// m = -n * ln(p) / (ln2)^2, then → next_power_of_two
 fn optimal_bits(n: usize, fp: f64) -> usize {
     let m = -(n as f64) * fp.ln() / (core::f64::consts::LN_2.powi(2));
-    // Round up to next multiple of 8 so we address whole bytes.
-    let m = m.ceil() as usize;
-    (m + 7) & !7
+    let m = (m.ceil() as usize).max(64);
+    // Round to next power of two for bitmask addressing.
+    m.next_power_of_two()
 }
 
 /// Tracks how the backing memory was allocated so `Drop` can free it correctly.
@@ -53,8 +56,8 @@ pub struct MmapBloom {
     ptr: *mut u8,
     /// Usable length in bytes (= num_bits / 8).
     len_bytes: usize,
-    /// Total number of usable bits (= len_bytes * 8).
-    num_bits: u64,
+    /// Bitmask for fast modulo: `num_bits - 1` (num_bits is always power of 2).
+    mask: u64,
     /// Number of elements inserted (approximate — counts every insert call).
     count: usize,
     /// How the memory was allocated, for correct deallocation.
@@ -83,7 +86,7 @@ impl MmapBloom {
         Self {
             ptr,
             len_bytes,
-            num_bits: bits as u64,
+            mask: (bits as u64) - 1,
             count: 0,
             alloc_kind,
         }
@@ -167,69 +170,77 @@ impl MmapBloom {
         (ptr, AllocKind::Heap)
     }
 
-    /// Compute the k bit positions for a given item using double hashing.
-    /// h_i = h1 + i * h2  (mod num_bits)
-    #[inline(always)]
-    fn bit_positions<T: Hash>(&self, item: &T) -> [u64; NUM_HASHES as usize] {
-        let mut h1_state = ahash::AHasher::default();
-        item.hash(&mut h1_state);
-        let h1 = h1_state.finish();
-
-        // Second hash: fold and mix.
-        let h2 = h1
-            .wrapping_mul(0x517cc1b727220a95)
-            .wrapping_add(0x6c62272e07bb0142);
-
-        let mut positions = [0u64; NUM_HASHES as usize];
-        for i in 0..NUM_HASHES as u64 {
-            positions[i as usize] = (h1.wrapping_add(i.wrapping_mul(h2))) % self.num_bits;
-        }
-        positions
-    }
-
-    /// Set bit at position `pos`.
-    #[inline(always)]
-    fn set_bit(&mut self, pos: u64) {
-        let byte_idx = (pos / 8) as usize;
-        let bit_idx = (pos % 8) as u8;
-        debug_assert!(byte_idx < self.len_bytes);
-        // SAFETY: `pos < num_bits` (enforced by modulo in `bit_positions`),
-        // and `num_bits == len_bytes * 8`, so `byte_idx < len_bytes` always holds.
-        unsafe {
-            let byte = &mut *self.ptr.add(byte_idx);
-            *byte |= 1 << bit_idx;
-        }
-    }
-
-    /// Test bit at position `pos`.
+    /// Compute double-hash seeds for an item.
+    ///
+    /// h2 is derived via a MurmurHash3-style finalizer to decorrelate it from
+    /// h1 — critical for low false-positive rates with power-of-2 masking.
+    /// The `| 1` forces h2 odd (coprime with any power of 2) so all bit
+    /// positions are reachable.
     #[inline(always)]
-    fn test_bit(&self, pos: u64) -> bool {
-        let byte_idx = (pos / 8) as usize;
-        let bit_idx = (pos % 8) as u8;
-        debug_assert!(byte_idx < self.len_bytes);
-        // SAFETY: same invariant as `set_bit`.
-        unsafe {
-            let byte = *self.ptr.add(byte_idx);
-            byte & (1 << bit_idx) != 0
-        }
+    fn hash_seeds<T: Hash + ?Sized>(item: &T) -> (u64, u64) {
+        let mut state = ahash::AHasher::default();
+        item.hash(&mut state);
+        let h1 = state.finish();
+        // MurmurHash3 64-bit finalizer — avalanches all bits.
+        let mut x = h1;
+        x ^= x >> 33;
+        x = x.wrapping_mul(0xff51afd7ed558ccd);
+        x ^= x >> 33;
+        x = x.wrapping_mul(0xc4ceb9fe1a85ec53);
+        x ^= x >> 33;
+        (h1, x | 1)
     }
 
     /// Insert an item into the bloom filter.
+    ///
+    /// Computes each bit position inline — no intermediate array.
+    /// Uses enhanced double hashing: h_i = h1 + i*h2 + i*(i-1)/2 to
+    /// eliminate correlation artefacts with power-of-2 sizing.
     #[inline]
-    pub fn insert<T: Hash>(&mut self, item: &T) {
-        let positions = self.bit_positions(item);
-        for &pos in &positions {
-            self.set_bit(pos);
+    pub fn insert<T: Hash + ?Sized>(&mut self, item: &T) {
+        let (h1, h2) = Self::hash_seeds(item);
+        let mask = self.mask;
+        let mut composite = h1;
+        for i in 0..NUM_HASHES as u64 {
+            let pos = composite & mask;
+            let byte_idx = (pos >> 3) as usize;
+            let bit_idx = (pos & 7) as u8;
+            // SAFETY: pos < num_bits (mask guarantees), num_bits == len_bytes * 8.
+            unsafe {
+                let byte = &mut *self.ptr.add(byte_idx);
+                *byte |= 1 << bit_idx;
+            }
+            // Enhanced double hashing: next = h1 + (i+1)*h2 + (i+1)*i/2
+            //   = composite + h2 + i
+            composite = composite.wrapping_add(h2).wrapping_add(i);
         }
         self.count += 1;
     }
 
     /// Check if an item is probably in the set.
-    /// Returns `false` only when the item is *definitely* absent.
+    ///
+    /// Returns `false` as soon as *any* bit is unset — on the common "absent"
+    /// path this exits after testing only 1-2 bits instead of all 7.
     #[inline]
-    pub fn contains<T: Hash>(&self, item: &T) -> bool {
-        let positions = self.bit_positions(item);
-        positions.iter().all(|&pos| self.test_bit(pos))
+    pub fn contains<T: Hash + ?Sized>(&self, item: &T) -> bool {
+        let (h1, h2) = Self::hash_seeds(item);
+        let mask = self.mask;
+        let mut composite = h1;
+        for i in 0..NUM_HASHES as u64 {
+            let pos = composite & mask;
+            let byte_idx = (pos >> 3) as usize;
+            let bit_idx = (pos & 7) as u8;
+            // SAFETY: same invariant as `insert`.
+            let set = unsafe {
+                let byte = *self.ptr.add(byte_idx);
+                byte & (1 << bit_idx) != 0
+            };
+            if !set {
+                return false;
+            }
+            composite = composite.wrapping_add(h2).wrapping_add(i);
+        }
+        true
     }
 
     /// Approximate number of insertions performed.
@@ -295,7 +306,7 @@ impl Drop for MmapBloom {
 impl std::fmt::Debug for MmapBloom {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("MmapBloom")
-            .field("num_bits", &self.num_bits)
+            .field("num_bits", &(self.mask + 1))
             .field("count", &self.count)
             .field("size_bytes", &self.len_bytes)
             .field("alloc_kind", &self.alloc_kind)
@@ -314,7 +325,7 @@ impl Clone for MmapBloom {
         Self {
             ptr,
             len_bytes: self.len_bytes,
-            num_bits: self.num_bits,
+            mask: self.mask,
             count: self.count,
             alloc_kind,
         }
@@ -427,9 +438,9 @@ mod tests {
     #[test]
     fn test_size_reasonable() {
         let bloom = MmapBloom::new(1_000_000);
-        // For 1M items at 1% FP: ~1.2 MB
+        // For 1M items at 1% FP: ~1.2 MB optimal, rounded to next power of 2 → 2 MB.
         assert!(bloom.size_bytes() > 1_000_000);
-        assert!(bloom.size_bytes() < 2_000_000);
+        assert!(bloom.size_bytes() <= 2_097_152); // 2 MiB (16 Mbit)
     }
 
     #[test]
@@ -442,9 +453,9 @@ mod tests {
     #[test]
     fn test_optimal_bits() {
         let bits = optimal_bits(1_000_000, 0.01);
-        // Should be ~9.58M bits ≈ 1.2 MB
-        assert!(bits > 9_000_000);
-        assert!(bits < 10_000_000);
+        // ~9.58M optimal → next power of 2 = 16_777_216 (2^24)
+        assert!(bits.is_power_of_two());
+        assert_eq!(bits, 16_777_216);
     }
 
     #[test]