fix: use maturin develop for cross-platform compatibility

PowerShell on Windows doesn't expand glob patterns like *.whl.
Using 'maturin develop --release' which builds and installs in one step.

Author: mrecachinas

.github/workflows/pythonpackage.yml

@@ -35,9 +35,7 @@ jobs:
           pip install maturin pytest pytest-benchmark
 
       - name: Build and install with maturin
-        run: |
-          maturin build --release
-          pip install target/wheels/*.whl
+        run: maturin develop --release
 
       - name: Run tests
         run: pytest -vls test/

src/lib.rs

@@ -2,6 +2,13 @@
 //!
 //! This module provides blazingly fast bitwise Hamming distance calculation
 //! using SIMD intrinsics where available.
+//!
+//! # Optimizations
+//! - Branchless hex parsing with 256-byte lookup table
+//! - SIMD vectorized processing (AVX2/SSE4.1/NEON)
+//! - Batched horizontal summation to minimize lane reductions
+//! - Unsafe bounds elimination in hot loops
+//! - Algorithm selection based on input size thresholds
 
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::*;
@@ -10,6 +17,24 @@ use std::sync::atomic::{AtomicU8, Ordering};
 /// Lookup table for popcount of 4-bit values (0-15)
 const LOOKUP: [u8; 16] = [0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4];
 
+/// Branchless hex character to nibble lookup table (256 entries)
+/// Invalid characters map to 0xFF for easy detection
+const HEX_LOOKUP: [u8; 256] = {
+    let mut table = [0xFFu8; 256];
+    let mut i = 0u8;
+    loop {
+        table[i as usize] = match i {
+            b'0'..=b'9' => i - b'0',
+            b'A'..=b'F' => i - b'A' + 10,
+            b'a'..=b'f' => i - b'a' + 10,
+            _ => 0xFF,
+        };
+        if i == 255 { break; }
+        i += 1;
+    }
+    table
+};
+
 /// Algorithm selection constants
 const ALGO_CLASSIC: u8 = 0;
 const ALGO_NATIVE: u8 = 1;
@@ -20,11 +45,15 @@ const ALGO_AVX2: u8 = 3;
 #[cfg(target_arch = "aarch64")]
 const ALGO_NEON: u8 = 4;
 
+/// Thresholds for algorithm selection (tuned for typical CPU cache behavior)
+const SCALAR_THRESHOLD: usize = 16; // Below this, scalar may beat SIMD
+const SSE_THRESHOLD: usize = 64;    // Use SSE for medium strings
+
 /// Current algorithm selection (global state)
 static CURRENT_ALGO: AtomicU8 = AtomicU8::new(ALGO_NATIVE);
 
-/// Classic popcount implementation using bit manipulation
-#[inline]
+/// Classic popcount implementation using bit manipulation (Wilkes-Wheeler-Gill)
+#[inline(always)]
 fn popcnt64_classic(mut x: u64) -> u64 {
     const M1: u64 = 0x5555555555555555;
     const M2: u64 = 0x3333333333333333;
@@ -37,70 +66,152 @@ fn popcnt64_classic(mut x: u64) -> u64 {
 }
 
 /// Native popcount implementation using CPU instruction
-#[inline]
+#[inline(always)]
 fn popcnt64_native(x: u64) -> u64 {
     x.count_ones() as u64
 }
 
+/// Branchless hex character to nibble conversion using lookup table
+/// Returns 0xFF for invalid characters
+#[inline(always)]
+fn hex_char_to_nibble(c: u8) -> u8 {
+    // SAFETY: c is u8, so always in bounds of 256-element table
+    unsafe { *HEX_LOOKUP.get_unchecked(c as usize) }
+}
+
 /// Convert a hex character to its numeric value (0-15)
 /// Returns None if the character is not a valid hex digit
-#[inline]
+#[inline(always)]
 fn hex_char_to_val(c: u8) -> Option<u8> {
-    match c {
-        b'0'..=b'9' => Some(c - b'0'),
-        b'A'..=b'F' => Some(c - b'A' + 10),
-        b'a'..=b'f' => Some(c - b'a' + 10),
-        _ => None,
-    }
+    let val = hex_char_to_nibble(c);
+    if val == 0xFF { None } else { Some(val) }
 }
 
 /// Calculate hamming distance between two hex strings using classic algorithm
+/// Optimized with branchless lookup and bounds check elimination
+#[inline(always)]
 fn hamming_distance_string_classic(a: &[u8], b: &[u8]) -> Result<u64, &'static str> {
+    let len = a.len();
     let mut result: u64 = 0;
-    for i in 0..a.len() {
-        let val1 = hex_char_to_val(a[i]).ok_or("hex string contains invalid char")?;
-        let val2 = hex_char_to_val(b[i]).ok_or("hex string contains invalid char")?;
-        result += LOOKUP[(val1 ^ val2) as usize] as u64;
+    let mut i = 0;
+    
+    // Process 4 hex chars at a time to reduce loop overhead
+    while i + 4 <= len {
+        // SAFETY: i + 3 < len verified by loop condition
+        unsafe {
+            let val1_0 = hex_char_to_nibble(*a.get_unchecked(i));
+            let val2_0 = hex_char_to_nibble(*b.get_unchecked(i));
+            let val1_1 = hex_char_to_nibble(*a.get_unchecked(i + 1));
+            let val2_1 = hex_char_to_nibble(*b.get_unchecked(i + 1));
+            let val1_2 = hex_char_to_nibble(*a.get_unchecked(i + 2));
+            let val2_2 = hex_char_to_nibble(*b.get_unchecked(i + 2));
+            let val1_3 = hex_char_to_nibble(*a.get_unchecked(i + 3));
+            let val2_3 = hex_char_to_nibble(*b.get_unchecked(i + 3));
+            
+            // Check all 8 values for validity (0xFF indicates invalid)
+            // Use bitwise OR to combine checks - any 0xFF will result in high bit set
+            let invalid = (val1_0 | val2_0 | val1_1 | val2_1 | val1_2 | val2_2 | val1_3 | val2_3) & 0xF0;
+            if invalid != 0 {
+                return Err("hex string contains invalid char");
+            }
+            
+            result += *LOOKUP.get_unchecked((val1_0 ^ val2_0) as usize) as u64
+                   + *LOOKUP.get_unchecked((val1_1 ^ val2_1) as usize) as u64
+                   + *LOOKUP.get_unchecked((val1_2 ^ val2_2) as usize) as u64
+                   + *LOOKUP.get_unchecked((val1_3 ^ val2_3) as usize) as u64;
+        }
+        i += 4;
+    }
+    
+    // Handle remaining characters
+    while i < len {
+        // SAFETY: i < len verified by loop condition
+        unsafe {
+            let val1 = hex_char_to_nibble(*a.get_unchecked(i));
+            let val2 = hex_char_to_nibble(*b.get_unchecked(i));
+            if (val1 | val2) & 0xF0 != 0 {
+                return Err("hex string contains invalid char");
+            }
+            result += *LOOKUP.get_unchecked((val1 ^ val2) as usize) as u64;
+        }
+        i += 1;
     }
+    
     Ok(result)
 }
 
 /// Calculate hamming distance between two byte arrays using classic algorithm
+/// Optimized with loop unrolling and bounds check elimination
+#[inline(always)]
 fn hamming_distance_bytes_classic(a: &[u8], b: &[u8], max_dist: i64) -> u64 {
-    let mut difference: u64 = 0;
     let length = a.len();
 
     if max_dist < 0 {
-        // Full distance calculation
+        // Full distance calculation - heavily optimized
+        let mut difference: u64 = 0;
         let mut i = 0;
-        // Process 8 bytes at a time
+        
+        // Process 32 bytes at a time (4 x 8-byte chunks)
+        while i + 32 <= length {
+            // SAFETY: i + 31 < length verified by loop condition
+            unsafe {
+                let a0 = u64::from_ne_bytes(*(a.as_ptr().add(i) as *const [u8; 8]));
+                let b0 = u64::from_ne_bytes(*(b.as_ptr().add(i) as *const [u8; 8]));
+                let a1 = u64::from_ne_bytes(*(a.as_ptr().add(i + 8) as *const [u8; 8]));
+                let b1 = u64::from_ne_bytes(*(b.as_ptr().add(i + 8) as *const [u8; 8]));
+                let a2 = u64::from_ne_bytes(*(a.as_ptr().add(i + 16) as *const [u8; 8]));
+                let b2 = u64::from_ne_bytes(*(b.as_ptr().add(i + 16) as *const [u8; 8]));
+                let a3 = u64::from_ne_bytes(*(a.as_ptr().add(i + 24) as *const [u8; 8]));
+                let b3 = u64::from_ne_bytes(*(b.as_ptr().add(i + 24) as *const [u8; 8]));
+                
+                difference += popcnt64_classic(a0 ^ b0)
+                           + popcnt64_classic(a1 ^ b1)
+                           + popcnt64_classic(a2 ^ b2)
+                           + popcnt64_classic(a3 ^ b3);
+            }
+            i += 32;
+        }
+        
+        // Process remaining 8-byte chunks
         while i + 8 <= length {
-            let a_chunk = u64::from_ne_bytes(a[i..i + 8].try_into().unwrap());
-            let b_chunk = u64::from_ne_bytes(b[i..i + 8].try_into().unwrap());
-            difference += popcnt64_classic(a_chunk ^ b_chunk);
+            unsafe {
+                let a_chunk = u64::from_ne_bytes(*(a.as_ptr().add(i) as *const [u8; 8]));
+                let b_chunk = u64::from_ne_bytes(*(b.as_ptr().add(i) as *const [u8; 8]));
+                difference += popcnt64_classic(a_chunk ^ b_chunk);
+            }
             i += 8;
         }
+        
         // Process remaining bytes
         while i < length {
-            difference += popcnt64_classic((a[i] ^ b[i]) as u64);
+            unsafe {
+                difference += popcnt64_classic((*a.get_unchecked(i) ^ *b.get_unchecked(i)) as u64);
+            }
             i += 1;
         }
         difference
     } else {
         // Early termination if exceeds max_dist
+        let max_dist_u64 = max_dist as u64;
+        let mut difference: u64 = 0;
         let mut i = 0;
+        
         while i + 8 <= length {
-            let a_chunk = u64::from_ne_bytes(a[i..i + 8].try_into().unwrap());
-            let b_chunk = u64::from_ne_bytes(b[i..i + 8].try_into().unwrap());
-            difference += popcnt64_classic(a_chunk ^ b_chunk);
-            if difference > max_dist as u64 {
+            unsafe {
+                let a_chunk = u64::from_ne_bytes(*(a.as_ptr().add(i) as *const [u8; 8]));
+                let b_chunk = u64::from_ne_bytes(*(b.as_ptr().add(i) as *const [u8; 8]));
+                difference += popcnt64_classic(a_chunk ^ b_chunk);
+            }
+            if difference > max_dist_u64 {
                 return 0;
             }
             i += 8;
         }
         while i < length {
-            difference += popcnt64_classic((a[i] ^ b[i]) as u64);
-            if difference > max_dist as u64 {
+            unsafe {
+                difference += popcnt64_classic((*a.get_unchecked(i) ^ *b.get_unchecked(i)) as u64);
+            }
+            if difference > max_dist_u64 {
                 return 0;
             }
             i += 1;
@@ -110,37 +221,74 @@ fn hamming_distance_bytes_classic(a: &[u8], b: &[u8], max_dist: i64) -> u64 {
 }
 
 /// Calculate hamming distance between two byte arrays using native popcount
+/// Optimized with aggressive loop unrolling and bounds check elimination
+#[inline(always)]
 fn hamming_distance_bytes_native(a: &[u8], b: &[u8], max_dist: i64) -> u64 {
-    let mut difference: u64 = 0;
     let length = a.len();
 
     if max_dist < 0 {
+        let mut difference: u64 = 0;
         let mut i = 0;
+        
+        // Process 32 bytes at a time (4 x 8-byte chunks) to saturate execution units
+        while i + 32 <= length {
+            unsafe {
+                let a0 = u64::from_ne_bytes(*(a.as_ptr().add(i) as *const [u8; 8]));
+                let b0 = u64::from_ne_bytes(*(b.as_ptr().add(i) as *const [u8; 8]));
+                let a1 = u64::from_ne_bytes(*(a.as_ptr().add(i + 8) as *const [u8; 8]));
+                let b1 = u64::from_ne_bytes(*(b.as_ptr().add(i + 8) as *const [u8; 8]));
+                let a2 = u64::from_ne_bytes(*(a.as_ptr().add(i + 16) as *const [u8; 8]));
+                let b2 = u64::from_ne_bytes(*(b.as_ptr().add(i + 16) as *const [u8; 8]));
+                let a3 = u64::from_ne_bytes(*(a.as_ptr().add(i + 24) as *const [u8; 8]));
+                let b3 = u64::from_ne_bytes(*(b.as_ptr().add(i + 24) as *const [u8; 8]));
+                
+                difference += popcnt64_native(a0 ^ b0)
+                           + popcnt64_native(a1 ^ b1)
+                           + popcnt64_native(a2 ^ b2)
+                           + popcnt64_native(a3 ^ b3);
+            }
+            i += 32;
+        }
+        
+        // Process remaining 8-byte chunks
         while i + 8 <= length {
-            let a_chunk = u64::from_ne_bytes(a[i..i + 8].try_into().unwrap());
-            let b_chunk = u64::from_ne_bytes(b[i..i + 8].try_into().unwrap());
-            difference += popcnt64_native(a_chunk ^ b_chunk);
+            unsafe {
+                let a_chunk = u64::from_ne_bytes(*(a.as_ptr().add(i) as *const [u8; 8]));
+                let b_chunk = u64::from_ne_bytes(*(b.as_ptr().add(i) as *const [u8; 8]));
+                difference += popcnt64_native(a_chunk ^ b_chunk);
+            }
             i += 8;
         }
+        
+        // Process remaining bytes
         while i < length {
-            difference += popcnt64_native((a[i] ^ b[i]) as u64);
+            unsafe {
+                difference += (*a.get_unchecked(i) ^ *b.get_unchecked(i)).count_ones() as u64;
+            }
             i += 1;
         }
         difference
     } else {
+        let max_dist_u64 = max_dist as u64;
+        let mut difference: u64 = 0;
         let mut i = 0;
+        
         while i + 8 <= length {
-            let a_chunk = u64::from_ne_bytes(a[i..i + 8].try_into().unwrap());
-            let b_chunk = u64::from_ne_bytes(b[i..i + 8].try_into().unwrap());
-            difference += popcnt64_native(a_chunk ^ b_chunk);
-            if difference > max_dist as u64 {
+            unsafe {
+                let a_chunk = u64::from_ne_bytes(*(a.as_ptr().add(i) as *const [u8; 8]));
+                let b_chunk = u64::from_ne_bytes(*(b.as_ptr().add(i) as *const [u8; 8]));
+                difference += popcnt64_native(a_chunk ^ b_chunk);
+            }
+            if difference > max_dist_u64 {
                 return 0;
             }
             i += 8;
         }
         while i < length {
-            difference += popcnt64_native((a[i] ^ b[i]) as u64);
-            if difference > max_dist as u64 {
+            unsafe {
+                difference += (*a.get_unchecked(i) ^ *b.get_unchecked(i)).count_ones() as u64;
+            }
+            if difference > max_dist_u64 {
                 return 0;
             }
             i += 1;