diff --git a/README.rst b/README.rst index fc060d4..d1162bd 100644 --- a/README.rst +++ b/README.rst @@ -183,7 +183,7 @@ Benchmark --------- All benchmarks were run on an Apple M4 Max (ARM64, 16 logical cores, 64 GiB) -with hexhamming v3.0.0, ``rustc`` 1.96.1, and Python 3.14.6. Values are the +with hexhamming v3.0.0, ``rustc`` 1.97.1, and Python 3.14.6. Values are the median of the means from three independent runs. Raw Rust (no Python overhead) @@ -192,6 +192,24 @@ Raw Rust (no Python overhead) These numbers show the pure computation time using Rust's ``criterion`` benchmarks (``cargo bench --no-default-features``), with no Python/PyO3 overhead. +Issue #51 fixed-width array matrix (1024 records; median of three run medians) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The matrix uses deterministic random no-match and exact-midpoint cases for +16-byte and 32-byte records. Each run uses +``--warm-up-time 1 --measurement-time 1 --sample-size 20``. + +==================================== =========== =========== +Case 16-byte (ns) 32-byte (ns) +==================================== =========== =========== +random no-match / first 397.5 797.8 +random no-match / best 407.0 992.2 +random no-match / all 523.7 1047.7 +exact midpoint / first 216.3 422.7 +exact midpoint / best 216.6 414.0 +exact midpoint / all 540.9 822.0 +==================================== =========== =========== + ================================================ =========== Name Mean (ns) ================================================ =========== @@ -257,6 +275,23 @@ all_within_dist [512×16, at end] 537.5 all_within_dist [16384×64, mid] 30,725.4 ====================================================== =========== +Issue #51 Python buffer matrix (1024 records; median of three run medians) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These end-to-end timings use ``timeit.repeat`` with 10,000 calls per sample, +including the PyO3 wrapper and buffer-protocol path. + +==================================== =========== =========== +Case 16-byte (ns) 32-byte (ns) +==================================== =========== =========== +random no-match / first 443.4 836.8 +random no-match / best 487.0 850.2 +random no-match / all 564.8 851.7 +exact midpoint / first 272.2 468.1 +exact midpoint / best 295.0 483.8 +exact midpoint / all 638.8 921.0 +==================================== =========== =========== + For random inputs, the direct APIs also avoid the temporary big integers used by an equivalent standard-library implementation: @@ -276,6 +311,7 @@ dominates (roughly 30–40 ns on this machine). For large inputs (1024+ chars, 16384-element arrays), computation dominates and Python overhead is negligible. Byte operations release the GIL at 16 KiB, while immutable strings use a zero-copy detached path from 4 KiB. Array wrappers release the GIL -at 64 KiB and parallelize with Rayon at 5 MiB; the ``first`` variant additionally -short-circuits on the first hit, so a match near the start is much faster than -one near the end. +at 64 KiB; generic scans parallelize with Rayon at 5 MiB, while the optimized +16/32-byte NEON scanners use a measured 16 MiB crossover. The ``first`` variant +additionally short-circuits on the first hit, so a match near the start is much +faster than one near the end. diff --git a/benches/hamming_bench.rs b/benches/hamming_bench.rs index 01d10b7..a5041d3 100644 --- a/benches/hamming_bench.rs +++ b/benches/hamming_bench.rs @@ -9,7 +9,7 @@ use hexhamming::hex_hamming_distance_pack; // Hex sizes are character counts; byte sizes are the corresponding decoded lengths. const HEX_SIZES: [usize; 5] = [16, 32, 64, 128, 254]; -const BYTE_SIZES: [usize; 5] = [8, 16, 32, 64, 127]; +const BYTE_SIZES: [usize; 8] = [8, 16, 32, 64, 127, 128, 256, 512]; fn pseudo_random_bytes(len: usize, seed: u64) -> Vec { let mut state = seed; @@ -287,6 +287,124 @@ fn bench_array_random_and_boundaries(c: &mut Criterion) { group.finish(); } +fn fixed_width_array_case(width: usize, scenario: &str) -> (Vec, Vec, i64) { + const NUM_ELEMENTS: usize = 1024; + let small = pseudo_random_bytes(width, 0x51 + width as u64); + let mut big = pseudo_random_bytes(NUM_ELEMENTS * width, 0xA1 + width as u64); + let (match_index, max_dist) = match scenario { + "random_no_match" => (None, 0), + "exact_early" => (Some(0), 0), + "exact_mid" => (Some(NUM_ELEMENTS / 2), 0), + "exact_late" => (Some(NUM_ELEMENTS - 1), 0), + "threshold_d_minus_1" => (Some(NUM_ELEMENTS / 2), 3), + "threshold_d" => (Some(NUM_ELEMENTS / 2), 4), + "threshold_d_plus_1" => (Some(NUM_ELEMENTS / 2), 5), + _ => unreachable!("unknown fixed-width benchmark scenario"), + }; + if let Some(index) = match_index { + let record = &mut big[index * width..(index + 1) * width]; + record.copy_from_slice(&small); + if scenario.starts_with("threshold_") { + record[0] ^= 0x0F; + } + } + (big, small, max_dist) +} + +fn bench_fixed_width_array_matrix(c: &mut Criterion) { + let scenarios = [ + "random_no_match", + "exact_early", + "exact_mid", + "exact_late", + "threshold_d_minus_1", + "threshold_d", + "threshold_d_plus_1", + ]; + + for width in [16usize, 32] { + let mut group = c.benchmark_group(format!("array_matrix/{width}byte_records")); + for scenario in scenarios { + let (big, small, max_dist) = fixed_width_array_case(width, scenario); + group.bench_function(format!("{scenario}/first"), |bencher| { + bencher.iter(|| { + bytes_array_first_within_dist( + black_box(&big), + black_box(&small), + black_box(max_dist), + ) + }) + }); + group.bench_function(format!("{scenario}/best"), |bencher| { + bencher.iter(|| { + bytes_array_best_within_dist( + black_box(&big), + black_box(&small), + black_box(max_dist), + ) + }) + }); + group.bench_function(format!("{scenario}/all"), |bencher| { + bencher.iter(|| { + bytes_array_all_within_dist( + black_box(&big), + black_box(&small), + black_box(max_dist), + ) + }) + }); + } + group.finish(); + } +} + +fn bench_fixed_width_parallel_crossover(c: &mut Criterion) { + const PAR_THRESHOLDS: [(&str, usize); 2] = [ + ("legacy", 5 * 1024 * 1024), + ("fixed_width", 16 * 1024 * 1024), + ]; + let mut group = c.benchmark_group("array_matrix/parallel_crossover"); + group.sample_size(10); + for width in [16usize, 32] { + let small = pseudo_random_bytes(width, 0xC1 + width as u64); + for &(threshold_name, threshold_bytes) in &PAR_THRESHOLDS { + let threshold_elements = threshold_bytes / width; + for count in [ + threshold_elements - 1, + threshold_elements, + threshold_elements + 1, + ] { + let big = pseudo_random_bytes(count * width, 0xD1 + count as u64); + group.bench_function( + format!("{threshold_name}/{width}byte/{count}elements/best"), + |bencher| { + bencher.iter(|| { + bytes_array_best_within_dist( + black_box(&big), + black_box(&small), + black_box(0), + ) + }) + }, + ); + group.bench_function( + format!("{threshold_name}/{width}byte/{count}elements/all"), + |bencher| { + bencher.iter(|| { + bytes_array_all_within_dist( + black_box(&big), + black_box(&small), + black_box(0), + ) + }) + }, + ); + } + } + } + group.finish(); +} + #[cfg(target_arch = "aarch64")] fn bench_hex_string_pack(c: &mut Criterion) { // AArch64-only group for the packed NEON hex-string path. @@ -309,6 +427,8 @@ criterion_group!( bench_bytes_within_dist, bench_array_api, bench_array_random_and_boundaries, + bench_fixed_width_array_matrix, + bench_fixed_width_parallel_crossover, bench_hex_string_pack ); #[cfg(not(target_arch = "aarch64"))] @@ -318,6 +438,8 @@ criterion_group!( bench_bytes_by_algo, bench_bytes_within_dist, bench_array_api, - bench_array_random_and_boundaries + bench_array_random_and_boundaries, + bench_fixed_width_array_matrix, + bench_fixed_width_parallel_crossover ); criterion_main!(benches); diff --git a/src/api.rs b/src/api.rs index 784afa8..f361e4b 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,3 +1,5 @@ +#[cfg(target_arch = "aarch64")] +use crate::ALGO_NEON; use crate::{ hamming_distance_bytes_dispatch, hamming_distance_string_dispatch, select_bytes_kernel_for_width, BytesKernel, ALGO_CLASSIC, ALGO_NATIVE, CURRENT_ALGO, @@ -10,11 +12,55 @@ use std::sync::atomic::Ordering; /// Minimum total byte size of big_array before we use rayon parallel paths. const PAR_THRESHOLD_BYTES: usize = 5 * 1024 * 1024; +/// The fixed-width NEON scanners make serial scans substantially cheaper, so +/// use a larger crossover before paying Rayon scheduling and partition costs. +const FIXED_WIDTH_PAR_THRESHOLD_BYTES: usize = 16 * 1024 * 1024; /// Keep byte-array scans to a small number of coarse jobs. More workers spend /// more time scheduling these very small per-record calculations than running /// them on current many-core CPUs. const PAR_JOBS: usize = 4; +type ArrayFirstScanner = fn(&[u8], &[u8], i64) -> Option; +type ArrayBestScanner = fn(&[u8], &[u8], i64) -> Option<(u64, usize)>; +type ArrayAllScanner = fn(&[u8], &[u8], i64) -> Vec<(u64, usize)>; + +#[derive(Clone, Copy)] +struct ArrayScanner { + first: ArrayFirstScanner, + best: ArrayBestScanner, + all: ArrayAllScanner, +} + +#[inline] +fn select_array_scanner_for_width(width: usize) -> Option { + #[cfg(target_arch = "aarch64")] + { + let algo = CURRENT_ALGO.load(Ordering::Relaxed); + if algo != ALGO_NATIVE && algo != ALGO_NEON { + return None; + } + return match width { + 16 => Some(ArrayScanner { + first: crate::neon_simd::array_first_neon_16, + best: crate::neon_simd::array_best_neon_16, + all: crate::neon_simd::array_all_neon_16, + }), + 32 => Some(ArrayScanner { + first: crate::neon_simd::array_first_neon_32, + best: crate::neon_simd::array_best_neon_32, + all: crate::neon_simd::array_all_neon_32, + }), + _ => None, + }; + } + + #[cfg(not(target_arch = "aarch64"))] + { + let _ = width; + None + } +} + #[inline] fn partition_element_ranges(num_elements: usize) -> [(usize, usize); PAR_JOBS] { let base = num_elements / PAR_JOBS; @@ -103,6 +149,9 @@ pub fn bytes_array_first_within_dist( if big_array.len() % small_array.len() != 0 { return Err("array_of_elems size must be multiplier of elem_to_compare"); } + if let Some(scanner) = select_array_scanner_for_width(small_array.len()) { + return Ok((scanner.first)(big_array, small_array, max_dist)); + } // `first` has early-exit semantics: the serial scan returns as soon as the // first match is found, which is essentially free for early/common matches. // Parallelizing this requires a full non-short-circuiting scan to compute @@ -149,7 +198,16 @@ pub fn bytes_array_best_within_dist( return Err("array_of_elems size must be multiplier of elem_to_compare"); } let kernel = select_bytes_kernel_for_width(small_array.len()); - if big_array.len() < PAR_THRESHOLD_BYTES { + let scanner = select_array_scanner_for_width(small_array.len()); + let parallel_threshold = if scanner.is_some() { + FIXED_WIDTH_PAR_THRESHOLD_BYTES + } else { + PAR_THRESHOLD_BYTES + }; + if big_array.len() < parallel_threshold { + if let Some(scanner) = scanner { + return Ok((scanner.best)(big_array, small_array, max_dist)); + } return Ok(serial_best_within_dist( big_array, small_array, @@ -166,8 +224,11 @@ pub fn bytes_array_best_within_dist( .with_max_len(1) .map(|&(start, end)| { let chunk = &big_array[start * elem_size..end * elem_size]; - serial_best_within_dist(chunk, small_array, max_dist, kernel) - .map(|(distance, index)| (distance, index + start)) + let best = match scanner { + Some(scanner) => (scanner.best)(chunk, small_array, max_dist), + None => serial_best_within_dist(chunk, small_array, max_dist, kernel), + }; + best.map(|(distance, index)| (distance, index + start)) }) .reduce(|| None, merge_best)) } @@ -225,7 +286,16 @@ pub fn bytes_array_all_within_dist( return Err("array_of_elems size must be multiplier of elem_to_compare"); } let kernel = select_bytes_kernel_for_width(small_array.len()); - if big_array.len() < PAR_THRESHOLD_BYTES { + let scanner = select_array_scanner_for_width(small_array.len()); + let parallel_threshold = if scanner.is_some() { + FIXED_WIDTH_PAR_THRESHOLD_BYTES + } else { + PAR_THRESHOLD_BYTES + }; + if big_array.len() < parallel_threshold { + if let Some(scanner) = scanner { + return Ok((scanner.all)(big_array, small_array, max_dist)); + } return Ok(serial_all_within_dist( big_array, small_array, @@ -241,7 +311,11 @@ pub fn bytes_array_all_within_dist( .with_max_len(1) .map(|&(start, end)| { let chunk = &big_array[start * elem_size..end * elem_size]; - serial_all_within_dist(chunk, small_array, max_dist, kernel) + let matches = match scanner { + Some(scanner) => (scanner.all)(chunk, small_array, max_dist), + None => serial_all_within_dist(chunk, small_array, max_dist, kernel), + }; + matches .into_iter() .map(|(distance, index)| (distance, index + start)) .collect() diff --git a/src/neon_simd.rs b/src/neon_simd.rs index fab641b..c1cec82 100644 --- a/src/neon_simd.rs +++ b/src/neon_simd.rs @@ -163,6 +163,209 @@ pub(crate) unsafe fn hamming_distance_bytes_neon(a: &[u8], b: &[u8], max_dist: i } } +#[inline(always)] +unsafe fn hamming_distance_neon_fixed( + record: *const u8, + query: *const u8, +) -> u64 { + let query0 = vld1q_u8(query); + let record0 = vld1q_u8(record); + let mut distance = vaddlvq_u8(vcntq_u8(veorq_u8(record0, query0))) as u64; + + if WIDTH == 32 { + let query1 = vld1q_u8(query.add(16)); + let record1 = vld1q_u8(record.add(16)); + distance += vaddlvq_u8(vcntq_u8(veorq_u8(record1, query1))) as u64; + } + distance +} + +#[inline(always)] +unsafe fn hamming_distance_neon_fixed4( + records: *const u8, + query: *const u8, +) -> [u64; 4] { + [ + hamming_distance_neon_fixed::(records, query), + hamming_distance_neon_fixed::(records.add(WIDTH), query), + hamming_distance_neon_fixed::(records.add(WIDTH * 2), query), + hamming_distance_neon_fixed::(records.add(WIDTH * 3), query), + ] +} + +#[inline(always)] +fn within_fixed_threshold(distance: u64, max_dist: i64) -> bool { + max_dist < 0 || distance <= max_dist as u64 +} + +unsafe fn array_first_neon( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Option { + let count = big_array.len() / WIDTH; + let mut index = 0; + while index + 4 <= count { + let distances = hamming_distance_neon_fixed4::( + big_array.as_ptr().add(index * WIDTH), + small_array.as_ptr(), + ); + for (lane, &distance) in distances.iter().enumerate() { + if within_fixed_threshold(distance, max_dist) { + return Some(index + lane); + } + } + index += 4; + } + while index < count { + let distance = hamming_distance_neon_fixed::( + big_array.as_ptr().add(index * WIDTH), + small_array.as_ptr(), + ); + if within_fixed_threshold(distance, max_dist) { + return Some(index); + } + index += 1; + } + None +} + +unsafe fn array_best_neon( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Option<(u64, usize)> { + let count = big_array.len() / WIDTH; + let mut best: Option<(u64, usize)> = None; + let mut index = 0; + + while index + 4 <= count { + let distances = hamming_distance_neon_fixed4::( + big_array.as_ptr().add(index * WIDTH), + small_array.as_ptr(), + ); + for (lane, &distance) in distances.iter().enumerate() { + let candidate_index = index + lane; + let eligible = match best { + Some((best_distance, _)) => distance < best_distance, + None => within_fixed_threshold(distance, max_dist), + }; + if !eligible { + continue; + } + if best.is_none() || distance < best.unwrap().0 { + best = Some((distance, candidate_index)); + if distance == 0 { + return best; + } + } + } + index += 4; + } + + while index < count { + let distance = hamming_distance_neon_fixed::( + big_array.as_ptr().add(index * WIDTH), + small_array.as_ptr(), + ); + let eligible = match best { + Some((best_distance, _)) => distance < best_distance, + None => within_fixed_threshold(distance, max_dist), + }; + if eligible { + best = Some((distance, index)); + if distance == 0 { + return best; + } + } + index += 1; + } + best +} + +unsafe fn array_all_neon( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Vec<(u64, usize)> { + let count = big_array.len() / WIDTH; + let mut matches = Vec::new(); + let mut index = 0; + + while index + 4 <= count { + let distances = hamming_distance_neon_fixed4::( + big_array.as_ptr().add(index * WIDTH), + small_array.as_ptr(), + ); + for (lane, &distance) in distances.iter().enumerate() { + if within_fixed_threshold(distance, max_dist) { + matches.push((distance, index + lane)); + } + } + index += 4; + } + + while index < count { + let distance = hamming_distance_neon_fixed::( + big_array.as_ptr().add(index * WIDTH), + small_array.as_ptr(), + ); + if within_fixed_threshold(distance, max_dist) { + matches.push((distance, index)); + } + index += 1; + } + matches +} + +pub(crate) fn array_first_neon_16( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Option { + unsafe { array_first_neon::<16>(big_array, small_array, max_dist) } +} + +pub(crate) fn array_best_neon_16( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Option<(u64, usize)> { + unsafe { array_best_neon::<16>(big_array, small_array, max_dist) } +} + +pub(crate) fn array_all_neon_16( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Vec<(u64, usize)> { + unsafe { array_all_neon::<16>(big_array, small_array, max_dist) } +} + +pub(crate) fn array_first_neon_32( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Option { + unsafe { array_first_neon::<32>(big_array, small_array, max_dist) } +} + +pub(crate) fn array_best_neon_32( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Option<(u64, usize)> { + unsafe { array_best_neon::<32>(big_array, small_array, max_dist) } +} + +pub(crate) fn array_all_neon_32( + big_array: &[u8], + small_array: &[u8], + max_dist: i64, +) -> Vec<(u64, usize)> { + unsafe { array_all_neon::<32>(big_array, small_array, max_dist) } +} + /// NEON vectorized hamming distance for hex strings. /// Processes 16 ASCII hex chars per iteration using: /// - vqtbl1q_u8 for branchless hex→nibble conversion diff --git a/src/tests.rs b/src/tests.rs index 782c881..c6a531c 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -478,3 +478,113 @@ fn test_string_dispatch_with_max_mixed_pattern() { } } } + +fn array_oracle( + big: &[u8], + small: &[u8], + max_dist: i64, +) -> (Option, Option<(u64, usize)>, Vec<(u64, usize)>) { + let width = small.len(); + let mut first = None; + let mut best = None; + let mut all = Vec::new(); + for (index, record) in big.chunks_exact(width).enumerate() { + let distance = expected_byte_distance(record, small); + if max_dist >= 0 && distance > max_dist as u64 { + continue; + } + if first.is_none() { + first = Some(index); + } + if best + .map(|(best_distance, best_index)| { + distance < best_distance || (distance == best_distance && index < best_index) + }) + .unwrap_or(true) + { + best = Some((distance, index)); + } + all.push((distance, index)); + } + (first, best, all) +} + +#[test] +fn test_fixed_width_array_scanners_match_randomized_oracle() { + for algorithm in ["native", "classic"] { + crate::set_algorithm(algorithm).unwrap(); + for &width in &[1usize, 3, 7, 15, 16, 17, 31, 32, 33] { + let count = 37; + let mut state = 0xA5A5_1234_5678_9ABCu64 ^ width as u64; + let mut next_byte = || { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (state >> 56) as u8 + }; + let small: Vec = (0..width).map(|_| next_byte()).collect(); + let mut big: Vec = (0..count * width).map(|_| next_byte()).collect(); + + // Duplicate exact matches test lowest-index tie behavior and all + // result ordering. A four-bit near match supplies d-1/d/d+1 + // threshold cases without relying on random distances. + for &index in &[2usize, 19, 36] { + big[index * width..(index + 1) * width].copy_from_slice(&small); + } + let near_index = 12; + big[near_index * width..(near_index + 1) * width].copy_from_slice(&small); + big[near_index * width] ^= 0b1111; + + for max_dist in [0, 3, 4, 5, 8, -1] { + let expected = array_oracle(&big, &small, max_dist); + assert_eq!( + bytes_array_first_within_dist(&big, &small, max_dist).unwrap(), + expected.0, + "first mismatch algorithm={algorithm} width={width} max_dist={max_dist}" + ); + assert_eq!( + bytes_array_best_within_dist(&big, &small, max_dist).unwrap(), + expected.1, + "best mismatch algorithm={algorithm} width={width} max_dist={max_dist}" + ); + assert_eq!( + bytes_array_all_within_dist(&big, &small, max_dist).unwrap(), + expected.2, + "all mismatch algorithm={algorithm} width={width} max_dist={max_dist}" + ); + } + } + } + crate::set_algorithm("native").unwrap(); +} + +#[test] +fn test_parallel_fixed_width_scanners_preserve_boundaries_and_order() { + crate::set_algorithm("native").unwrap(); + let width = 16; + let count = (16 * 1024 * 1024) / width + 7; + let small = vec![0u8; width]; + let mut big = vec![0xFFu8; count * width]; + let quarter = count / 4; + let exact_indices = [quarter - 1, quarter, quarter + 1, count - 1]; + for &index in &exact_indices { + big[index * width..(index + 1) * width].copy_from_slice(&small); + } + + assert_eq!( + bytes_array_first_within_dist(&big, &small, 0).unwrap(), + Some(exact_indices[0]) + ); + assert_eq!( + bytes_array_best_within_dist(&big, &small, 0).unwrap(), + Some((0, exact_indices[0])) + ); + let all = bytes_array_all_within_dist(&big, &small, 0).unwrap(); + assert_eq!( + all, + exact_indices + .into_iter() + .map(|index| (0, index)) + .collect::>() + ); +} diff --git a/test/test_hexhamming.py b/test/test_hexhamming.py index d61ebd2..a10d243 100644 --- a/test/test_hexhamming.py +++ b/test/test_hexhamming.py @@ -1,14 +1,16 @@ #!/usr/bin/env python +import random from platform import machine + import pytest from hexhamming import ( - check_hexstrings_within_dist, + check_bytes_arrays_all_within_dist, + check_bytes_arrays_best_within_dist, + check_bytes_arrays_first_within_dist, check_bytes_within_dist, - hamming_distance_string, + check_hexstrings_within_dist, hamming_distance_bytes, - check_bytes_arrays_first_within_dist, - check_bytes_arrays_best_within_dist, - check_bytes_arrays_all_within_dist, + hamming_distance_string, set_algo, ) @@ -393,6 +395,38 @@ def test_check_bytes_arrays_all_within_dist_calculation( assert expected == check_bytes_arrays_all_within_dist(bytes1, bytes2, max_dist) +@pytest.mark.parametrize("width", (16, 32)) +def test_fixed_width_array_apis_randomized_oracle(width): + rng = random.Random(0x51_0000 + width) + count = 37 + needle = bytes(rng.randrange(256) for _ in range(width)) + records = [bytes(rng.randrange(256) for _ in range(width)) for _ in range(count)] + for index in (2, 19, 36): + records[index] = needle + near = bytearray(needle) + near[0] ^= 0x0F + records[12] = bytes(near) + array = b"".join(records) + + for max_dist in (0, 3, 4, 5, 8): + distances = [ + (int.from_bytes(record, "big") ^ int.from_bytes(needle, "big")).bit_count() + for record in records + ] + expected = [ + (distance, index) + for index, distance in enumerate(distances) + if distance <= max_dist + ] + assert check_bytes_arrays_first_within_dist(array, needle, max_dist) == ( + expected[0][1] if expected else -1 + ) + assert check_bytes_arrays_best_within_dist(array, needle, max_dist) == ( + min(expected, key=lambda item: (item[0], item[1])) if expected else (-1, -1) + ) + assert check_bytes_arrays_all_within_dist(array, needle, max_dist) == expected + + ############################ # Benchmarks # diff --git a/test/test_performance_matrix.py b/test/test_performance_matrix.py index e338e4c..2edd492 100644 --- a/test/test_performance_matrix.py +++ b/test/test_performance_matrix.py @@ -1,8 +1,13 @@ import random import pytest - -from hexhamming import hamming_distance_bytes, hamming_distance_string +from hexhamming import ( + check_bytes_arrays_all_within_dist, + check_bytes_arrays_best_within_dist, + check_bytes_arrays_first_within_dist, + hamming_distance_bytes, + hamming_distance_string, +) def random_bytes(size): @@ -56,3 +61,62 @@ def test_hamming_distance_bytes_gil_boundary_bench(benchmark, size): a = random_bytes(size) b = random_bytes(size + 1)[:size] benchmark(hamming_distance_bytes, a, b) + + +def fixed_width_array_case(width, scenario): + count = 1024 + rng = random.Random(0x51_0000 + width) + needle = bytes(rng.randrange(256) for _ in range(width)) + big = bytearray(rng.randrange(256) for _ in range(count * width)) + if scenario == "random_no_match": + max_dist = 0 + index = None + elif scenario == "exact_early": + max_dist, index = 0, 0 + elif scenario == "exact_mid": + max_dist, index = 0, count // 2 + elif scenario == "exact_late": + max_dist, index = 0, count - 1 + elif scenario == "threshold_d_minus_1": + max_dist, index = 3, count // 2 + elif scenario == "threshold_d": + max_dist, index = 4, count // 2 + elif scenario == "threshold_d_plus_1": + max_dist, index = 5, count // 2 + else: + raise AssertionError(f"unknown scenario: {scenario}") + + if index is not None: + start = index * width + big[start : start + width] = needle + if scenario.startswith("threshold_"): + big[start] ^= 0x0F + return bytes(big), needle, max_dist + + +@pytest.mark.benchmark(group="array_scan_matrix") +@pytest.mark.parametrize("width", (16, 32)) +@pytest.mark.parametrize( + "scenario", + ( + "random_no_match", + "exact_early", + "exact_mid", + "exact_late", + "threshold_d_minus_1", + "threshold_d", + "threshold_d_plus_1", + ), +) +@pytest.mark.parametrize( + "operation", + ( + check_bytes_arrays_first_within_dist, + check_bytes_arrays_best_within_dist, + check_bytes_arrays_all_within_dist, + ), + ids=("first", "best", "all"), +) +def test_fixed_width_array_scan_matrix_bench(benchmark, width, scenario, operation): + big, needle, max_dist = fixed_width_array_case(width, scenario) + benchmark(operation, big, needle, max_dist)