Merge: Fast Search for Georgian & Hashing w/out Stack Protection 🔞

Author: ashvardanian

.clang-format

@@ -8,6 +8,23 @@ ReflowComments: true
 UseTab: Never
 IndentPPDirectives: None
 
+# StringZilla-specific macros
+# Visibility/linkage macros that act like storage class specifiers
+StatementAttributeLikeMacros:
+  - SZ_PUBLIC
+  - SZ_INTERNAL
+  - SZ_DYNAMIC
+  - SZ_EXTERNAL
+
+# Attribute-like macros (clang-format 12+)
+AttributeMacros:
+  - SZ_NO_STACK_PROTECTOR
+  - sz_align_
+
+# Macros that behave like type qualifiers in parameters
+TypenameMacros:
+  - sz_at_least_
+
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: false
 AlignEscapedNewlines: true

.gitignore

@@ -31,16 +31,11 @@ node_modules/
 .tmp/
 
 # Recommended datasets
-utf8.txt
-leipzig1M.txt
-enwik9.txt
-xlsum.csv
-acgt_100.txt
-acgt_100k.txt
-acgt_10k.txt
-acgt_10m.txt
-acgt_1k.txt
-acgt_1m.txt
+/utf8.txt
+/enwik9.txt
+/xlsum.csv
+/acgt*.txt
+/leipzig*.txt
 
 # StringZilla-specific log files
 /failed_sz_*

CMakeLists.txt

@@ -861,5 +861,5 @@ if (STRINGZILLA_INSTALL)
     endif ()
 
     install(DIRECTORY ${STRINGZILLA_INCLUDE_BUILD_DIR} DESTINATION ${STRINGZILLA_INCLUDE_INSTALL_DIR})
-    install(DIRECTORY ./c/ DESTINATION /usr/src/${PROJECT_NAME}/)
+    install(DIRECTORY c/ DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/src)
 endif ()

include/stringzilla/compare.h

@@ -347,10 +347,9 @@ SZ_PUBLIC sz_ordering_t sz_order_skylake(sz_cptr_t a, sz_size_t a_length, sz_cpt
     b_vec.zmm = _mm512_maskz_loadu_epi8(head_mask, b);
     __mmask64 mask_not_equal = _mm512_cmpneq_epi8_mask(a_vec.zmm, b_vec.zmm);
     if (mask_not_equal != 0) {
+        // Reload from original memory (L1 cached) to avoid ZMM-to-stack spill.
         sz_u64_t first_diff = _tzcnt_u64(mask_not_equal);
-        char a_char = a_vec.u8s[first_diff];
-        char b_char = b_vec.u8s[first_diff];
-        return sz_order_scalars_(a_char, b_char);
+        return sz_order_scalars_(a[first_diff], b[first_diff]);
     }
     else if (head_length == a_length && head_length == b_length) { return sz_equal_k; }
     else { a += head_length, b += head_length, a_length -= head_length, b_length -= head_length; }
@@ -362,10 +361,9 @@ SZ_PUBLIC sz_ordering_t sz_order_skylake(sz_cptr_t a, sz_size_t a_length, sz_cpt
         b_vec.zmm = _mm512_loadu_si512(b);
         mask_not_equal = _mm512_cmpneq_epi8_mask(a_vec.zmm, b_vec.zmm);
         if (mask_not_equal != 0) {
+            // Reload from original memory (L1 cached) to avoid ZMM-to-stack spill.
             sz_u64_t first_diff = _tzcnt_u64(mask_not_equal);
-            char a_char = a_vec.u8s[first_diff];
-            char b_char = b_vec.u8s[first_diff];
-            return sz_order_scalars_(a_char, b_char);
+            return sz_order_scalars_(a[first_diff], b[first_diff]);
         }
         a += 64, b += 64, a_length -= 64, b_length -= 64;
     }
@@ -381,10 +379,9 @@ SZ_PUBLIC sz_ordering_t sz_order_skylake(sz_cptr_t a, sz_size_t a_length, sz_cpt
         // been cheaper, if we didn't have to apply `_mm256_movemask_epi8` afterwards.
         mask_not_equal = _mm512_cmpneq_epi8_mask(a_vec.zmm, b_vec.zmm);
         if (mask_not_equal != 0) {
+            // Reload from original memory (L1 cached) to avoid ZMM-to-stack spill.
             sz_u64_t first_diff = _tzcnt_u64(mask_not_equal);
-            char a_char = a_vec.u8s[first_diff];
-            char b_char = b_vec.u8s[first_diff];
-            return sz_order_scalars_(a_char, b_char);
+            return sz_order_scalars_(a[first_diff], b[first_diff]);
         }
         // From logic perspective, the hardest cases are "abc\0" and "abc".
         // The result must be `sz_greater_k`, as the latter is shorter.

include/stringzilla/find.h

@@ -1132,17 +1132,34 @@ SZ_PUBLIC sz_cptr_t sz_find_byteset_haswell(sz_cptr_t text, sz_size_t length, sz
 
     // Let's unzip even and odd elements and replicate them into both lanes of the YMM register.
     // That way when we invoke `_mm256_shuffle_epi8` we can use the same mask for both lanes.
+    // Load the 32-byte filter as two 16-byte halves, separate even/odd bytes, pack, and broadcast to YMM.
+    sz_u128_vec_t byte_mask_vec;
+    sz_u128_vec_t filter_lo_vec, filter_hi_vec;
+    sz_u128_vec_t lo_evens_vec, hi_evens_vec;
+    sz_u128_vec_t lo_odds_vec, hi_odds_vec;
+    sz_u128_vec_t evens_xmm_vec, odds_xmm_vec;
     sz_u256_vec_t filter_even_vec, filter_odd_vec;
-    for (sz_size_t i = 0; i != 16; ++i)
-        filter_even_vec.u8s[i] = filter->_u8s[i * 2], filter_odd_vec.u8s[i] = filter->_u8s[i * 2 + 1];
-    filter_even_vec.xmms[1] = filter_even_vec.xmms[0];
-    filter_odd_vec.xmms[1] = filter_odd_vec.xmms[0];
 
     sz_u256_vec_t text_vec;
     sz_u256_vec_t matches_vec;
     sz_u256_vec_t lower_nibbles_vec, higher_nibbles_vec;
     sz_u256_vec_t bitset_even_vec, bitset_odd_vec;
     sz_u256_vec_t bitmask_vec, bitmask_lookup_vec;
+
+    byte_mask_vec.xmm = _mm_set1_epi16(0x00ff);
+
+    filter_lo_vec.xmm = _mm_lddqu_si128((__m128i const *)(filter));
+    filter_hi_vec.xmm = _mm_lddqu_si128((__m128i const *)(filter) + 1);
+    lo_evens_vec.xmm = _mm_and_si128(filter_lo_vec.xmm, byte_mask_vec.xmm);
+    hi_evens_vec.xmm = _mm_and_si128(filter_hi_vec.xmm, byte_mask_vec.xmm);
+    lo_odds_vec.xmm = _mm_srli_epi16(filter_lo_vec.xmm, 8);
+    hi_odds_vec.xmm = _mm_srli_epi16(filter_hi_vec.xmm, 8);
+
+    evens_xmm_vec.xmm = _mm_packus_epi16(lo_evens_vec.xmm, hi_evens_vec.xmm);
+    odds_xmm_vec.xmm = _mm_packus_epi16(lo_odds_vec.xmm, hi_odds_vec.xmm);
+    filter_even_vec.ymm = _mm256_set_m128i(evens_xmm_vec.xmm, evens_xmm_vec.xmm);
+    filter_odd_vec.ymm = _mm256_set_m128i(odds_xmm_vec.xmm, odds_xmm_vec.xmm);
+
     bitmask_lookup_vec.ymm = _mm256_set_epi8(                       //
         -128, 64, 32, 16, 8, 4, 2, 1, -128, 64, 32, 16, 8, 4, 2, 1, //
         -128, 64, 32, 16, 8, 4, 2, 1, -128, 64, 32, 16, 8, 4, 2, 1);

include/stringzilla/hash.h

@@ -279,7 +279,7 @@ SZ_DYNAMIC void sz_sha256_state_digest(sz_sha256_state_t const *state, sz_u8_t d
 SZ_PUBLIC sz_u64_t sz_bytesum_serial(sz_cptr_t text, sz_size_t length);
 
 /** @copydoc sz_hash */
-SZ_PUBLIC sz_u64_t sz_hash_serial(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_serial(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
 
 /** @copydoc sz_fill_random */
 SZ_PUBLIC void sz_fill_random_serial(sz_ptr_t text, sz_size_t length, sz_u64_t nonce);
@@ -296,7 +296,7 @@ SZ_PUBLIC sz_u64_t sz_hash_state_digest_serial(sz_hash_state_t const *state);
 #if SZ_USE_WESTMERE
 
 /** @copydoc sz_hash */
-SZ_PUBLIC sz_u64_t sz_hash_westmere(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_westmere(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
 
 /** @copydoc sz_fill_random */
 SZ_PUBLIC void sz_fill_random_westmere(sz_ptr_t text, sz_size_t length, sz_u64_t nonce);
@@ -338,7 +338,7 @@ SZ_PUBLIC sz_u64_t sz_bytesum_haswell(sz_cptr_t text, sz_size_t length);
 SZ_PUBLIC sz_u64_t sz_bytesum_skylake(sz_cptr_t text, sz_size_t length);
 
 /** @copydoc sz_hash */
-SZ_PUBLIC sz_u64_t sz_hash_skylake(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_skylake(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
 
 /** @copydoc sz_fill_random */
 SZ_PUBLIC void sz_fill_random_skylake(sz_ptr_t text, sz_size_t length, sz_u64_t nonce);
@@ -360,7 +360,7 @@ SZ_PUBLIC sz_u64_t sz_hash_state_digest_skylake(sz_hash_state_t const *state);
 SZ_PUBLIC sz_u64_t sz_bytesum_ice(sz_cptr_t text, sz_size_t length);
 
 /** @copydoc sz_hash */
-SZ_PUBLIC sz_u64_t sz_hash_ice(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_ice(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
 
 /** @copydoc sz_fill_random */
 SZ_PUBLIC void sz_fill_random_ice(sz_ptr_t text, sz_size_t length, sz_u64_t nonce);
@@ -395,7 +395,7 @@ SZ_PUBLIC sz_u64_t sz_bytesum_neon(sz_cptr_t text, sz_size_t length);
 #if SZ_USE_NEON_AES
 
 /** @copydoc sz_hash */
-SZ_PUBLIC sz_u64_t sz_hash_neon(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_neon(sz_cptr_t text, sz_size_t length, sz_u64_t seed);
 
 /** @copydoc sz_fill_random */
 SZ_PUBLIC void sz_fill_random_neon(sz_ptr_t text, sz_size_t length, sz_u64_t nonce);
@@ -824,7 +824,7 @@ SZ_INTERNAL sz_u64_t sz_hash_state_finalize_serial_(sz_hash_state_t const *state
     return mixed_in_register.u64s[0];
 }
 
-SZ_PUBLIC sz_u64_t sz_hash_serial(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_serial(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
     if (length <= 16) {
         // Initialize the AES block with a given seed
         sz_align_(16) sz_hash_minimal_t_ state;
@@ -1328,7 +1328,7 @@ SZ_INTERNAL sz_u64_t sz_hash_state_finalize_westmere_(sz_hash_state_t const *sta
     return _mm_cvtsi128_si64(mixed_in_register);
 }
 
-SZ_PUBLIC sz_u64_t sz_hash_westmere(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_westmere(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
 
     if (length <= 16) {
         // Initialize the AES block with a given seed
@@ -2223,7 +2223,7 @@ SZ_PUBLIC void sz_hash_state_init_skylake(sz_hash_state_t *state, sz_u64_t seed)
     state->ins_length = 0;
 }
 
-SZ_PUBLIC sz_u64_t sz_hash_skylake(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_skylake(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
 
     if (length <= 16) {
         // Initialize the AES block with a given seed
@@ -2562,7 +2562,7 @@ SZ_PUBLIC sz_u64_t sz_bytesum_ice(sz_cptr_t text, sz_size_t length) {
     }
 }
 
-SZ_PUBLIC sz_u64_t sz_hash_ice(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_ice(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
 
     // For short strings the "masked loads" are identical to Skylake-X and
     // the "logic" is identical to Haswell.
@@ -3477,7 +3477,7 @@ SZ_PUBLIC sz_u64_t sz_hash_state_digest_neon(sz_hash_state_t const *state) {
     }
 }
 
-SZ_PUBLIC sz_u64_t sz_hash_neon(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
+SZ_PUBLIC SZ_NO_STACK_PROTECTOR sz_u64_t sz_hash_neon(sz_cptr_t start, sz_size_t length, sz_u64_t seed) {
     if (length <= 16) {
         // Initialize the AES block with a given seed
         sz_align_(16) sz_hash_minimal_t_ state;

include/stringzilla/types.h

@@ -153,25 +153,45 @@
  *  - `SZ_DYNAMIC` is used for functions that are part of the public API, but are dispatched at runtime.
  *  - `SZ_EXTERNAL` is used for third-party libraries that are linked dynamically.
  */
+
+#if defined(__cplusplus)
+#define SZ_C_INLINE inline
+#else
+#define SZ_C_INLINE inline static
+#endif
+
 #if SZ_DYNAMIC_DISPATCH
 #if defined(_WIN32) || defined(__CYGWIN__)
 #define SZ_DYNAMIC __declspec(dllexport)
 #define SZ_EXTERNAL __declspec(dllimport)
-#define SZ_PUBLIC inline static
-#define SZ_INTERNAL inline static
+#define SZ_PUBLIC SZ_C_INLINE
+#define SZ_INTERNAL SZ_C_INLINE
 #else
 #define SZ_DYNAMIC extern __attribute__((visibility("default")))
 #define SZ_EXTERNAL extern
-#define SZ_PUBLIC __attribute__((unused)) inline static
-#define SZ_INTERNAL __attribute__((always_inline)) inline static
+#define SZ_PUBLIC __attribute__((unused)) SZ_C_INLINE
+#define SZ_INTERNAL __attribute__((always_inline)) SZ_C_INLINE
 #endif // _WIN32 || __CYGWIN__
 #else
-#define SZ_DYNAMIC inline static
+#define SZ_DYNAMIC SZ_C_INLINE
 #define SZ_EXTERNAL extern
-#define SZ_PUBLIC inline static
-#define SZ_INTERNAL inline static
+#define SZ_PUBLIC SZ_C_INLINE
+#define SZ_INTERNAL SZ_C_INLINE
 #endif // SZ_DYNAMIC_DISPATCH
 
+/**
+ *  @brief  Disables stack protection for performance-critical functions.
+ *
+ *  GCC's `-fstack-protector-strong` inserts stack canary checks for functions with local arrays
+ *  or buffers. For hash functions that use fixed-size state structures, this is unnecessary
+ *  overhead (~10 cycles per call). This macro opts out of stack protection for such functions.
+ */
+#if defined(__GNUC__) || defined(__clang__)
+#define SZ_NO_STACK_PROTECTOR __attribute__((no_stack_protector))
+#else
+#define SZ_NO_STACK_PROTECTOR
+#endif
+
 /**
  *  @brief  Alignment macro for N-byte alignment.
  */
@@ -1358,11 +1378,21 @@ SZ_INTERNAL sz_size_t sz_size_log2i_nonzero(sz_size_t x) {
 
 /**
  *  @brief Compute the smallest power of two greater than or equal to @p x.
- *  @pre Unlike the commonly used trick with `clz` intrinsics, is valid across the whole range of `x`, @b including
- * 0.
+ *  @note  Uses LZCNT/CLZ for efficient computation on modern CPUs.
+ *         Edge cases: bit_ceil(0) = 0, bit_ceil(1) = 1.
  *  @see https://stackoverflow.com/a/10143264
  */
 SZ_INTERNAL sz_size_t sz_size_bit_ceil(sz_size_t x) {
+#if defined(__LZCNT__) || defined(__BMI__)
+    // Edge cases: 0 and 1 return themselves, avoids undefined clz(0).
+    if (x <= 1) return x;
+#if SZ_IS_64BIT_
+    return (sz_size_t)1 << (64 - sz_u64_clz(x - 1));
+#else
+    return (sz_size_t)1 << (32 - sz_u32_clz((sz_u32_t)(x - 1)));
+#endif
+#else
+    // The following trick is valid for 0 input as well.
     x--;
     x |= x >> 1;
     x |= x >> 2;
@@ -1374,6 +1404,7 @@ SZ_INTERNAL sz_size_t sz_size_bit_ceil(sz_size_t x) {
 #endif
     x++;
     return x;
+#endif
 }
 
 /**