Test of hash functions, no gain

Author: glorieux-f

test/src/java/com/github/oeuvres/alix/util/CharArrayHashBenchmark.java

@@ -15,47 +15,56 @@
 /**
  * Benchmark CharArrayHash vs Lucene CharArrayMap on a large word list.
  *
- * CLI: --file <path> UTF-8 file, one token per line (optional) --target <N>
- * desired unique tokens (default 600000) --mode <hash|map|both> which
- * structure(s) to test (default both) --lowercase lowercase tokens before
- * insert (optional)
- *
- * Notes: - Uses Δ(used heap) after GC as a coarse footprint estimator. - For
- * reproducible results, fix heap: -Xms4g -Xmx4g -XX:+UseG1GC - Requires your
- * CharArrayHash class on classpath.
+ * Notes:
+ * - This version avoids "same char[] instance" lookup bias by using separate
+ *   precomputed probe arrays for hits.
+ * - Build simulates a copy-required token stream: CharArrayHash copies from a reused buffer
+ *   into its slab, CharArrayMap must allocate a stable char[] per key.
+ * - Uses Δ(used heap) after GC as a coarse footprint estimator.
  */
 public class CharArrayHashBenchmark
 {
+    private static final Random RNG = new Random(0xC0FFEE);
 
     public static void main(String[] args) throws Exception
     {
         final String lexicon = "word.csv";
         InputStream is = French.class.getResourceAsStream(lexicon);
         Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
-        List<char[]> tokens = loadTokens(reader);
+
+        List<String> tokens = loadTokensAsStrings(reader);
         System.out.println("Loaded tokens: " + fmt(tokens.size()));
 
-        List<char[]> hits = new ArrayList<>(tokens);
-        List<char[]> misses = makeMisses(tokens);
+        // Precompute lookup probes (distinct arrays) so timing excludes allocation.
+        List<char[]> hitProbes = makeHitProbes(tokens);
+        List<char[]> missProbes = makeMissesFromHits(hitProbes);
 
         System.out.println("\n=== Lucene CharArrayMap<Integer> ===");
-        runMap(tokens, hits, misses);
+        runMap(tokens, hitProbes, missProbes);
+
         System.out.println("\n=== CharArrayHash ===");
-        runHash(tokens, hits, misses);
+        runHash(tokens, hitProbes, missProbes);
     }
 
     // ---------------------- CharArrayHash ----------------------
 
-    private static void runHash(List<char[]> tokens, List<char[]> hits, List<char[]> misses)
+    private static void runHash(List<String> tokens, List<char[]> hits, List<char[]> misses)
     {
         forceGC();
         long mem0 = usedMem();
         long t0 = System.nanoTime();
 
         CharArrayHash h = new CharArrayHash(tokens.size());
+
+        // Reused buffer: realistic for token streams (copy-required).
+        char[] buf = new char[32];
         int lastOrd = -1;
-        for (char[] w : tokens) {
-            int r = h.add(w, 0, w.length);
+
+        for (String s : tokens) {
+            int len = s.length();
+            if (buf.length < len) buf = new char[len];
+            s.getChars(0, len, buf, 0);
+            int r = h.add(buf, 0, len);
             lastOrd = (r >= 0) ? r : (-r - 1);
         }
         h.trimToSize();
@@ -82,21 +91,26 @@ private static void runHash(List<char[]> tokens, List<char[]> hits, List<char[]>
         });
 
         print("CharArrayHash", tokens.size(), buildNs, bestHitNs, bestMissNs, bytes);
-        if (lastOrd == 42) System.out.print(""); // keep reachable
+        if (lastOrd == 42) System.out.print("");
     }
 
     // ---------------------- CharArrayMap<Integer> ----------------------
 
-    private static void runMap(List<char[]> tokens, List<char[]> hits, List<char[]> misses)
+    private static void runMap(List<String> tokens, List<char[]> hits, List<char[]> misses)
     {
         forceGC();
         long mem0 = usedMem();
         long t0 = System.nanoTime();
 
         CharArrayMap<Integer> map = new CharArrayMap<>(tokens.size(), false);
+
+        // Copy-required: store stable char[] keys, so allocate per token.
         int id = 0;
-        for (char[] w : tokens) {
-            map.put(w, id++); // uses char[] API
+        for (String s : tokens) {
+            int len = s.length();
+            char[] k = new char[len];
+            s.getChars(0, len, k, 0);
+            map.put(k, id++);
         }
 
         long buildNs = System.nanoTime() - t0;
@@ -126,27 +140,33 @@ private static void runMap(List<char[]> tokens, List<char[]> hits, List<char[]>
         if (id == 7) System.out.print("");
     }
 
-    // ---------------------- helpers ----------------------
+    // ---------------------- data prep ----------------------
 
-    private static final Random RNG = new Random(0xC0FFEE);
-
-    private static List<char[]> loadTokens(Reader reader) throws IOException
+    private static List<String> loadTokensAsStrings(Reader reader) throws IOException
     {
-        List<char[]> out = new ArrayList<>(600000);
+        List<String> out = new ArrayList<>(600000);
         CSVReader csvReader = new CSVReader(reader, ',', 1, '"', 16384);
-        // header line
         if (!csvReader.readRow()) throw new IOException("Empty file?");
         while (csvReader.readRow()) {
-        		final int wordLength = csvReader.getCell(0).length();
-        		final char[] word = new char[wordLength];
-        		csvReader.getCell(0).getChars(0, wordLength, word, 0);
-        		out.add(word);
+            out.add(csvReader.getCellAsString(0));
         }
         Collections.shuffle(out, RNG);
         return out;
     }
 
-    private static List<char[]> makeMisses(List<char[]> hits)
+    private static List<char[]> makeHitProbes(List<String> tokens)
+    {
+        List<char[]> hits = new ArrayList<>(tokens.size());
+        for (String s : tokens) {
+            int len = s.length();
+            char[] w = new char[len];
+            s.getChars(0, len, w, 0);
+            hits.add(w);
+        }
+        return hits;
+    }
+
+    private static List<char[]> makeMissesFromHits(List<char[]> hits)
     {
         List<char[]> misses = new ArrayList<>(hits.size());
         for (char[] w : hits) {
@@ -158,6 +178,8 @@ private static List<char[]> makeMisses(List<char[]> hits)
         return misses;
     }
 
+    // ---------------------- helpers ----------------------
+
     private static void forceGC()
     {
         try {
@@ -204,11 +226,9 @@ private static void print(String name, int n, long buildNs, long hitNs, long mis
         System.out.println("Name:           " + name);
         System.out.println("Entries:        " + fmt(n));
         System.out.println("Build time:     " + String.format(Locale.ROOT, "%.2f ms", buildMs));
-        System.out.println(
-                "Hit lookup:     " + String.format(Locale.ROOT, "%.2f ms", hitMs) + "  (" + fmtQps(hitQps) + " qps)");
-        System.out.println(
-                "Miss lookup:    " + String.format(Locale.ROOT, "%.2f ms", missMs) + "  (" + fmtQps(missQps) + " qps)");
-        System.out.println("Heap Δ (bytes): " + fmt(bytes));
+        System.out.println("Hit lookup:     " + String.format(Locale.ROOT, "%.2f ms", hitMs) + "  (" + fmtQps(hitQps) + " qps)");
+        System.out.println("Miss lookup:    " + String.format(Locale.ROOT, "%.2f ms", missMs) + "  (" + fmtQps(missQps) + " qps)");
+        System.out.println("Heap \u0394 (bytes): " + fmt(bytes));
         System.out.println("Bytes/entry:    " + (Double.isNaN(bpe) ? "n/a" : String.format(Locale.ROOT, "%.2f", bpe)));
     }
 
@@ -223,5 +243,4 @@ private static String fmtQps(double qps)
         if (qps >= 1_000) return String.format(Locale.ROOT, "%.1fk", qps);
         return String.format(Locale.ROOT, "%.0f", qps);
     }
-
 }

util/src/java/com/github/oeuvres/alix/util/CharArrayHash.java

@@ -132,7 +132,7 @@ public int add(char[] key, int off, int len)
         checkBounds(key, off, len);
         if (len > MAX_KEY_LENGTH) throw new IllegalArgumentException("key length > 65535: " + len);
 
-        final int h = Murmur3.hashChars(key, off, len, SEED);
+        final int h = murmur3(key, off, len);
         int i = h & mask;
 
         for (;;) {
@@ -176,7 +176,7 @@ public int find(char[] key, int off, int len)
         checkBounds(key, off, len);
         if (len > MAX_KEY_LENGTH) return -1; // cannot exist (see add)
 
-        final int h = Murmur3.hashChars(key, off, len, SEED);
+        final int h = murmur3(key, off, len);
         int i = h & mask;
 
         for (;;) {
@@ -293,7 +293,6 @@ public void freeze()
     // ---- Internals -----------------------------------------------------------
 
     private static final float LOAD_FACTOR = 0.75f;
-    private static final int SEED = 0x9747b28c;
     private static final int MAX_KEY_LENGTH = 0xFFFF;
 
     // Hash table: stores ords (>=0) or -1 if empty.
@@ -433,58 +432,100 @@ private void rehash(int newCap)
         }
     }
 
+
     /**
-     * Murmur3-32 hashing specialized for {@code char[]} inputs (UTF-16 code units).
+     * Computes Murmur3-32 for a UTF-16 {@code char[]} slice using the x86_32 mixing constants.
      *
-     * <p>Processes two {@code char}s per 32-bit block (little-endian packing). The tail handles a
-     * single remaining {@code char} if present. The finalization step mixes in the total number of
-     * bytes processed (i.e., {@code len << 1}).
+     * @param a    input array of UTF-16 code units
+     * @param off  start offset (inclusive)
+     * @param len  number of {@code char} code units to read
+     * @return the 32-bit hash value
+     * @throws IndexOutOfBoundsException if {@code off} or {@code len} are invalid for {@code a}
      */
-    private static final class Murmur3
+    static int murmur3(final char[] a, final int off, final int len)
     {
-        /**
-         * Computes Murmur3-32 for a UTF-16 {@code char[]} slice using the x86_32 mixing constants.
-         *
-         * @param a    input array of UTF-16 code units
-         * @param off  start offset (inclusive)
-         * @param len  number of {@code char} code units to read
-         * @param seed 32-bit seed (use a fixed constant for deterministic behavior)
-         * @return the 32-bit hash value
-         * @throws IndexOutOfBoundsException if {@code off} or {@code len} are invalid for {@code a}
-         */
-        static int hashChars(final char[] a, final int off, final int len, final int seed)
-        {
-            final int n16 = len >>> 1;
-            final boolean odd = (len & 1) != 0;
-            int h1 = seed, idx = off;
-            final int c1 = 0xcc9e2d51, c2 = 0x1b873593;
-
-            for (int i = 0; i < (n16 << 1); i += 2) {
-                int k1 = (a[idx] & 0xFFFF) | ((a[idx + 1] & 0xFFFF) << 16);
-                idx += 2;
-                k1 *= c1;
-                k1 = (k1 << 15) | (k1 >>> 17);
-                k1 *= c2;
-                h1 ^= k1;
-                h1 = (h1 << 13) | (h1 >>> 19);
-                h1 = h1 * 5 + 0xe6546b64;
-            }
-            if (odd) {
-                int k1 = (a[idx] & 0xFFFF);
-                k1 *= c1;
-                k1 = (k1 << 15) | (k1 >>> 17);
-                k1 *= c2;
-                h1 ^= k1;
-            }
-            final int bytes = len << 1;
-            h1 ^= bytes;
-            h1 ^= (h1 >>> 16);
-            h1 *= 0x85ebca6b;
-            h1 ^= (h1 >>> 13);
-            h1 *= 0xc2b2ae35;
-            h1 ^= (h1 >>> 16);
-            return h1;
+        final int seed = 0x9747b28c;
+        final int n16 = len >>> 1;
+        final boolean odd = (len & 1) != 0;
+        int h1 = seed, idx = off;
+        final int c1 = 0xcc9e2d51, c2 = 0x1b873593;
+
+        for (int i = 0; i < (n16 << 1); i += 2) {
+            int k1 = (a[idx] & 0xFFFF) | ((a[idx + 1] & 0xFFFF) << 16);
+            idx += 2;
+            k1 *= c1;
+            k1 = (k1 << 15) | (k1 >>> 17);
+            k1 *= c2;
+            h1 ^= k1;
+            h1 = (h1 << 13) | (h1 >>> 19);
+            h1 = h1 * 5 + 0xe6546b64;
         }
+        if (odd) {
+            int k1 = (a[idx] & 0xFFFF);
+            k1 *= c1;
+            k1 = (k1 << 15) | (k1 >>> 17);
+            k1 *= c2;
+            h1 ^= k1;
+        }
+        final int bytes = len << 1;
+        h1 ^= bytes;
+        h1 ^= (h1 >>> 16);
+        h1 *= 0x85ebca6b;
+        h1 ^= (h1 >>> 13);
+        h1 *= 0xc2b2ae35;
+        h1 ^= (h1 >>> 16);
+        return h1;
     }
 
+    /**
+     * 31-based polynomial hash (String-style) over UTF-16 code units.
+     *
+     * Important: if you index with (h & mask) in a power-of-two table, you should
+     * smear the result (h ^ (h >>> 16)) to improve low-bit distribution.
+     */
+    static int hash31(final char[] a, final int off, final int len)
+    {
+        int h = 0;
+        final int end = off + len;
+        for (int i = off; i < end; i++) {
+            h = 31 * h + a[i];
+        }
+        // smear like HashMap to improve low bits for power-of-two masking
+        h ^= (h >>> 16);
+        return h;
+    }
+    
+    /**
+     * Murmur3 fmix32 finalizer (avalanche).
+     *
+     * <p>Mixes all bits of {@code h} so that small differences in input
+     * produce large differences in output. Useful after a cheap rolling hash
+     * (e.g., 31-polynomial) when indexing with power-of-two masking.</p>
+     */
+    static int fmix32(int h)
+    {
+        h ^= (h >>> 16);
+        h *= 0x85ebca6b;
+        h ^= (h >>> 13);
+        h *= 0xc2b2ae35;
+        h ^= (h >>> 16);
+        return h;
+    }
+    
+    /**
+     * Cheap rolling hash (String-style) + fmix32 avalanche.
+     *
+     * <p>Good compromise for power-of-two hash tables: fast loop, robust final bits.</p>
+     */
+    static int hash31_fmix32(final char[] a, final int off, final int len)
+    {
+        int h = 0;
+        final int end = off + len;
+        for (int i = off; i < end; i++) {
+            h = 31 * h + a[i];
+        }
+        // incorporate length (like String does implicitly via iteration count, but explicit helps)
+        h ^= (len << 1);
+        return fmix32(h);
+    }
 }