Add nodejs zlib binary compatibility and update benchmark

Author: puzrin

.github/workflows/ci.yml

@@ -15,9 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        # This is last node with original zlib.
-        # It's required to test deflate binary equality.
-        node-version: [ '12.16' ]
+        node-version: [ '24' ]
 
     steps:
     - uses: actions/checkout@v2

README.md

@@ -19,30 +19,16 @@ develop native C modules for CPU-intensive tasks. Enjoy the result!
 __Benchmarks:__
 
 
-node v12.16.3 (zlib 1.2.9), 1mb input sample:
+node v24, 1mb input sample:
 
 ```
-deflate-imaya x 4.75 ops/sec ±4.93% (15 runs sampled)
-deflate-pako x 10.38 ops/sec ±0.37% (29 runs sampled)
-deflate-zlib x 17.74 ops/sec ±0.77% (46 runs sampled)
-gzip-pako x 8.86 ops/sec ±1.41% (29 runs sampled)
-inflate-imaya x 107 ops/sec ±0.69% (77 runs sampled)
-inflate-pako x 131 ops/sec ±1.74% (82 runs sampled)
-inflate-zlib x 258 ops/sec ±0.66% (88 runs sampled)
-ungzip-pako x 115 ops/sec ±1.92% (80 runs sampled)
-```
-
-node v14.15.0 (google's zlib), 1mb output sample:
-
-```
-deflate-imaya x 4.93 ops/sec ±3.09% (16 runs sampled)
-deflate-pako x 10.22 ops/sec ±0.33% (29 runs sampled)
-deflate-zlib x 18.48 ops/sec ±0.24% (48 runs sampled)
-gzip-pako x 10.16 ops/sec ±0.25% (28 runs sampled)
-inflate-imaya x 110 ops/sec ±0.41% (77 runs sampled)
-inflate-pako x 134 ops/sec ±0.66% (83 runs sampled)
-inflate-zlib x 402 ops/sec ±0.74% (87 runs sampled)
-ungzip-pako x 113 ops/sec ±0.62% (80 runs sampled)
+deflate-pako x 14.27 ops/sec ±3.41% (37 runs sampled)
+deflate-pako-zlib-hash x 10.60 ops/sec ±0.50% (29 runs sampled)
+deflate-zlib x 30.30 ops/sec ±0.61% (51 runs sampled)
+gzip-pako x 13.48 ops/sec ±0.50% (36 runs sampled)
+inflate-pako x 138 ops/sec ±1.26% (75 runs sampled)
+inflate-zlib x 397 ops/sec ±1.37% (81 runs sampled)
+ungzip-pako x 125 ops/sec ±1.46% (73 runs sampled)
 ```
 
 zlib's test is partially affected by marshalling (that make sense for inflate only).
@@ -143,14 +129,6 @@ Pako does not contain some specific zlib functions:
   modes.
 
 
-pako for enterprise
--------------------
-
-Available as part of the Tidelift Subscription
-
-The maintainers of pako and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source dependencies you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. [Learn more.](https://tidelift.com/subscription/pkg/npm-pako?utm_source=npm-pako&utm_medium=referral&utm_campaign=enterprise&utm_term=repo)
-
-
 Authors
 -------
 

benchmark/implementations/deflate-imaya/index.js

@@ -0,0 +1,8 @@
+'use strict';
+
+const pako = require('../../../');
+
+// Pako with the classic zlib hash (legacyHash), instead of fast ANZAC++.
+exports.run = (data, level) => {
+  return pako.deflate(data.typed, { level: level, legacyHash: true });
+};

benchmark/implementations/deflate-pako-zlib-hash/index.js

@@ -3,7 +3,6 @@
   "version": "0.0.0",
   "dependencies": {
     "ansi": "^0.3.1",
-    "benchmark": "^2.1.4",
-    "zlibjs": "^0.3.1"
+    "benchmark": "^2.1.4"
   }
 }

benchmark/implementations/inflate-imaya/index.js

@@ -77,6 +77,10 @@ const {
  * [http://zlib.net/manual.html#Advanced](http://zlib.net/manual.html#Advanced)
  * for more information on these.
  *
+ * - `legacyHash` (Boolean) - use the classic zlib hash instead of the default
+ *   ANZAC++ hash. The default matches node.js output; enable to match canonical
+ *   zlib output instead.
+ *
  * Additional options, for internal needs:
  *
  * - `chunkSize` - size of generated data chunks (16K by default)
@@ -115,7 +119,8 @@ function Deflate(options) {
     chunkSize: 16384,
     windowBits: 15,
     memLevel: 8,
-    strategy: Z_DEFAULT_STRATEGY
+    strategy: Z_DEFAULT_STRATEGY,
+    legacyHash: false
   }, options || {});
 
   let opt = this.options;
@@ -142,7 +147,8 @@ function Deflate(options) {
     opt.method,
     opt.windowBits,
     opt.memLevel,
-    opt.strategy
+    opt.strategy,
+    opt.legacyHash
   );
 
   if (status !== Z_OK) {

benchmark/package.json

@@ -144,10 +144,21 @@ let HASH = HASH_ZLIB;
  *    the last MIN_MATCH-1 bytes of the input file).
  */
 const INSERT_STRING = (s, str) => {
-  /* UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]); */
-  s.ins_h = HASH(s, s.ins_h, s.window[str + MIN_MATCH - 1]);
-  const hash_head = s.prev[str & s.w_mask] = s.head[s.ins_h];
-  s.head[s.ins_h] = str;
+  let h;
+  if (s.legacy_hash) {
+    /* UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]); */
+    h = s.ins_h = HASH(s, s.ins_h, s.window[str + MIN_MATCH - 1]);
+  } else {
+    // ANZAC++ hash: reads 4 bytes, matches node.js zlib output (legacyHash
+    // restores classic zlib hash). Faster, with fewer collisions.
+    const w = s.window;
+    // Read 4 bytes little-endian. Math.imul reproduces C uint32 overflow in
+    // `(value * 66521 + 66521) >> 16` exactly.
+    const value = w[str] | (w[str + 1] << 8) | (w[str + 2] << 16) | (w[str + 3] << 24);
+    h = s.ins_h = ((Math.imul(value, 66521) + 66521) >>> 16) & s.hash_mask;
+  }
+  const hash_head = s.prev[str & s.w_mask] = s.head[h];
+  s.head[h] = str;
   return hash_head;
 };
 
@@ -423,7 +434,20 @@ const fill_window = (s) => {
     s.lookahead += n;
 
     /* Initialize the hash value now that we have some input: */
-    if (s.lookahead + s.insert >= MIN_MATCH) {
+    if (!s.legacy_hash) {
+      /* The 4-byte hash reads one extra byte, so it needs one more available. */
+      if (s.lookahead + s.insert > MIN_MATCH) {
+        str = s.strstart - s.insert;
+        while (s.insert) {
+          INSERT_STRING(s, str);
+          str++;
+          s.insert--;
+          if (s.lookahead + s.insert <= MIN_MATCH) {
+            break;
+          }
+        }
+      }
+    } else if (s.lookahead + s.insert >= MIN_MATCH) {
       str = s.strstart - s.insert;
       s.ins_h = s.window[str];
 
@@ -433,11 +457,7 @@ const fill_window = (s) => {
 //        Call update_hash() MIN_MATCH-3 more times
 //#endif
       while (s.insert) {
-        /* UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); */
-        s.ins_h = HASH(s, s.ins_h, s.window[str + MIN_MATCH - 1]);
-
-        s.prev[str & s.w_mask] = s.head[s.ins_h];
-        s.head[s.ins_h] = str;
+        INSERT_STRING(s, str);
         str++;
         s.insert--;
         if (s.lookahead + s.insert < MIN_MATCH) {
@@ -775,16 +795,18 @@ const deflate_fast = (s, flush) => {
       {
         s.strstart += s.match_length;
         s.match_length = 0;
-        s.ins_h = s.window[s.strstart];
-        /* UPDATE_HASH(s, s.ins_h, s.window[s.strstart+1]); */
-        s.ins_h = HASH(s, s.ins_h, s.window[s.strstart + 1]);
+        if (s.legacy_hash) {
+          s.ins_h = s.window[s.strstart];
+          /* UPDATE_HASH(s, s.ins_h, s.window[s.strstart+1]); */
+          s.ins_h = HASH(s, s.ins_h, s.window[s.strstart + 1]);
 
 //#if MIN_MATCH != 3
 //                Call UPDATE_HASH() MIN_MATCH-3 more times
 //#endif
-        /* If lookahead < MIN_MATCH, ins_h is garbage, but it does not
-         * matter since it will be recomputed at next deflate call.
-         */
+          /* If lookahead < MIN_MATCH, ins_h is garbage, but it does not
+           * matter since it will be recomputed at next deflate call.
+           */
+        }
       }
     } else {
       /* No match, output a literal byte */
@@ -1230,6 +1252,7 @@ function DeflateState() {
   this.head = null;   /* Heads of the hash chains or NIL. */
 
   this.ins_h = 0;       /* hash index of string to be inserted */
+  this.legacy_hash = 0; /* use classic zlib hash instead of default ANZAC++ */
   this.hash_size = 0;   /* number of elements in hash table */
   this.hash_bits = 0;   /* log2(hash_size) */
   this.hash_mask = 0;   /* hash_size-1 */
@@ -1452,7 +1475,7 @@ const deflateSetHeader = (strm, head) => {
 };
 
 
-const deflateInit2 = (strm, level, method, windowBits, memLevel, strategy) => {
+const deflateInit2 = (strm, level, method, windowBits, memLevel, strategy, legacyHash) => {
 
   if (!strm) { // === Z_NULL
     return Z_STREAM_ERROR;
@@ -1498,7 +1521,13 @@ const deflateInit2 = (strm, level, method, windowBits, memLevel, strategy) => {
   s.w_size = 1 << s.w_bits;
   s.w_mask = s.w_size - 1;
 
+  s.legacy_hash = legacyHash ? 1 : 0;
+
   s.hash_bits = memLevel + 7;
+  /* ANZAC++ hash needs >= 15 hash bits to span its 4 read bytes. */
+  if (!s.legacy_hash && s.hash_bits < 15) {
+    s.hash_bits = 15;
+  }
   s.hash_size = 1 << s.hash_bits;
   s.hash_mask = s.hash_size - 1;
   s.hash_shift = ~~((s.hash_bits + MIN_MATCH - 1) / MIN_MATCH);
@@ -2002,12 +2031,7 @@ const deflateSetDictionary = (strm, dictionary) => {
     let str = s.strstart;
     let n = s.lookahead - (MIN_MATCH - 1);
     do {
-      /* UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); */
-      s.ins_h = HASH(s, s.ins_h, s.window[str + MIN_MATCH - 1]);
-
-      s.prev[str & s.w_mask] = s.head[s.ins_h];
-
-      s.head[s.ins_h] = str;
+      INSERT_STRING(s, str);
       str++;
     } while (--n);
     s.strstart = str;