Merge pull request #27 from ACCESS-NRI/xxhash

Add faster (~6-10x) hashing algorithm - `xxh3` : see https://github.com/Cyan4973/xxHash#benchmarks

Author: charles-turner-1

.conda/env_dev.yml

@@ -15,3 +15,4 @@ dependencies:
     - pytest
     - pytest-cov
     - versioneer
+    - python-xxhash

.conda/meta.yaml

@@ -27,6 +27,7 @@ requirements:
         - python
         - six
         - pyyaml
+        - python-xxhash
 test:
     imports:
         - yamanifest

pyproject.toml

@@ -13,7 +13,8 @@ dynamic = ["version"]
 requires-python = ">=3.10"
 dependencies = [
     "PyYAML",
-    "six"
+    "six",
+    "xxhash",
 ]
 
 [project.scripts]

test/test_manifest.py

@@ -82,7 +82,7 @@ def test_manifest_read_write():
     files = ['file1','file2']
 
     for filepath in files:
-        mf1.add(os.path.join('test',filepath),['md5','sha1'])
+        mf1.add(os.path.join('test',filepath),['binhash-xxh','md5','sha1'])
 
     assert(len(mf1) == len(files))
 
@@ -116,7 +116,7 @@ def test_manifest_netcdf():
         mf1 = mf.Manifest('mf1.yaml')
 
         for filepath in glob.glob('*.nc'):
-            mf1.add(filepath,['binhash','md5','sha1'])
+            mf1.add(filepath,['binhash-xxh','binhash','md5','sha1'])
 
         mf1.dump()
 
@@ -125,7 +125,7 @@ def test_manifest_netcdf():
         mf2 = mf.Manifest('mf2.yaml')
 
         for filepath in glob.glob('*.nc'):
-            mf2.add(filepath,['binhash','md5','sha1'])
+            mf2.add(filepath,['binhash-xxh','binhash','md5','sha1'])
 
         mf2.dump()
 
@@ -139,7 +139,7 @@ def test_manifest_netcdf():
 
         mf1 = mf.Manifest('mf1.yaml')
 
-        mf1.add(glob.glob('*.nc'),['binhash'])
+        mf1.add(glob.glob('*.nc'),['binhash-xxh','binhash'])
         mf1.add(hashfn=['md5','sha1'])
 
     assert(mf1.equals(mf2))
@@ -152,10 +152,10 @@ def test_manifest_netcdf_changed_time():
 
         for filepath in glob.glob('*.nc'):
             touch(filepath)
-            mf3.add(filepath,['md5','sha1','binhash'])
+            mf3.add(filepath,['md5','sha1','binhash','binhash-xxh'])
 
         mf3.dump()
-        mf3.add(filepath,['md5','sha1','binhash'])
+        mf3.add(filepath,['md5','sha1','binhash','binhash-xxh'])
         mf2 = mf.Manifest('mf2.yaml')
         mf2.load()
 
@@ -176,7 +176,7 @@ def test_manifest_hash_with_binhash():
         mf4 = mf.Manifest('mf4.yaml')
 
         for filepath in glob.glob('*.bin'):
-            mf4.add(filepath,hashfn=['binhash', 'binhash-nomtime'])
+            mf4.add(filepath,hashfn=['binhash', 'binhash-nomtime', 'binhash-xxh'])
 
         mf4.dump()
         assert(mf4.check())
@@ -185,7 +185,7 @@ def test_manifest_hash_with_binhash():
 
         for filepath in glob.glob('*.bin'):
             touch(filepath)
-            mf5.add(filepath,hashfn=['binhash', 'binhash-nomtime'])
+            mf5.add(filepath,hashfn=['binhash', 'binhash-nomtime', 'binhash-xxh'])
 
         hashvals = {}
         assert(not mf4.check())

yamanifest/hashing.py

@@ -21,6 +21,7 @@
 from __future__ import absolute_import, print_function
 
 import hashlib
+import xxhash # Fast hashing library
 import io
 import os
 import sys
@@ -31,11 +32,13 @@
 # List of supported hashes and the ordering used to determine relative expense of
 # calculation
 supported_hashes = [
-    'binhash', 'binhash-nomtime', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'
+    'binhash-xxh','binhash', 'binhash-nomtime', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'
 ]
 
-def _binhash(path, size, include_mtime):
-    m = hashlib.new('md5')
+def _binhash(path, size, include_mtime, use_xxh=False):
+
+    m = xxhash.xxh3_64() if use_xxh else hashlib.new('md5')
+
     with io.open(path, mode="rb") as fd:
         # Size limited hashing, so prepend the filename, size and optionally modification time 
         hashstring = os.path.basename(path) + str(os.path.getsize(path))
@@ -70,7 +73,9 @@ def hash(path, hashfn, size=one_hundred_megabytes):
     if hashfn not in supported_hashes:
         sys.stderr.write('\nUnsupported hash function {}, skipping {}\n'.format(hashfn, path))
     try:
-        if hashfn == 'binhash':
+        if hashfn == 'binhash-xxh':
+            return _binhash(path, one_hundred_megabytes, True, use_xxh=True)
+        elif hashfn == 'binhash':
             return _binhash(path, one_hundred_megabytes, True)
         elif hashfn == 'binhash-nomtime':
             return _binhash(path, one_hundred_megabytes, False)