Merge pull request #25 from ACCESS-NRI/Remove-nchash-dependency

Remove nchash dependency

Author: charles-turner-1

.conda/env_dev.yml

@@ -12,7 +12,6 @@ dependencies:
     - libnetcdf
     - six
     - PyYAML
-    - nchash>=0.1.5
     - pytest
     - pytest-cov
     - versioneer

.conda/meta.yaml

@@ -27,8 +27,6 @@ requirements:
         - python
         - six
         - pyyaml
-        - nchash
-
 test:
     imports:
         - yamanifest
@@ -42,4 +40,4 @@ about:
     license: Apache Software
     license_file: LICENSE
     summary: {{ project.get('description') }}
-    license_family: Apache
+    license_family: Apache

.github/workflows/CI.yml

@@ -55,6 +55,21 @@ jobs:
           auto-update-conda: false
           show-channel-urls: true
 
+      - name: Enforce .tar.bz2 packages
+          # Temporary work-arounds while the action uibcdf/action-build-and-upload-conda-packages gets updated:
+          # We create a `~/.condarc` file with the correct options to enforce the use of `.tar.bz2` packages
+          # and we set the channels to be used by conda build
+        shell: bash
+        run: |
+            cat > ~/.condarc << EOF
+            conda-build:
+                pkg_format: .tar.bz2
+            channels:
+                - accessnri
+                - conda-forge
+                - nodefaults
+            EOF
+
       - name: Verify conda recipe
         shell: bash -el {0}
         # Ignores: 

pyproject.toml

@@ -13,7 +13,6 @@ dynamic = ["version"]
 requires-python = ">=3.10"
 dependencies = [
     "PyYAML",
-    "nchash>=0.1.5",
     "six"
 ]
 
@@ -40,4 +39,4 @@ style = "pep440"
 versionfile_source = "yamanifest/_version.py"
 versionfile_build = "yamanifest/_version.py"
 tag_prefix = ""
-parentdir_prefix = "yamanifest-"
+parentdir_prefix = "yamanifest-"

test/test_manifest.py

@@ -16,10 +16,12 @@
 
 from __future__ import print_function
 
-import pytest
-import sys, os, time, glob
+import glob
+import os
 import shutil
-import pdb # Add pdb.set_trace() to set breakpoints
+import sys
+
+import pytest
 
 print("Version: {}".format(sys.version))
 
@@ -28,7 +30,8 @@
 
 verbose = True
 
-import os
+
+
 def touch(fname, times=None):
     with open(fname, 'a'):
         os.utime(fname, times)
@@ -113,7 +116,7 @@ def test_manifest_netcdf():
         mf1 = mf.Manifest('mf1.yaml')
 
         for filepath in glob.glob('*.nc'):
-            mf1.add(filepath,['nchash','md5','sha1'])
+            mf1.add(filepath,['binhash','md5','sha1'])
 
         mf1.dump()
 
@@ -122,7 +125,7 @@ def test_manifest_netcdf():
         mf2 = mf.Manifest('mf2.yaml')
 
         for filepath in glob.glob('*.nc'):
-            mf2.add(filepath,['nchash','md5','sha1'])
+            mf2.add(filepath,['binhash','md5','sha1'])
 
         mf2.dump()
 
@@ -136,7 +139,7 @@ def test_manifest_netcdf():
 
         mf1 = mf.Manifest('mf1.yaml')
 
-        mf1.add(glob.glob('*.nc'),['nchash'])
+        mf1.add(glob.glob('*.nc'),['binhash'])
         mf1.add(hashfn=['md5','sha1'])
 
     assert(mf1.equals(mf2))
@@ -149,10 +152,10 @@ def test_manifest_netcdf_changed_time():
 
         for filepath in glob.glob('*.nc'):
             touch(filepath)
-            mf3.add(filepath,['nchash','md5','sha1'])
+            mf3.add(filepath,['md5','sha1','binhash'])
 
         mf3.dump()
-
+        mf3.add(filepath,['md5','sha1','binhash'])
         mf2 = mf.Manifest('mf2.yaml')
         mf2.load()
 
@@ -198,13 +201,13 @@ def test_manifest_find():
 
     for filepath in mf1:
         # Test for hashes we know should be in the manifest
-        for hashfn in ['nchash','md5','sha1']:
+        for hashfn in ['md5','sha1']:
             hashval = mf1.get(filepath,hashfn)
             print(hashfn,hashval,filepath,mf1.find(hashfn,hashval))
             assert(mf1.find(hashfn,hashval) == filepath)
 
         # Test for one we know shouldn't be there
-        for hashfn in ['binhash',]:
+        for hashfn in ['binhash-nomtime',]:
             hashval = mf1.get(filepath,hashfn)
             print(hashfn,hashval,filepath,mf1.find(hashfn,hashval))
             assert(mf1.find(hashfn,hashval) == None)
@@ -213,9 +216,9 @@ def test_manifest_find():
 
         mf2 = mf.Manifest('mf2.yaml')
 
-        # Make a manifest only with nchash
+        # Make a manifest only with md5
         for filepath in glob.glob('*.nc'):
-            mf1.add(filepath,['nchash'])
+            mf1.add(filepath,['md5'])
 
         # Update with hashes from mf1
         mf2.update_matching_hashes(mf1)
@@ -227,7 +230,7 @@ def test_manifest_find():
     mf3 = mf.Manifest('mf3.yaml')
 
     for filepath in glob.glob(os.path.join('test','testfiles','*.nc')):
-        mf3.add(filepath,['nchash'])
+        mf3.add(filepath,['md5'])
 
     mf3.update_matching_hashes(mf1)
 
@@ -241,7 +244,7 @@ def test_manifest_with_mixed_file_types():
         mf6 = mf.Manifest('mf6.yaml')
 
         for filepath in glob.glob('*.bin') + glob.glob('*.nc'):
-            mf6.add(filepath,hashfn=['nchash','binhash'])
+            mf6.add(filepath,hashfn=['binhash'])
 
         mf6.dump()
         assert(mf6.check())
@@ -261,7 +264,7 @@ def test_open_manifest_and_add():
         mf7 = mf.Manifest('mf7.yaml')
 
         for filepath in glob.glob('*.nc'):
-            mf7.add(filepath,hashfn=['nchash','binhash'])
+            mf7.add(filepath,hashfn=['binhash'])
 
         mf7.dump()
 
@@ -271,7 +274,7 @@ def test_open_manifest_and_add():
         mf7.load()
 
         for filepath in glob.glob('*.bin'):
-            mf7.add(filepath,hashfn=['nchash','binhash'])
+            mf7.add(filepath,hashfn=['binhash'])
 
         mf7.dump()
 
@@ -293,7 +296,7 @@ def test_yamf():
     with cd(os.path.join('test','testfiles_copy')):
 
         files  = glob.glob('*.bin') + glob.glob('*.nc')
-        yamf.main_parse_args(["add","-n","mf8.yaml", "-s", "binhash", "-s", "nchash"] + files)
+        yamf.main_parse_args(["add","-n","mf8.yaml", "-s", "binhash"] + files)
 
         mf8 = mf.Manifest('mf8.yaml')
         mf8.load()
@@ -302,7 +305,7 @@ def test_yamf():
         mf6.load()
 
         assert(mf8.equals(mf6))
-        assert(yamf.main_parse_args(["check","-n","mf8.yaml", "-s", "binhash", "-s", "nchash"]))
+        assert(yamf.main_parse_args(["check","-n","mf8.yaml", "-s", "binhash"]))
 
 
 def test_shortcircuit_condition():
@@ -362,17 +365,17 @@ def test_shortcircuit_add():
         mf6 = mf.Manifest('mf6.yaml')
 
         for filepath in glob.glob('*.bin') + glob.glob('*.nc'):
-            mf6.add(filepath,hashfn=['nchash','binhash'],shortcircuit=True)
+            mf6.add(filepath,hashfn=['md5'],shortcircuit=True)
 
         mf6.dump()
         # print("mf6: ",mf6.data)
         # pdb.set_trace()
 
         assert(mf6.check())
 
-        # Should have no nchash for the bin files
+        # Should have no sha1 for the bin files
         for filepath in glob.glob('*.bin'):
-            assert(mf6.get(filepath,hashfn='nchash') == None)
+            assert(mf6.get(filepath,hashfn='sha1') == None)
 
         # Should have no binhash for the netcdf files
         for filepath in glob.glob('*.nc'):
@@ -385,7 +388,7 @@ def test_malformed_file():
         mf9 = mf.Manifest('mf9.yaml')
 
         for filepath in glob.glob('*.nc'):
-            mf9.add(filepath,['nchash','md5','sha1'])
+            mf9.add(filepath,['md5','sha1'])
 
         # Intentionally alter the format string
         mf9.header["format"] = 'bogus'
@@ -499,4 +502,4 @@ def test_binhash_nomtime():
 
         assert(not mf1.equals(mf2))
         assert(not mf1.equals(mf3))
-        assert(mf2.equals(mf4))
+        assert(mf2.equals(mf4))

yamanifest/hashing.py

@@ -18,33 +18,22 @@
 limitations under the License.
 """
 
-from __future__ import print_function, absolute_import
+from __future__ import absolute_import, print_function
 
 import hashlib
 import io
 import os
 import sys
-from nchash import NCDataHash, NotNetcdfFileError
 
 length=io.DEFAULT_BUFFER_SIZE
 one_hundred_megabytes = 104857600
 
 # List of supported hashes and the ordering used to determine relative expense of
 # calculation
 supported_hashes = [
-    'nchash', 'binhash', 'binhash-nomtime', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'
+    'binhash', 'binhash-nomtime', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'
 ]
 
-def _nchash(path):
-    hashval = ''
-    m = NCDataHash(path)
-    try:
-        hashval = m.gethash()
-    except NotNetcdfFileError as e:
-        sys.stderr.write(str(e))
-        hashval = None
-    return hashval
-
 def _binhash(path, size, include_mtime):
     m = hashlib.new('md5')
     with io.open(path, mode="rb") as fd:
@@ -81,9 +70,7 @@ def hash(path, hashfn, size=one_hundred_megabytes):
     if hashfn not in supported_hashes:
         sys.stderr.write('\nUnsupported hash function {}, skipping {}\n'.format(hashfn, path))
     try:
-        if hashfn == 'nchash':
-            return _nchash(path)
-        elif hashfn == 'binhash':
+        if hashfn == 'binhash':
             return _binhash(path, one_hundred_megabytes, True)
         elif hashfn == 'binhash-nomtime':
             return _binhash(path, one_hundred_megabytes, False)