Use libdnf.utils.checksum_{check,value}

libdnf has the canonical implementation of checksum handling. We aim to
replace all use of dnf.yum.misc.checksum() with this. In doing so, this
fixes installing previously downloaded and transcoded rpms to support

rpm-software-management/librepo#222

This also has some minor performance benefits: librepo's checksum
handling employs caching of previously downloaded files via extended
attributes. This works for ordinary rpms in the dnf cache, but does not
work (yet) for rpm paths specified on the command line due to
rpm-software-management/librepo#233. That
issue is pretty minor, and the fix ends up in libdnf later.

The previous implementation maps all runtime errors to MiscError. We do
this still by taking the libdnf.error.Error class (defined in SWIG) and
map it directly back to the Python exception as before.

Author: malmond77

dnf/package.py

@@ -30,6 +30,8 @@
 import dnf.rpm
 import dnf.yum.misc
 import hawkey
+import libdnf.error
+import libdnf.utils
 import logging
 import os
 import rpm
@@ -56,7 +58,10 @@ def _chksum(self):
             return self._priv_chksum
         if self._from_cmdline:
             chksum_type = dnf.yum.misc.get_default_chksum_type()
-            chksum_val = dnf.yum.misc.checksum(chksum_type, self.location)
+            try:
+                chksum_val = libdnf.utils.checksum_value(chksum_type, self.location)
+            except libdnf.error.Error as e:
+                raise MiscError(e)
             return (hawkey.chksum_type(chksum_type),
                     binascii.unhexlify(chksum_val))
         return super(Package, self).chksum
@@ -330,10 +335,7 @@ def verifyLocalPkg(self):
         if self._from_cmdline:
             return True # local package always verifies against itself
         (chksum_type, chksum) = self.returnIdSum()
-        real_sum = dnf.yum.misc.checksum(chksum_type, self.localPkg(),
-                                         datasize=self._size)
-        if real_sum != chksum:
-            logger.debug(_('%s: %s check failed: %s vs %s'),
-                         self, chksum_type, real_sum, chksum)
-            return False
-        return True
+        try:
+            return libdnf.utils.checksum_check(chksum_type, self.localPkg(), chksum)
+        except libdnf.error.Error as e:
+            raise MiscError(e)

dnf/yum/misc.py

@@ -41,7 +41,6 @@
 import shutil
 import tempfile
 
-_available_checksums = set(['md5', 'sha1', 'sha256', 'sha384', 'sha512'])
 _default_checksums = ['sha256']
 
 
@@ -68,119 +67,9 @@ def re_full_search_needed(s):
             return True
     return False
 
-
-class Checksums(object):
-    """ Generate checksum(s), on given pieces of data. Producing the
-        Length and the result(s) when complete. """
-
-    def __init__(self, checksums=None, ignore_missing=False, ignore_none=False):
-        if checksums is None:
-            checksums = _default_checksums
-        self._sumalgos = []
-        self._sumtypes = []
-        self._len = 0
-
-        done = set()
-        for sumtype in checksums:
-            if sumtype == 'sha':
-                sumtype = 'sha1'
-            if sumtype in done:
-                continue
-
-            if sumtype in _available_checksums:
-                sumalgo = hashlib.new(sumtype)
-            elif ignore_missing:
-                continue
-            else:
-                raise MiscError('Error Checksumming, bad checksum type %s' %
-                                sumtype)
-            done.add(sumtype)
-            self._sumtypes.append(sumtype)
-            self._sumalgos.append(sumalgo)
-        if not done and not ignore_none:
-            raise MiscError('Error Checksumming, no valid checksum type')
-
-    def __len__(self):
-        return self._len
-
-    # Note that len(x) is assert limited to INT_MAX, which is 2GB on i686.
-    length = property(fget=lambda self: self._len)
-
-    def update(self, data):
-        self._len += len(data)
-        for sumalgo in self._sumalgos:
-            data = data.encode('utf-8') if isinstance(data, unicode) else data
-            sumalgo.update(data)
-
-    def read(self, fo, size=2**16):
-        data = fo.read(size)
-        self.update(data)
-        return data
-
-    def hexdigests(self):
-        ret = {}
-        for sumtype, sumdata in zip(self._sumtypes, self._sumalgos):
-            ret[sumtype] = sumdata.hexdigest()
-        return ret
-
-    def hexdigest(self, checksum=None):
-        if checksum is None:
-            if not self._sumtypes:
-                return None
-            checksum = self._sumtypes[0]
-        if checksum == 'sha':
-            checksum = 'sha1'
-        return self.hexdigests()[checksum]
-
-    def digests(self):
-        ret = {}
-        for sumtype, sumdata in zip(self._sumtypes, self._sumalgos):
-            ret[sumtype] = sumdata.digest()
-        return ret
-
-    def digest(self, checksum=None):
-        if checksum is None:
-            if not self._sumtypes:
-                return None
-            checksum = self._sumtypes[0]
-        if checksum == 'sha':
-            checksum = 'sha1'
-        return self.digests()[checksum]
-
 def get_default_chksum_type():
     return _default_checksums[0]
 
-def checksum(sumtype, file, CHUNK=2**16, datasize=None):
-    """takes filename, hand back Checksum of it
-       sumtype = md5 or sha/sha1/sha256/sha512 (note sha == sha1)
-       filename = /path/to/file
-       CHUNK=65536 by default"""
-
-    # chunking brazenly lifted from Ryan Tomayko
-
-    if isinstance(file, basestring):
-        try:
-            with open(file, 'rb', CHUNK) as fo:
-                return checksum(sumtype, fo, CHUNK, datasize)
-        except (IOError, OSError):
-            raise MiscError('Error opening file for checksum: %s' % file)
-
-    try:
-        # assumes file is a file-like-object
-        data = Checksums([sumtype])
-        while data.read(file, CHUNK):
-            if datasize is not None and data.length > datasize:
-                break
-
-        # This screws up the length, but that shouldn't matter. We only care
-        # if this checksum == what we expect.
-        if datasize is not None and datasize != data.length:
-            return '!%u!%s' % (datasize, data.hexdigest(sumtype))
-
-        return data.hexdigest(sumtype)
-    except (IOError, OSError) as e:
-        raise MiscError('Error reading file for checksum: %s' % file)
-
 class GenericHolder(object):
     """Generic Holder class used to hold other objects of known types
        It exists purely to be able to do object.somestuff, object.someotherstuff