dedupe sample_compare

Author: chapmanjacobd

xklb/media/dedupe.py

@@ -407,7 +407,8 @@ def get_fs_duplicates(args) -> List[dict]:
             hash_results = list(pool.map(sample_compare.full_hash_file, check_paths))
         hash_groups = defaultdict(list)
         for path, hash in zip(check_paths, hash_results):
-            hash_groups[hash].append(path)
+            if hash is not None:
+                hash_groups[hash].append(path)
         for paths in hash_groups.values():
             if len(paths) > 1:
                 keep_path = paths[0]

xklb/scripts/sample_compare.py

@@ -11,9 +11,12 @@
 def full_hash_file(path):
     sha256_hash = hashlib.sha256()
 
-    with open(path, "rb") as file:
-        for byte_block in iter(lambda: file.read(1048576), b""):
-            sha256_hash.update(byte_block)
+    try:
+        with open(path, "rb") as file:
+            for byte_block in iter(lambda: file.read(1048576), b""):
+                sha256_hash.update(byte_block)
+    except FileNotFoundError:
+        return None
 
     return sha256_hash.hexdigest()