dedupe sample_hash

Author: chapmanjacobd

xklb/media/dedupe.py

@@ -378,8 +378,11 @@ def get_fs_duplicates(args) -> List[dict]:
     for path, hash in zip(sample_hash_paths, hash_results):
         for m in media:
             if m['path'] == path:
-                m['hash'] = hash
-                args.db["media"].upsert(m, pk=["path"], alter=True)  # save sample-hash back to db
+                if hash is None:
+                    del m
+                else:
+                    m['hash'] = hash
+                    args.db["media"].upsert(m, pk=["path"], alter=True)  # save sample-hash back to db
 
     sample_hash_groups = defaultdict(list)
     for m in media:

xklb/scripts/sample_hash.py

@@ -29,7 +29,11 @@ def threadpool_read(path, segments, chunk_size, max_workers=10):
 
 
 def sample_hash_file(path, threads=1, gap=0.1, chunk_size=None):
-    file_stats = Path(path).stat()
+    try:
+        file_stats = Path(path).stat()
+    except FileNotFoundError:
+        return None
+
     disk_usage = (
         file_stats.st_blocks * 512
     )  # https://github.com/python/cpython/blob/main/Doc/library/os.rst#files-and-directories