Optionally disable checksumming (issue #118) (#207)

Author: pauldreik

Makefile.am

@@ -20,7 +20,9 @@ TESTS=testcases/largefilesupport.sh \
       testcases/checksum_options.sh \
       testcases/md5collisions.sh \
       testcases/sha1collisions.sh \
-      testcases/checksum_buffersize.sh
+      testcases/checksum_buffersize.sh \
+      testcases/verify_nochecksum.sh
+
 
 AUXFILES=testcases/common_funcs.sh \
          testcases/md5collisions/letter_of_rec.ps \

NEWS

@@ -1,3 +1,6 @@
+next
+optionally disable the checksum step by giving -checksum none
+optionally show progress
 1.7.0
 requires a C++17 capable compiler.
 new fast non-cryptographic hash xxh

README.md

@@ -89,7 +89,7 @@ Rdfind uses the following algorithm. If N is the number of files to search throu
 8. Remove files from list that have the same size but different first bytes.
 9. Sort on device and inode(speeds up file reading). Read a few bytes from the end of each file (last bytes).
 10. Remove files from list that have the same size but different last bytes.
-11. Sort on device and inode(speeds up file reading). Perform a checksum calculation for each file.
+11. Sort on device and inode(speeds up file reading). Perform a checksum calculation for each file (unless disabled with -checksum none).
 12. Only keep files on the list with the same size and checksum. These are duplicates.
 13. Sort list on size, priority number, and depth. The first file for every set of duplicates is considered to be the original.
 14. If flag ”-makeresultsfile true”, then print results file (default).

inofficial_cmake/CMakeLists.txt

@@ -66,6 +66,7 @@ set(testscripts
     testcases/verify_dryrun_option.sh
     testcases/verify_filesize_option.sh
     testcases/verify_maxfilesize_option.sh
+    testcases/verify_nochecksum.sh
     testcases/verify_ranking.sh
     testcases/verify_size_savings.sh)
 

rdfind.1

@@ -76,11 +76,14 @@ Follow symlinks. Default is false.
 Removes items found which have identical inode and device ID. Default
 is true.
 .TP
-.BR \-checksum " " \fImd5\fR|\fIsha1\fR|\fIsha256\fR|\fIsha512|\fIxxh128\fR
+.BR \-checksum " " \fInone\fR|\fImd5\fR|\fIsha1\fR|\fIsha256\fR|\fIsha512|\fIxxh128\fR
 What type of checksum to be used: md5, sha1, sha256, sha512 or xxh128. The default is
 sha1 since version 1.4.0. xxh128 is a very fast checksum, but not of cryptographic
 quality. xxh support is optional and requires that rdfind was configured with
 --with-xxhash. In case xxh is used but there is no support, an error is returned.
+Checksum none can be used to skip checksumming altogether. \fBThis is not recommended!\fR
+In case files of the same size have contents that differ it is likely they are falsely
+consider duplicates, leading to file removal (depending on other options).
 .TP
 .BR \-buffersize " " \fIN\fR
 Chunksize in bytes when calculating the checksum

rdfind.cc

@@ -64,7 +64,7 @@ usage()
     << " -followsymlinks    true |(false) follow symlinks\n"
     << " -removeidentinode (true)| false  ignore files with nonunique "
        "device and inode\n"
-    << " -checksum           md5 |(sha1)| sha256 | sha512 | xxh128\n"
+    << " -checksum          none | md5 |(sha1)| sha256 | sha512 | xxh128\n"
     << indent << "checksum type\n"
     << indent << "xxh128 is very fast, but is noncryptographic.\n"
     << " -buffersize N\n"
@@ -116,6 +116,7 @@ struct Options
   bool usesha256 = false;    // use sha256 checksum to check for similarity
   bool usesha512 = false;    // use sha512 checksum to check for similarity
   bool usexxh128 = false;    // use xxh128 checksum to check for similarity
+  bool nochecksum = false;   // skip using checksumming (unsafe!)
   bool deterministic = true; // be independent of filesystem order
   bool showprogress = false; // show progress while reading file contents
   std::size_t buffersize = 1 << 20; // chunksize to use when reading files
@@ -196,8 +197,12 @@ parseOptions(Parser& parser)
                      "reconfigure and rebuild '--with-xxhash'\n";
         std::exit(EXIT_FAILURE);
 #endif
+      } else if (parser.parsed_string_is("none")) {
+        std::cout
+          << "DANGER! -checksum none given, will skip the checksumming stage\n";
+        o.nochecksum = true;
       } else {
-        std::cerr << "expected md5/sha1/sha256/sha512/xxh128, not \""
+        std::cerr << "expected none/md5/sha1/sha256/sha512/xxh128, not \""
                   << parser.get_parsed_string() << "\"\n";
         std::exit(EXIT_FAILURE);
       }
@@ -273,8 +278,9 @@ parseOptions(Parser& parser)
 
   // done with parsing of options. remaining arguments are files and dirs.
 
-  // decide what checksum to use - if no checksum is set, force sha1!
-  if (!o.usemd5 && !o.usesha1 && !o.usesha256 && !o.usesha512 && !o.usexxh128) {
+  // decide what checksum to use, default to sha1
+  if (!o.usemd5 && !o.usesha1 && !o.usesha256 && !o.usesha512 && !o.usexxh128 &&
+      !o.nochecksum) {
     o.usesha1 = true;
   }
   return o;

testcases/verify_nochecksum.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+# Ensures the skip checksumming stage works as intended.
+#
+
+set -e
+. "$(dirname "$0")/common_funcs.sh"
+
+# make a file which is longer than "first bytes" and "last bytes" together,
+# so we can make two files that differ only in the middle and will
+# need checksumming to see they are different.
+makefiles() {
+  FIRSTBYTES=1000
+  MIDDLEBYTES=1000
+  LASTBYTES=1000
+  for f in a b; do
+    (
+      head -c$FIRSTBYTES </dev/zero
+      head -c$MIDDLEBYTES </dev/urandom
+      head -c$LASTBYTES </dev/zero
+    ) >$f
+  done
+}
+
+reset_teststate
+makefiles
+
+# with no checksum, we should falsely believe the files are equal
+$rdfind -checksum none a* b* \
+  | grep "files that are not unique" >output.log
+
+verify [ "$(cat output.log)" = "It seems like you have 2 files that are not unique" ]
+
+# with checksumming (the default) the files should not be considered equal.
+$rdfind -checksum sha1 a* b* \
+  | grep "files that are not unique" >output.log
+
+verify [ "$(cat output.log)" = "It seems like you have 0 files that are not unique" ]
+
+dbgecho "all is good for the checksum=none test!"