feat: calculate digest of the persistent cache piece to check the integrity of the metadata (#980)

Signed-off-by: Gaius <[email protected]>

Author: gaius-qi

Cargo.lock

@@ -30,7 +30,7 @@ dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.2.1
 dragonfly-client-util = { path = "dragonfly-client-util", version = "0.2.11" }
 dragonfly-client-init = { path = "dragonfly-client-init", version = "0.2.11" }
 thiserror = "1.0"
-dragonfly-api = "=2.1.25"
+dragonfly-api = "=2.1.27"
 reqwest = { version = "0.12.4", features = [
     "stream",
     "native-tls",

Cargo.toml

@@ -303,25 +303,19 @@ impl Piece {
         }
     }
 
-    /// calculate_digest return the digest of the piece metadata,
-    /// which is different from the digest of the piece content.
+    /// calculate_digest return the digest of the piece metadata, including the piece number,
+    /// offset, length and content digest. The digest is used to check the integrity of the
+    /// piece metadata.
     pub fn calculate_digest(&self) -> String {
         let crc = Crc::<u32, Table<16>>::new(&CRC_32_ISCSI);
-        let mut crc_digest = crc.digest();
+        let mut digest = crc.digest();
+        digest.update(&self.number.to_be_bytes());
+        digest.update(&self.offset.to_be_bytes());
+        digest.update(&self.length.to_be_bytes());
+        digest.update(self.digest.as_bytes());
 
-        crc_digest.update(&self.number.to_be_bytes());
-        crc_digest.update(&self.offset.to_be_bytes());
-        crc_digest.update(&self.length.to_be_bytes());
-        crc_digest.update(self.digest.as_bytes());
-
-        if let Some(parent_id) = &self.parent_id {
-            crc_digest.update(parent_id.as_bytes());
-        }
-
-        let digest =
-            digest::Digest::new(digest::Algorithm::Crc32, crc_digest.finalize().to_string());
-
-        digest.to_string()
+        let encoded = digest.finalize().to_string();
+        digest::Digest::new(digest::Algorithm::Crc32, encoded).to_string()
     }
 }
 
@@ -957,12 +951,11 @@ mod tests {
             offset: 0,
             length: 1024,
             digest: "crc32:3810626145".to_string(),
-            parent_id: Some("d3c4e940ad06c47fc36ac67801e6f8e36cb4".to_string()),
             ..Default::default()
         };
 
         let digest = piece.calculate_digest();
-        assert_eq!(digest, "crc32:523852508");
+        assert_eq!(digest, "crc32:3874114958");
     }
 
     #[test]

dragonfly-client-storage/src/metadata.rs

@@ -129,7 +129,7 @@ impl FromStr for Digest {
     }
 }
 
-/// calculate_file_hash calculates the hash of a file.
+/// calculate_file_digest calculates the digest of a file.
 #[instrument(skip_all)]
 pub fn calculate_file_digest(algorithm: Algorithm, path: &Path) -> ClientResult<Digest> {
     let f = std::fs::File::open(path)?;

dragonfly-client-util/src/digest/mod.rs

@@ -882,6 +882,8 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
                 cost: None,
                 created_at: None,
             }),
+            // Calculate the digest of the piece metadata, including the number, offset, length and
+            // content digest. The digest is used to verify the integrity of the piece metadata.
             digest: Some(piece.calculate_digest()),
         }))
     }
@@ -1409,15 +1411,18 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler {
         Ok(Response::new(DownloadPersistentCachePieceResponse {
             piece: Some(Piece {
                 number: piece.number,
-                parent_id: piece.parent_id,
+                parent_id: piece.parent_id.clone(),
                 offset: piece.offset,
                 length: piece.length,
-                digest: piece.digest,
+                digest: piece.digest.clone(),
                 content: Some(content),
                 traffic_type: None,
                 cost: None,
                 created_at: None,
             }),
+            // Calculate the digest of the piece metadata, including the number, offset, length and
+            // content digest. The digest is used to verify the integrity of the piece metadata.
+            digest: Some(piece.calculate_digest()),
         }))
     }
 }

dragonfly-client/src/grpc/dfdaemon_upload.rs

@@ -121,29 +121,30 @@ impl Downloader for GRPCDownloader {
             return Err(Error::InvalidParameter);
         };
 
+        let Some(content) = piece.content else {
+            return Err(Error::InvalidParameter);
+        };
+
+        // Calculate the digest of the piece metadata and compare it with the expected digest,
+        // it verifies the integrity of the piece metadata.
         let piece_metadata = metadata::Piece {
             number,
             length: piece.length,
             offset: piece.offset,
             digest: piece.digest.clone(),
-            parent_id: piece.parent_id.clone(),
             ..Default::default()
         };
 
-        if let Some(expected_digest) = &response.digest {
-            let actual_digest = piece_metadata.calculate_digest();
-            if expected_digest != &actual_digest {
-                return Err(Error::ValidationError(format!(
-                    "checksum mismatch, expected: {}, actual: {}",
-                    expected_digest, actual_digest
-                )));
+        if let Some(expected_digest) = response.digest {
+            let digest = piece_metadata.calculate_digest();
+            if expected_digest != digest {
+                return Err(Error::DigestMismatch(
+                    expected_digest.to_string(),
+                    digest.to_string(),
+                ));
             }
         }
 
-        let Some(content) = piece.content else {
-            return Err(Error::InvalidParameter);
-        };
-
         Ok((content, piece.offset, piece.digest))
     }
 
@@ -179,6 +180,26 @@ impl Downloader for GRPCDownloader {
             return Err(Error::InvalidParameter);
         };
 
+        // Calculate the digest of the piece metadata and compare it with the expected digest,
+        // it verifies the integrity of the piece metadata.
+        let piece_metadata = metadata::Piece {
+            number,
+            length: piece.length,
+            offset: piece.offset,
+            digest: piece.digest.clone(),
+            ..Default::default()
+        };
+
+        if let Some(expected_digest) = response.digest {
+            let digest = piece_metadata.calculate_digest();
+            if expected_digest != digest {
+                return Err(Error::DigestMismatch(
+                    expected_digest.to_string(),
+                    digest.to_string(),
+                ));
+            }
+        }
+
         Ok((content, piece.offset, piece.digest))
     }
 }