From 04c71e6e2bd23e4c11efb163feb1f46616c9f29b Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 14 Feb 2025 12:00:53 +0800 Subject: [PATCH] feat: calculate digest of the persistent cache piece to check the integrity of the metadata (#980) Signed-off-by: Gaius --- Cargo.lock | 4 +- Cargo.toml | 2 +- dragonfly-client-storage/src/metadata.rs | 29 +++++------- dragonfly-client-util/src/digest/mod.rs | 2 +- dragonfly-client/src/grpc/dfdaemon_upload.rs | 9 +++- .../src/resource/piece_downloader.rs | 45 ++++++++++++++----- 6 files changed, 55 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d08a8f2..ca86d59d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -923,9 +923,9 @@ dependencies = [ [[package]] name = "dragonfly-api" -version = "2.1.25" +version = "2.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "303e4462081b03a42316ec8569d9ce5fc6a220a4489d2029bf6120311a9dd2a8" +checksum = "eb145fdba5fa39c2506f2f85baa8157b8703add719b329e35ab4337cf5635ce7" dependencies = [ "prost 0.13.4", "prost-types", diff --git a/Cargo.toml b/Cargo.toml index b97ed874..7547f48e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.2.1 dragonfly-client-util = { path = "dragonfly-client-util", version = "0.2.11" } dragonfly-client-init = { path = "dragonfly-client-init", version = "0.2.11" } thiserror = "1.0" -dragonfly-api = "=2.1.25" +dragonfly-api = "=2.1.27" reqwest = { version = "0.12.4", features = [ "stream", "native-tls", diff --git a/dragonfly-client-storage/src/metadata.rs b/dragonfly-client-storage/src/metadata.rs index 89d4c48c..e1a4f7d7 100644 --- a/dragonfly-client-storage/src/metadata.rs +++ b/dragonfly-client-storage/src/metadata.rs @@ -303,25 +303,19 @@ impl Piece { } } - /// calculate_digest return the digest of the piece metadata, - /// which is different from the digest of the piece content. + /// calculate_digest return the digest of the piece metadata, including the piece number, + /// offset, length and content digest. The digest is used to check the integrity of the + /// piece metadata. pub fn calculate_digest(&self) -> String { let crc = Crc::>::new(&CRC_32_ISCSI); - let mut crc_digest = crc.digest(); + let mut digest = crc.digest(); + digest.update(&self.number.to_be_bytes()); + digest.update(&self.offset.to_be_bytes()); + digest.update(&self.length.to_be_bytes()); + digest.update(self.digest.as_bytes()); - crc_digest.update(&self.number.to_be_bytes()); - crc_digest.update(&self.offset.to_be_bytes()); - crc_digest.update(&self.length.to_be_bytes()); - crc_digest.update(self.digest.as_bytes()); - - if let Some(parent_id) = &self.parent_id { - crc_digest.update(parent_id.as_bytes()); - } - - let digest = - digest::Digest::new(digest::Algorithm::Crc32, crc_digest.finalize().to_string()); - - digest.to_string() + let encoded = digest.finalize().to_string(); + digest::Digest::new(digest::Algorithm::Crc32, encoded).to_string() } } @@ -957,12 +951,11 @@ mod tests { offset: 0, length: 1024, digest: "crc32:3810626145".to_string(), - parent_id: Some("d3c4e940ad06c47fc36ac67801e6f8e36cb4".to_string()), ..Default::default() }; let digest = piece.calculate_digest(); - assert_eq!(digest, "crc32:523852508"); + assert_eq!(digest, "crc32:3874114958"); } #[test] diff --git a/dragonfly-client-util/src/digest/mod.rs b/dragonfly-client-util/src/digest/mod.rs index 7dc6d524..7050beea 100644 --- a/dragonfly-client-util/src/digest/mod.rs +++ b/dragonfly-client-util/src/digest/mod.rs @@ -129,7 +129,7 @@ impl FromStr for Digest { } } -/// calculate_file_hash calculates the hash of a file. +/// calculate_file_digest calculates the digest of a file. #[instrument(skip_all)] pub fn calculate_file_digest(algorithm: Algorithm, path: &Path) -> ClientResult { let f = std::fs::File::open(path)?; diff --git a/dragonfly-client/src/grpc/dfdaemon_upload.rs b/dragonfly-client/src/grpc/dfdaemon_upload.rs index 43246dc7..f370e2ee 100644 --- a/dragonfly-client/src/grpc/dfdaemon_upload.rs +++ b/dragonfly-client/src/grpc/dfdaemon_upload.rs @@ -882,6 +882,8 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler { cost: None, created_at: None, }), + // Calculate the digest of the piece metadata, including the number, offset, length and + // content digest. The digest is used to verify the integrity of the piece metadata. digest: Some(piece.calculate_digest()), })) } @@ -1409,15 +1411,18 @@ impl DfdaemonUpload for DfdaemonUploadServerHandler { Ok(Response::new(DownloadPersistentCachePieceResponse { piece: Some(Piece { number: piece.number, - parent_id: piece.parent_id, + parent_id: piece.parent_id.clone(), offset: piece.offset, length: piece.length, - digest: piece.digest, + digest: piece.digest.clone(), content: Some(content), traffic_type: None, cost: None, created_at: None, }), + // Calculate the digest of the piece metadata, including the number, offset, length and + // content digest. The digest is used to verify the integrity of the piece metadata. + digest: Some(piece.calculate_digest()), })) } } diff --git a/dragonfly-client/src/resource/piece_downloader.rs b/dragonfly-client/src/resource/piece_downloader.rs index dee260b0..097dac83 100644 --- a/dragonfly-client/src/resource/piece_downloader.rs +++ b/dragonfly-client/src/resource/piece_downloader.rs @@ -121,29 +121,30 @@ impl Downloader for GRPCDownloader { return Err(Error::InvalidParameter); }; + let Some(content) = piece.content else { + return Err(Error::InvalidParameter); + }; + + // Calculate the digest of the piece metadata and compare it with the expected digest, + // it verifies the integrity of the piece metadata. let piece_metadata = metadata::Piece { number, length: piece.length, offset: piece.offset, digest: piece.digest.clone(), - parent_id: piece.parent_id.clone(), ..Default::default() }; - if let Some(expected_digest) = &response.digest { - let actual_digest = piece_metadata.calculate_digest(); - if expected_digest != &actual_digest { - return Err(Error::ValidationError(format!( - "checksum mismatch, expected: {}, actual: {}", - expected_digest, actual_digest - ))); + if let Some(expected_digest) = response.digest { + let digest = piece_metadata.calculate_digest(); + if expected_digest != digest { + return Err(Error::DigestMismatch( + expected_digest.to_string(), + digest.to_string(), + )); } } - let Some(content) = piece.content else { - return Err(Error::InvalidParameter); - }; - Ok((content, piece.offset, piece.digest)) } @@ -179,6 +180,26 @@ impl Downloader for GRPCDownloader { return Err(Error::InvalidParameter); }; + // Calculate the digest of the piece metadata and compare it with the expected digest, + // it verifies the integrity of the piece metadata. + let piece_metadata = metadata::Piece { + number, + length: piece.length, + offset: piece.offset, + digest: piece.digest.clone(), + ..Default::default() + }; + + if let Some(expected_digest) = response.digest { + let digest = piece_metadata.calculate_digest(); + if expected_digest != digest { + return Err(Error::DigestMismatch( + expected_digest.to_string(), + digest.to_string(), + )); + } + } + Ok((content, piece.offset, piece.digest)) } }