diff --git a/Cargo.lock b/Cargo.lock index daff5618..423ec791 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -923,7 +923,7 @@ dependencies = [ [[package]] name = "dragonfly-client" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "blake3", @@ -996,7 +996,7 @@ dependencies = [ [[package]] name = "dragonfly-client-backend" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dragonfly-api", "dragonfly-client-core", @@ -1027,7 +1027,7 @@ dependencies = [ [[package]] name = "dragonfly-client-config" -version = "0.2.1" +version = "0.2.2" dependencies = [ "bytesize", "bytesize-serde", @@ -1053,7 +1053,7 @@ dependencies = [ [[package]] name = "dragonfly-client-core" -version = "0.2.1" +version = "0.2.2" dependencies = [ "headers 0.4.0", "hyper 1.5.2", @@ -1072,7 +1072,7 @@ dependencies = [ [[package]] name = "dragonfly-client-init" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "clap", @@ -1090,13 +1090,12 @@ dependencies = [ [[package]] name = "dragonfly-client-storage" -version = "0.2.1" +version = "0.2.2" dependencies = [ "base16ct", "bincode", "chrono", "crc32c", - "crc32fast", "dragonfly-api", "dragonfly-client-config", "dragonfly-client-core", @@ -1116,7 +1115,7 @@ dependencies = [ [[package]] name = "dragonfly-client-util" -version = "0.2.1" +version = "0.2.2" dependencies = [ "base16ct", "base64 0.22.1", @@ -1506,7 +1505,7 @@ dependencies = [ [[package]] name = "hdfs" -version = "0.2.1" +version = "0.2.2" dependencies = [ "dragonfly-client-backend", "dragonfly-client-core", diff --git a/Cargo.toml b/Cargo.toml index 9c255632..5318735b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.2.1" +version = "0.2.2" authors = ["The Dragonfly Developers"] homepage = "https://d7y.io/" repository = "https://github.com/dragonflyoss/client.git" @@ -22,13 +22,13 @@ readme = "README.md" edition = "2021" [workspace.dependencies] -dragonfly-client = { path = "dragonfly-client", version = "0.2.1" } -dragonfly-client-core = { path = "dragonfly-client-core", version = "0.2.1" } -dragonfly-client-config = { path = "dragonfly-client-config", version = "0.2.1" } -dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.2.1" } -dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.2.1" } -dragonfly-client-util = { path = "dragonfly-client-util", version = "0.2.1" } -dragonfly-client-init = { path = "dragonfly-client-init", version = "0.2.1" } +dragonfly-client = { path = "dragonfly-client", version = "0.2.2" } +dragonfly-client-core = { path = "dragonfly-client-core", version = "0.2.2" } +dragonfly-client-config = { path = "dragonfly-client-config", version = "0.2.2" } +dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.2.2" } +dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.2.2" } +dragonfly-client-util = { path = "dragonfly-client-util", version = "0.2.2" } +dragonfly-client-init = { path = "dragonfly-client-init", version = "0.2.2" } thiserror = "1.0" dragonfly-api = "=2.1.3" reqwest = { version = "0.12.4", features = [ @@ -61,7 +61,6 @@ rustls-pki-types = "1.10.1" rustls-pemfile = "2.2.0" sha2 = "0.10" blake3 = "1.5.5" -crc32fast = "1.4.2" crc32c = "0.6.8" uuid = { version = "1.11", features = ["v4"] } hex = "0.4" diff --git a/dragonfly-client-storage/Cargo.toml b/dragonfly-client-storage/Cargo.toml index 6dc7972d..cd62b45b 100644 --- a/dragonfly-client-storage/Cargo.toml +++ b/dragonfly-client-storage/Cargo.toml @@ -23,7 +23,6 @@ prost-wkt-types.workspace = true tokio.workspace = true tokio-util.workspace = true sha2.workspace = true -crc32fast.workspace = true crc32c.workspace = true base16ct.workspace = true num_cpus = "1.0" diff --git a/dragonfly-client-storage/src/content.rs b/dragonfly-client-storage/src/content.rs index 006710da..ace610ca 100644 --- a/dragonfly-client-storage/src/content.rs +++ b/dragonfly-client-storage/src/content.rs @@ -24,7 +24,6 @@ use tokio::fs::{self, File, OpenOptions}; use tokio::io::{ self, AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufReader, BufWriter, SeekFrom, }; -use tokio_util::io::InspectReader; use tracing::{error, info, instrument, warn}; /// DEFAULT_CONTENT_DIR is the default directory for store content. @@ -362,56 +361,6 @@ impl Content { }) } - /// write_piece_with_crc32_iso3309 writes the piece to the content with crc32 iso3309, there is - /// no hardware acceleration. - #[instrument(skip_all)] - pub async fn write_piece_with_crc32_iso3309( - &self, - task_id: &str, - offset: u64, - reader: &mut R, - ) -> Result { - // Use a buffer to read the piece. - let reader = BufReader::with_capacity(self.config.storage.write_buffer_size, reader); - - // Crc32 is used to calculate the hash of the piece. - let mut hasher = crc32fast::Hasher::new(); - - // InspectReader is used to calculate the hash of the piece. - let mut tee = InspectReader::new(reader, |bytes| { - hasher.update(bytes); - }); - - // Open the file and seek to the offset. - let task_path = self.create_or_get_task_path(task_id).await?; - let mut f = OpenOptions::new() - .create(true) - .truncate(false) - .write(true) - .open(task_path.as_path()) - .await - .inspect_err(|err| { - error!("open {:?} failed: {}", task_path, err); - })?; - - f.seek(SeekFrom::Start(offset)).await.inspect_err(|err| { - error!("seek {:?} failed: {}", task_path, err); - })?; - - // Copy the piece to the file. - let mut writer = BufWriter::with_capacity(self.config.storage.write_buffer_size, f); - let length = io::copy(&mut tee, &mut writer).await.inspect_err(|err| { - error!("copy {:?} failed: {}", task_path, err); - })?; - - // Calculate the hash of the piece. - let hash = hasher.finalize(); - Ok(WritePieceResponse { - length, - hash: base16ct::lower::encode_string(&hash.to_be_bytes()), - }) - } - /// get_task_path returns the task path by task id. #[instrument(skip_all)] fn get_task_path(&self, task_id: &str) -> PathBuf { diff --git a/dragonfly-client-storage/src/lib.rs b/dragonfly-client-storage/src/lib.rs index b1aecc49..381b1677 100644 --- a/dragonfly-client-storage/src/lib.rs +++ b/dragonfly-client-storage/src/lib.rs @@ -16,10 +16,7 @@ use dragonfly_api::common::v2::Range; use dragonfly_client_config::dfdaemon::Config; -use dragonfly_client_core::{ - error::{ErrorType, OrErr}, - Error, Result, -}; +use dragonfly_client_core::{Error, Result}; use dragonfly_client_util::digest::{Algorithm, Digest}; use reqwest::header::HeaderMap; use std::path::Path; @@ -337,17 +334,10 @@ impl Storage { parent_id: &str, reader: &mut R, ) -> Result { - let digest: Digest = expected_digest.parse().or_err(ErrorType::ParseError)?; - let response = if digest.is_crc32_iso3309() { - // Compatible with the old version. - self.content - .write_piece_with_crc32_iso3309(task_id, offset, reader) - .await? - } else { - self.content - .write_piece_with_crc32_castagnoli(task_id, offset, reader) - .await? - }; + let response = self + .content + .write_piece_with_crc32_castagnoli(task_id, offset, reader) + .await?; let length = response.length; let digest = Digest::new(Algorithm::Crc32, response.hash); diff --git a/dragonfly-client-util/Cargo.toml b/dragonfly-client-util/Cargo.toml index 5fbae839..97a76cc9 100644 --- a/dragonfly-client-util/Cargo.toml +++ b/dragonfly-client-util/Cargo.toml @@ -27,10 +27,10 @@ uuid.workspace = true hex.workspace = true openssl.workspace = true blake3.workspace = true -crc32fast.workspace = true crc32c.workspace = true base16ct.workspace = true base64 = "0.22.1" +crc32fast = "1.4.2" [dev-dependencies] tempfile.workspace = true diff --git a/dragonfly-client-util/src/digest/mod.rs b/dragonfly-client-util/src/digest/mod.rs index 123a59fa..1e3a6423 100644 --- a/dragonfly-client-util/src/digest/mod.rs +++ b/dragonfly-client-util/src/digest/mod.rs @@ -95,16 +95,6 @@ impl Digest { pub fn encoded(&self) -> &str { &self.encoded } - - // is_crc32_iso3309 checks if the crc32 encoded string uses the ISO 3309 polynomial. - pub fn is_crc32_iso3309(&self) -> bool { - self.algorithm == Algorithm::Crc32 && self.encoded.len() == 8 - } - - // is_crc32_castagnoli checks if the crc32 encoded string uses the Castagnoli polynomial. - pub fn is_crc32_castagnoli(&self) -> bool { - self.algorithm == Algorithm::Crc32 && self.encoded.len() == 10 - } } /// Digest implements the Display. @@ -235,26 +225,4 @@ mod tests { calculate_file_hash(Algorithm::Crc32, path).expect("failed to calculate Sha512 hash"); assert_eq!(digest.encoded(), expected_crc32); } - - #[test] - fn test_is_crc32_iso3309() { - let mut hasher = crc32fast::Hasher::new(); - hasher.update(b"test"); - let hash = hasher.finalize(); - - let digest = Digest::new( - Algorithm::Crc32, - base16ct::lower::encode_string(&hash.to_be_bytes()), - ); - assert!(digest.is_crc32_iso3309()); - } - - #[test] - fn test_is_crc32_castagnoli() { - let crc = crc32c::crc32c(b"test"); - let encoded = crc.to_string(); - - let digest = Digest::new(Algorithm::Crc32, encoded); - assert!(digest.is_crc32_castagnoli()); - } }