Skip to content

Commit d50838d

Browse files
weisdweisd
committed
fix(s3s-fs):add checksum_crc64nvme
1 parent b5fcbc4 commit d50838d

File tree

4 files changed

+96
-49
lines changed

4 files changed

+96
-49
lines changed

‎crates/s3s-e2e/src/basic.rs‎

Lines changed: 71 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ use crate::utils::*;
33

44
use std::sync::Arc;
55

6-
use base64::Engine as _;
76
use s3s_test::Result;
87
use s3s_test::TestFixture;
98
use s3s_test::TestSuite;
@@ -15,8 +14,6 @@ use aws_sdk_s3::primitives::SdkBody;
1514
use bytes::Bytes;
1615
use futures::StreamExt as _;
1716
use http_body_util::StreamBody;
18-
use sha2::Digest as _;
19-
use sha2::Sha256;
2017

2118
pub fn register(tcx: &mut TestContext) {
2219
case!(tcx, Basic, Essential, test_list_buckets);
@@ -393,56 +390,85 @@ impl Put {
393390
let bucket = self.bucket.as_str();
394391
let key = "with-checksum-trailer";
395392

396-
let body = {
397-
let bytes = Bytes::from_static(&[b'a'; 1024]);
398-
399-
let stream = futures::stream::repeat_with(move || {
400-
let frame = http_body::Frame::data(bytes.clone());
401-
Ok::<_, std::io::Error>(frame)
402-
});
403-
404-
let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
405-
ByteStream::new(SdkBody::from_body_1_x(body))
406-
};
407-
408-
let put_resp = s3
409-
.put_object()
410-
.bucket(bucket)
411-
.key(key)
412-
.checksum_algorithm(ChecksumAlgorithm::Crc32)
413-
.body(body)
414-
.send()
415-
.await?;
416-
417-
let put_crc32 = put_resp
418-
.checksum_crc32()
419-
.expect("PUT should return checksum when checksum_algorithm is used");
420-
421-
let resp = s3
422-
.get_object()
423-
.bucket(bucket)
424-
.key(key)
425-
.checksum_mode(ChecksumMode::Enabled)
426-
.send()
427-
.await?;
428-
429-
let get_crc32 = resp
430-
.checksum_crc32()
431-
.expect("GET should return checksum when checksum_mode is enabled and full object is returned")
432-
.to_owned();
393+
for checksum_algorithm in [
394+
ChecksumAlgorithm::Crc32,
395+
ChecksumAlgorithm::Crc32C,
396+
ChecksumAlgorithm::Sha1,
397+
ChecksumAlgorithm::Sha256,
398+
ChecksumAlgorithm::Crc64Nvme,
399+
] {
400+
let body = {
401+
let bytes = Bytes::from_static(&[b'a'; 1024]);
402+
403+
let stream = futures::stream::repeat_with(move || {
404+
let frame = http_body::Frame::data(bytes.clone());
405+
Ok::<_, std::io::Error>(frame)
406+
});
407+
408+
let body = WithSizeHint::new(StreamBody::new(stream.take(70)), 70 * 1024);
409+
ByteStream::new(SdkBody::from_body_1_x(body))
410+
};
411+
412+
let put_resp = s3
413+
.put_object()
414+
.bucket(bucket)
415+
.key(key)
416+
.checksum_algorithm(checksum_algorithm.clone())
417+
.body(body)
418+
.send()
419+
.await?;
433420

434-
let body = resp.body.collect().await?;
435-
let body = String::from_utf8(body.to_vec())?;
436-
assert_eq!(body, "a".repeat(70 * 1024));
421+
let put_resp_checksum = match checksum_algorithm {
422+
ChecksumAlgorithm::Crc32 => put_resp
423+
.checksum_crc32()
424+
.expect("PUT should return checksum when checksum_algorithm is used"),
425+
ChecksumAlgorithm::Crc32C => put_resp
426+
.checksum_crc32_c()
427+
.expect("PUT should return checksum when checksum_algorithm is used"),
428+
ChecksumAlgorithm::Sha1 => put_resp
429+
.checksum_sha1()
430+
.expect("PUT should return checksum when checksum_algorithm is used"),
431+
ChecksumAlgorithm::Sha256 => put_resp
432+
.checksum_sha256()
433+
.expect("PUT should return checksum when checksum_algorithm is used"),
434+
ChecksumAlgorithm::Crc64Nvme => put_resp
435+
.checksum_crc64_nvme()
436+
.expect("PUT should return checksum when checksum_algorithm is used"),
437+
_ => panic!("Unsupported checksum algorithm"),
438+
};
439+
440+
let mut resp = s3
441+
.get_object()
442+
.bucket(bucket)
443+
.key(key)
444+
.checksum_mode(ChecksumMode::Enabled)
445+
.send()
446+
.await?;
437447

438-
assert_eq!(get_crc32, put_crc32);
448+
let body = std::mem::replace(&mut resp.body, ByteStream::new(SdkBody::empty()))
449+
.collect()
450+
.await?;
451+
// let body = resp.body.collect().await?;
452+
let body = String::from_utf8(body.to_vec())?;
453+
assert_eq!(body, "a".repeat(70 * 1024));
454+
455+
let get_resp_checksum = match checksum_algorithm {
456+
ChecksumAlgorithm::Crc32 => resp.checksum_crc32(),
457+
ChecksumAlgorithm::Crc32C => resp.checksum_crc32_c(),
458+
ChecksumAlgorithm::Sha1 => resp.checksum_sha1(),
459+
ChecksumAlgorithm::Sha256 => resp.checksum_sha256(),
460+
ChecksumAlgorithm::Crc64Nvme => resp.checksum_crc64_nvme(),
461+
_ => panic!("Unsupported checksum algorithm"),
462+
};
463+
464+
assert_eq!(get_resp_checksum, Some(put_resp_checksum));
465+
}
439466

440467
Ok(())
441468
}
442469

443470
async fn test_put_object_with_content_checksums(self: Arc<Self>) -> Result {
444471
use base64::Engine;
445-
use sha2::{Digest, Sha256};
446472

447473
let s3 = &self.s3;
448474
let bucket = self.bucket.as_str();

‎crates/s3s-fs/src/checksum.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ pub fn modify_internal_info(info: &mut serde_json::Map<String, serde_json::Value
1515
if let Some(checksum_sha256) = &checksum.checksum_sha256 {
1616
info.insert("checksum_sha256".to_owned(), serde_json::Value::String(checksum_sha256.clone()));
1717
}
18+
19+
if let Some(checksum_crc64nvme) = &checksum.checksum_crc64nvme {
20+
info.insert("checksum_crc64nvme".to_owned(), serde_json::Value::String(checksum_crc64nvme.clone()));
21+
}
1822
}
1923

2024
pub fn from_internal_info(info: &InternalInfo) -> s3s::dto::Checksum {
@@ -31,5 +35,9 @@ pub fn from_internal_info(info: &InternalInfo) -> s3s::dto::Checksum {
3135
if let Some(checksum_sha256) = info.get("checksum_sha256") {
3236
ans.checksum_sha256 = Some(checksum_sha256.as_str().unwrap().to_owned());
3337
}
38+
39+
if let Some(checksum_crc64nvme) = info.get("checksum_crc64nvme") {
40+
ans.checksum_crc64nvme = Some(checksum_crc64nvme.as_str().unwrap().to_owned());
41+
}
3442
ans
3543
}

‎crates/s3s-fs/src/s3.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ impl S3 for FileSystem {
255255
checksum_crc32c: checksum.checksum_crc32c,
256256
checksum_sha1: checksum.checksum_sha1,
257257
checksum_sha256: checksum.checksum_sha256,
258+
checksum_crc64nvme: checksum.checksum_crc64nvme,
258259
..Default::default()
259260
};
260261
Ok(S3Response::new(output))
@@ -472,6 +473,7 @@ impl S3 for FileSystem {
472473
key,
473474
metadata,
474475
content_length,
476+
content_md5,
475477
..
476478
} = input;
477479

@@ -530,6 +532,17 @@ impl S3 for FileSystem {
530532

531533
let md5_sum = hex(md5_hash.finalize());
532534

535+
if let Some(content_md5) = content_md5 {
536+
tracing::warn!(content_md5 = %content_md5, md5_sum = %md5_sum, "content_md5 mismatch");
537+
let content_md5 = base64_simd::STANDARD
538+
.decode_to_vec(content_md5)
539+
.map_err(|_| s3_error!(InvalidArgument))?;
540+
let content_md5 = hex(content_md5);
541+
if content_md5 != md5_sum {
542+
return Err(s3_error!(BadDigest, "content_md5 mismatch"));
543+
}
544+
}
545+
533546
let checksum = checksum.finalize();
534547

535548
if let Some(trailers) = req.trailing_headers {
@@ -589,6 +602,7 @@ impl S3 for FileSystem {
589602
checksum_crc32c: checksum.checksum_crc32c,
590603
checksum_sha1: checksum.checksum_sha1,
591604
checksum_sha256: checksum.checksum_sha256,
605+
checksum_crc64nvme: checksum.checksum_crc64nvme,
592606
..Default::default()
593607
};
594608
Ok(S3Response::new(output))

‎scripts/s3s-e2e.sh‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,14 @@
22

33
cargo build -p s3s-e2e --release
44

5-
export AWS_ACCESS_KEY_ID=rustfsadmin
6-
export AWS_SECRET_ACCESS_KEY=rustfsadmin
5+
export AWS_ACCESS_KEY_ID=minioadmin
6+
export AWS_SECRET_ACCESS_KEY=minioadmin
77
export AWS_REGION=us-east-1
8-
# export AWS_REGION=cn-north-1
98
export AWS_ENDPOINT_URL=http://localhost:9000
109

1110
if [ -z "$RUST_LOG" ]; then
1211
export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug"
1312
fi
1413
export RUST_BACKTRACE=full
1514

16-
./target/release/s3s-e2e "$@"
15+
./target/release/s3s-e2e "$@"

0 commit comments

Comments
 (0)