Skip to content

Commit

Permalink
support crc64nvme in flex checksum (#3002)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lucix-aws
lucix-aws authored Feb 10, 2025
1 parent 7995348 commit 02b7d04
Show file tree
Hide file tree
Showing 7 changed files with 228 additions and 37 deletions.
9 changes: 9 additions & 0 deletions .changelog/f007dcb88454412692915fadefdcf8cc.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"id": "f007dcb8-8454-4126-9291-5fadefdcf8cc",
"type": "feature",
"description": "Support CRC64NVME flex checksums.",
"modules": [
"service/internal/checksum",
"service/internal/integrationtest"
]
}
99 changes: 86 additions & 13 deletions feature/s3/manager/integ_upload_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"fmt"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"log"
"reflect"
Expand All @@ -29,6 +30,8 @@ import (
"github.com/aws/smithy-go/middleware"
)

const crc64NVME = 0x9a6c_9329_ac4b_c9b5

var integBuf12MB = make([]byte, 1024*1024*12)
var integMD512MB = fmt.Sprintf("%x", md5.Sum(integBuf12MB))

Expand Down Expand Up @@ -124,6 +127,7 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
singlePartBytes := integBuf12MB[0:manager.DefaultUploadPartSize]
singlePartCRC32 := base64Sum(crc32.NewIEEE(), singlePartBytes)
singlePartCRC32C := base64Sum(crc32.New(crc32.MakeTable(crc32.Castagnoli)), singlePartBytes)
singlePartCRC64NVME := base64Sum(crc64.New(crc64.MakeTable(crc64NVME)), singlePartBytes)
singlePartSHA1 := base64Sum(sha1.New(), singlePartBytes)
singlePartSHA256 := base64Sum(sha256.New(), singlePartBytes)
singlePartMD5 := base64Sum(md5.New(), singlePartBytes)
Expand All @@ -132,6 +136,7 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
multiPartTailBytes := integBuf12MB[manager.DefaultUploadPartSize*2:]
multiPartTailCRC32 := base64Sum(crc32.NewIEEE(), multiPartTailBytes)
multiPartTailCRC32C := base64Sum(crc32.New(crc32.MakeTable(crc32.Castagnoli)), multiPartTailBytes)
multiPartTailCRC64NVME := base64Sum(crc64.New(crc64.MakeTable(crc64NVME)), multiPartTailBytes)
multiPartTailSHA1 := base64Sum(sha1.New(), multiPartTailBytes)
multiPartTailSHA256 := base64Sum(sha256.New(), multiPartTailBytes)
multiPartTailETag := fmt.Sprintf("%q", hexSum(md5.New(), multiPartTailBytes))
Expand All @@ -143,20 +148,25 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
multiPartSHA256 := base64SumOfSums(sha256.New(), []string{singlePartSHA256, singlePartSHA256, multiPartTailSHA256})
multiPartETag := `"4e982d58b6c2ce178ae042c23f9bca6e-3"` // Not obvious how this is computed

// s3 wants running checksum for crc64nvme
multiPartCRC64NVME := base64Sum(crc64.New(crc64.MakeTable(crc64NVME)), integBuf12MB)

cases := map[string]map[string]struct {
algorithm s3types.ChecksumAlgorithm
payload io.Reader
checksumCRC32 string
checksumCRC32C string
checksumSHA1 string
checksumSHA256 string
contentMD5 string
expectParts []s3types.CompletedPart
expectChecksumCRC32 string
expectChecksumCRC32C string
expectChecksumSHA1 string
expectChecksumSHA256 string
expectETag string
algorithm s3types.ChecksumAlgorithm
payload io.Reader
checksumCRC32 string
checksumCRC32C string
checksumCRC64NVME string
checksumSHA1 string
checksumSHA256 string
contentMD5 string
expectParts []s3types.CompletedPart
expectChecksumCRC32 string
expectChecksumCRC32C string
expectChecksumCRC64NVME string
expectChecksumSHA1 string
expectChecksumSHA256 string
expectETag string
}{
"auto single part": {
"no checksum algorithm passed": {
Expand All @@ -176,6 +186,12 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
expectChecksumCRC32C: singlePartCRC32C,
expectETag: singlePartETag,
},
"CRC64NVME": {
algorithm: s3types.ChecksumAlgorithmCrc64nvme,
payload: bytes.NewReader(singlePartBytes),
expectChecksumCRC64NVME: singlePartCRC64NVME,
expectETag: singlePartETag,
},
"SHA1": {
algorithm: s3types.ChecksumAlgorithmSha1,
payload: bytes.NewReader(singlePartBytes),
Expand All @@ -202,6 +218,12 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
expectChecksumCRC32C: singlePartCRC32C,
expectETag: singlePartETag,
},
"CRC64NVME": {
payload: bytes.NewReader(singlePartBytes),
checksumCRC64NVME: singlePartCRC64NVME,
expectChecksumCRC64NVME: singlePartCRC64NVME,
expectETag: singlePartETag,
},
"SHA1": {
payload: bytes.NewReader(singlePartBytes),
checksumSHA1: singlePartSHA1,
Expand Down Expand Up @@ -290,6 +312,29 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
expectChecksumCRC32C: multiPartCRC32C,
expectETag: multiPartETag,
},
"CRC64NVME": {
algorithm: s3types.ChecksumAlgorithmCrc64nvme,
payload: bytes.NewReader(multiPartBytes),
expectParts: []s3types.CompletedPart{
{
ChecksumCRC64NVME: aws.String(singlePartCRC64NVME),
ETag: aws.String(singlePartETag),
PartNumber: aws.Int32(1),
},
{
ChecksumCRC64NVME: aws.String(singlePartCRC64NVME),
ETag: aws.String(singlePartETag),
PartNumber: aws.Int32(2),
},
{
ChecksumCRC64NVME: aws.String(multiPartTailCRC64NVME),
ETag: aws.String(multiPartTailETag),
PartNumber: aws.Int32(3),
},
},
expectChecksumCRC64NVME: multiPartCRC64NVME,
expectETag: multiPartETag,
},
"SHA1": {
algorithm: s3types.ChecksumAlgorithmSha1,
payload: bytes.NewReader(multiPartBytes),
Expand Down Expand Up @@ -386,6 +431,30 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
expectChecksumCRC32C: multiPartCRC32C,
expectETag: multiPartETag,
},
"CRC64NVME": {
algorithm: s3types.ChecksumAlgorithmCrc64nvme,
payload: bytes.NewReader(multiPartBytes),
checksumCRC64NVME: multiPartCRC64NVME,
expectParts: []s3types.CompletedPart{
{
ChecksumCRC64NVME: aws.String(singlePartCRC64NVME),
ETag: aws.String(singlePartETag),
PartNumber: aws.Int32(1),
},
{
ChecksumCRC64NVME: aws.String(singlePartCRC64NVME),
ETag: aws.String(singlePartETag),
PartNumber: aws.Int32(2),
},
{
ChecksumCRC64NVME: aws.String(multiPartTailCRC64NVME),
ETag: aws.String(multiPartTailETag),
PartNumber: aws.Int32(3),
},
},
expectChecksumCRC64NVME: multiPartCRC64NVME,
expectETag: multiPartETag,
},
"SHA1": {
algorithm: s3types.ChecksumAlgorithmSha1,
payload: bytes.NewReader(multiPartBytes),
Expand Down Expand Up @@ -449,6 +518,7 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
ChecksumAlgorithm: c.algorithm,
ChecksumCRC32: toStringPtr(c.checksumCRC32),
ChecksumCRC32C: toStringPtr(c.checksumCRC32C),
ChecksumCRC64NVME: toStringPtr(c.checksumCRC64NVME),
ChecksumSHA1: toStringPtr(c.checksumSHA1),
ChecksumSHA256: toStringPtr(c.checksumSHA256),
ContentMD5: toStringPtr(c.contentMD5),
Expand All @@ -467,6 +537,9 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
if e, a := c.expectChecksumCRC32C, aws.ToString(out.ChecksumCRC32C); e != a {
t.Errorf("expect %v CRC32C checksum, got %v", e, a)
}
if e, a := c.expectChecksumCRC64NVME, aws.ToString(out.ChecksumCRC64NVME); e != a {
t.Errorf("expect %v CRC64NVME checksum, got %v", e, a)
}
if e, a := c.expectChecksumSHA1, aws.ToString(out.ChecksumSHA1); e != a {
t.Errorf("expect %v SHA1 checksum, got %v", e, a)
}
Expand Down
7 changes: 7 additions & 0 deletions feature/s3/manager/upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ type UploadOutput struct {
// The base64-encoded, 32-bit CRC32C checksum of the object.
ChecksumCRC32C *string

// The base64-encoded, 64-bit CRC64NVME checksum of the object.
ChecksumCRC64NVME *string

// The base64-encoded, 160-bit SHA-1 digest of the object.
ChecksumSHA1 *string

Expand Down Expand Up @@ -511,6 +514,7 @@ func (u *uploader) singlePart(r io.ReadSeeker, cleanup func()) (*UploadOutput, e
BucketKeyEnabled: aws.ToBool(out.BucketKeyEnabled),
ChecksumCRC32: out.ChecksumCRC32,
ChecksumCRC32C: out.ChecksumCRC32C,
ChecksumCRC64NVME: out.ChecksumCRC64NVME,
ChecksumSHA1: out.ChecksumSHA1,
ChecksumSHA256: out.ChecksumSHA256,
ETag: out.ETag,
Expand Down Expand Up @@ -653,6 +657,7 @@ func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadO
BucketKeyEnabled: aws.ToBool(completeOut.BucketKeyEnabled),
ChecksumCRC32: completeOut.ChecksumCRC32,
ChecksumCRC32C: completeOut.ChecksumCRC32C,
ChecksumCRC64NVME: completeOut.ChecksumCRC64NVME,
ChecksumSHA1: completeOut.ChecksumSHA1,
ChecksumSHA256: completeOut.ChecksumSHA256,
ETag: completeOut.ETag,
Expand Down Expand Up @@ -764,6 +769,8 @@ func (u *multiuploader) initChecksumAlgorithm() {
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32
case u.in.ChecksumCRC32C != nil:
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32c
case u.in.ChecksumCRC64NVME != nil:
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc64nvme
case u.in.ChecksumSHA1 != nil:
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha1
case u.in.ChecksumSHA256 != nil:
Expand Down
9 changes: 9 additions & 0 deletions service/internal/checksum/algorithms.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"fmt"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"strings"
"sync"
Expand All @@ -35,11 +36,15 @@ const (
AlgorithmCRC64NVME Algorithm = "CRC64NVME"
)

// inverted NVME polynomial as required by crc64.MakeTable
const crc64NVME = 0x9a6c_9329_ac4b_c9b5

var supportedAlgorithms = []Algorithm{
AlgorithmCRC32C,
AlgorithmCRC32,
AlgorithmSHA1,
AlgorithmSHA256,
AlgorithmCRC64NVME,
}

func (a Algorithm) String() string { return string(a) }
Expand Down Expand Up @@ -92,6 +97,8 @@ func NewAlgorithmHash(v Algorithm) (hash.Hash, error) {
return crc32.NewIEEE(), nil
case AlgorithmCRC32C:
return crc32.New(crc32.MakeTable(crc32.Castagnoli)), nil
case AlgorithmCRC64NVME:
return crc64.New(crc64.MakeTable(crc64NVME)), nil
default:
return nil, fmt.Errorf("unknown checksum algorithm, %v", v)
}
Expand All @@ -109,6 +116,8 @@ func AlgorithmChecksumLength(v Algorithm) (int, error) {
return crc32.Size, nil
case AlgorithmCRC32C:
return crc32.Size, nil
case AlgorithmCRC64NVME:
return crc64.Size, nil
default:
return 0, fmt.Errorf("unknown checksum algorithm, %v", v)
}
Expand Down
15 changes: 14 additions & 1 deletion service/internal/checksum/algorithms_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"encoding/base64"
"fmt"
"hash/crc32"
"hash/crc64"
"io"
"io/ioutil"
"strings"
Expand Down Expand Up @@ -69,6 +70,13 @@ func TestComputeChecksumReader(t *testing.T) {
ExpectRead: "hello world",
ExpectChecksum: "uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=",
},
"crc64nvme": {
Input: strings.NewReader("hello world"),
Algorithm: AlgorithmCRC64NVME,
ExpectChecksumLen: base64.StdEncoding.EncodedLen(crc64.Size),
ExpectRead: "hello world",
ExpectChecksum: "jSnVw/bqjr4=",
},
}

for name, c := range cases {
Expand Down Expand Up @@ -392,12 +400,13 @@ func TestFilterSupportedAlgorithms(t *testing.T) {
},
},
"mixed case": {
values: []string{"Crc32", "cRc32c", "shA1", "sHA256"},
values: []string{"Crc32", "cRc32c", "shA1", "sHA256", "crc64nvme"},
expectAlgorithms: []Algorithm{
AlgorithmCRC32,
AlgorithmCRC32C,
AlgorithmSHA1,
AlgorithmSHA256,
AlgorithmCRC64NVME,
},
},
}
Expand Down Expand Up @@ -442,6 +451,10 @@ func TestAlgorithmChecksumLength(t *testing.T) {
algorithm: AlgorithmSHA256,
expectLength: sha256.Size,
},
"crc64nvme": {
algorithm: AlgorithmCRC64NVME,
expectLength: crc64.Size,
},
}

for name, c := range cases {
Expand Down
Loading

0 comments on commit 02b7d04

Please sign in to comment.