Skip to content

Commit b376406

Browse files
klauspostklauspost
authored
Use provided sha256 hasher (#1749)
1 parent a6e4bc8 commit b376406

File tree

5 files changed

+87
-18
lines changed

5 files changed

+87
-18
lines changed

‎api.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -845,7 +845,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
845845
// Additionally, we also look if the initialized client is secure,
846846
// if yes then we don't need to perform streaming signature.
847847
req = signer.StreamingSignV4(req, accessKeyID,
848-
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC())
848+
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
849849
default:
850850
// Set sha256 sum for signature calculation only with signature version '4'.
851851
shaHeader := unsignedPayload

‎core_test.go

+24
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ const (
4141

4242
// Tests for Core GetObject() function.
4343
func TestGetObjectCore(t *testing.T) {
44+
if os.Getenv(serverEndpoint) == "" {
45+
t.Skip("SERVER_ENDPOINT not set")
46+
}
4447
if testing.Short() {
4548
t.Skip("skipping functional tests for the short runs")
4649
}
@@ -238,6 +241,9 @@ func TestGetObjectCore(t *testing.T) {
238241
// Tests GetObject to return Content-Encoding properly set
239242
// and overrides any auto decoding.
240243
func TestGetObjectContentEncoding(t *testing.T) {
244+
if os.Getenv(serverEndpoint) == "" {
245+
t.Skip("SERVER_ENDPOINT not set")
246+
}
241247
if testing.Short() {
242248
t.Skip("skipping functional tests for the short runs")
243249
}
@@ -311,6 +317,9 @@ func TestGetObjectContentEncoding(t *testing.T) {
311317

312318
// Tests get bucket policy core API.
313319
func TestGetBucketPolicy(t *testing.T) {
320+
if os.Getenv(serverEndpoint) == "" {
321+
t.Skip("SERVER_ENDPOINT not set")
322+
}
314323
if testing.Short() {
315324
t.Skip("skipping functional tests for short runs")
316325
}
@@ -374,6 +383,9 @@ func TestGetBucketPolicy(t *testing.T) {
374383

375384
// Tests Core CopyObject API implementation.
376385
func TestCoreCopyObject(t *testing.T) {
386+
if os.Getenv(serverEndpoint) == "" {
387+
t.Skip("SERVER_ENDPOINT not set")
388+
}
377389
if testing.Short() {
378390
t.Skip("skipping functional tests for short runs")
379391
}
@@ -497,6 +509,9 @@ func TestCoreCopyObject(t *testing.T) {
497509

498510
// Test Core CopyObjectPart implementation
499511
func TestCoreCopyObjectPart(t *testing.T) {
512+
if os.Getenv(serverEndpoint) == "" {
513+
t.Skip("SERVER_ENDPOINT not set")
514+
}
500515
if testing.Short() {
501516
t.Skip("skipping functional tests for short runs")
502517
}
@@ -650,6 +665,9 @@ func TestCoreCopyObjectPart(t *testing.T) {
650665

651666
// Test Core PutObject.
652667
func TestCorePutObject(t *testing.T) {
668+
if os.Getenv(serverEndpoint) == "" {
669+
t.Skip("SERVER_ENDPOINT not set")
670+
}
653671
if testing.Short() {
654672
t.Skip("skipping functional tests for short runs")
655673
}
@@ -744,6 +762,9 @@ func TestCorePutObject(t *testing.T) {
744762
}
745763

746764
func TestCoreGetObjectMetadata(t *testing.T) {
765+
if os.Getenv(serverEndpoint) == "" {
766+
t.Skip("SERVER_ENDPOINT not set")
767+
}
747768
if testing.Short() {
748769
t.Skip("skipping functional tests for the short runs")
749770
}
@@ -801,6 +822,9 @@ func TestCoreGetObjectMetadata(t *testing.T) {
801822
}
802823

803824
func TestCoreMultipartUpload(t *testing.T) {
825+
if os.Getenv(serverEndpoint) == "" {
826+
t.Skip("SERVER_ENDPOINT not set")
827+
}
804828
if testing.Short() {
805829
t.Skip("skipping functional tests for the short runs")
806830
}

‎pkg/credentials/file_test.go

+4
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,14 @@ package credentials
2020
import (
2121
"os"
2222
"path/filepath"
23+
"runtime"
2324
"testing"
2425
)
2526

2627
func TestFileAWS(t *testing.T) {
28+
if runtime.GOOS == "windows" {
29+
t.Skip("\"/bin/cat\": file does not exist")
30+
}
2731
os.Clearenv()
2832

2933
creds := NewFileAWSCredentials("credentials.sample", "")

‎pkg/signer/request-signature-streaming.go

+26-11
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ import (
2626
"strconv"
2727
"strings"
2828
"time"
29+
30+
md5simd "github.com/minio/md5-simd"
2931
)
3032

3133
// Reference for constants used below -
@@ -90,28 +92,28 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
9092

9193
// buildChunkStringToSign - returns the string to sign given chunk data
9294
// and previous signature.
93-
func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string {
95+
func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
9496
stringToSignParts := []string{
9597
streamingPayloadHdr,
9698
t.Format(iso8601DateFormat),
9799
getScope(region, t, ServiceTypeS3),
98100
previousSig,
99101
emptySHA256,
100-
hex.EncodeToString(sum256(chunkData)),
102+
chunkChecksum,
101103
}
102104

103105
return strings.Join(stringToSignParts, "\n")
104106
}
105107

106108
// buildTrailerChunkStringToSign - returns the string to sign given chunk data
107109
// and previous signature.
108-
func buildTrailerChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string {
110+
func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
109111
stringToSignParts := []string{
110112
streamingTrailerHdr,
111113
t.Format(iso8601DateFormat),
112114
getScope(region, t, ServiceTypeS3),
113115
previousSig,
114-
hex.EncodeToString(sum256(chunkData)),
116+
chunkChecksum,
115117
}
116118

117119
return strings.Join(stringToSignParts, "\n")
@@ -148,21 +150,21 @@ func buildChunkHeader(chunkLen int64, signature string) []byte {
148150
}
149151

150152
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
151-
func buildChunkSignature(chunkData []byte, reqTime time.Time, region,
153+
func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region,
152154
previousSignature, secretAccessKey string,
153155
) string {
154156
chunkStringToSign := buildChunkStringToSign(reqTime, region,
155-
previousSignature, chunkData)
157+
previousSignature, chunkCheckSum)
156158
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
157159
return getSignature(signingKey, chunkStringToSign)
158160
}
159161

160162
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
161-
func buildTrailerChunkSignature(chunkData []byte, reqTime time.Time, region,
163+
func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region,
162164
previousSignature, secretAccessKey string,
163165
) string {
164166
chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region,
165-
previousSignature, chunkData)
167+
previousSignature, chunkChecksum)
166168
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
167169
return getSignature(signingKey, chunkStringToSign)
168170
}
@@ -202,12 +204,17 @@ type StreamingReader struct {
202204
totalChunks int
203205
lastChunkSize int
204206
trailer http.Header
207+
sh256 md5simd.Hasher
205208
}
206209

207210
// signChunk - signs a chunk read from s.baseReader of chunkLen size.
208211
func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) {
209212
// Compute chunk signature for next header
210-
signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime,
213+
s.sh256.Reset()
214+
s.sh256.Write(s.chunkBuf[:chunkLen])
215+
chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil))
216+
217+
signature := buildChunkSignature(chunckChecksum, s.reqTime,
211218
s.region, s.prevSignature, s.secretAccessKey)
212219

213220
// For next chunk signature computation
@@ -239,8 +246,11 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) {
239246
s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...)
240247
}
241248

249+
s.sh256.Reset()
250+
s.sh256.Write(s.chunkBuf)
251+
chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil))
242252
// Compute chunk signature
243-
signature := buildTrailerChunkSignature(s.chunkBuf, s.reqTime,
253+
signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime,
244254
s.region, s.prevSignature, s.secretAccessKey)
245255

246256
// For next chunk signature computation
@@ -273,7 +283,7 @@ func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) {
273283
// StreamingSignV4 - provides chunked upload signatureV4 support by
274284
// implementing io.Reader.
275285
func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken,
276-
region string, dataLen int64, reqTime time.Time,
286+
region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher,
277287
) *http.Request {
278288
// Set headers needed for streaming signature.
279289
prepareStreamingRequest(req, sessionToken, dataLen, reqTime)
@@ -294,6 +304,7 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok
294304
chunkNum: 1,
295305
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
296306
lastChunkSize: int(dataLen % payloadChunkSize),
307+
sh256: sh256,
297308
}
298309
if len(req.Trailer) > 0 {
299310
stReader.trailer = req.Trailer
@@ -384,5 +395,9 @@ func (s *StreamingReader) Read(buf []byte) (int, error) {
384395

385396
// Close - this method makes underlying io.ReadCloser's Close method available.
386397
func (s *StreamingReader) Close() error {
398+
if s.sh256 != nil {
399+
s.sh256.Close()
400+
s.sh256 = nil
401+
}
387402
return s.baseReadCloser.Close()
388403
}

‎pkg/signer/request-signature-streaming_test.go

+32-6
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,37 @@ package signer
1919

2020
import (
2121
"bytes"
22+
fipssha256 "crypto/sha256"
2223
"encoding/hex"
24+
"hash"
2325
"io"
2426
"net/http"
2527
"testing"
2628
"time"
29+
30+
md5simd "github.com/minio/md5-simd"
31+
"github.com/minio/sha256-simd"
2732
)
2833

34+
// hashWrapper implements the md5simd.Hasher interface.
35+
type hashWrapper struct {
36+
hash.Hash
37+
}
38+
39+
func newSHA256Hasher() md5simd.Hasher {
40+
return &hashWrapper{Hash: fipssha256.New()}
41+
}
42+
43+
func (m *hashWrapper) Close() {
44+
m.Hash = nil
45+
}
46+
47+
func sum256hex(data []byte) string {
48+
hash := sha256.New()
49+
hash.Write(data)
50+
return hex.EncodeToString(hash.Sum(nil))
51+
}
52+
2953
func TestGetSeedSignature(t *testing.T) {
3054
accessKeyID := "AKIAIOSFODNN7EXAMPLE"
3155
secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
@@ -42,7 +66,7 @@ func TestGetSeedSignature(t *testing.T) {
4266
t.Fatalf("Failed to parse time - %v", err)
4367
}
4468

45-
req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime)
69+
req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime, newSHA256Hasher())
4670
actualSeedSignature := req.Body.(*StreamingReader).seedSignature
4771

4872
expectedSeedSignature := "38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079"
@@ -58,7 +82,8 @@ func TestChunkSignature(t *testing.T) {
5882
location := "us-east-1"
5983
secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
6084
expectedSignature := "ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648"
61-
actualSignature := buildChunkSignature(chunkData, reqTime, location, previousSignature, secretAccessKeyID)
85+
chunkCheckSum := sum256hex(chunkData)
86+
actualSignature := buildChunkSignature(chunkCheckSum, reqTime, location, previousSignature, secretAccessKeyID)
6287
if actualSignature != expectedSignature {
6388
t.Errorf("Expected %s but received %s", expectedSignature, actualSignature)
6489
}
@@ -72,7 +97,8 @@ func TestTrailerChunkSignature(t *testing.T) {
7297
location := "us-east-1"
7398
secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
7499
expectedSignature := "41e14ac611e27a8bb3d66c3bad6856f209297767d5dd4fc87d8fa9e422e03faf"
75-
actualSignature := buildTrailerChunkSignature(chunkData, reqTime, location, previousSignature, secretAccessKeyID)
100+
chunkCheckSum := sum256hex(chunkData)
101+
actualSignature := buildTrailerChunkSignature(chunkCheckSum, reqTime, location, previousSignature, secretAccessKeyID)
76102
if actualSignature != expectedSignature {
77103
t.Errorf("Expected %s but received %s", expectedSignature, actualSignature)
78104
}
@@ -90,7 +116,7 @@ func TestSetStreamingAuthorization(t *testing.T) {
90116

91117
dataLen := int64(65 * 1024)
92118
reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z")
93-
req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime)
119+
req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher())
94120

95121
expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079"
96122

@@ -117,7 +143,7 @@ func TestSetStreamingAuthorizationTrailer(t *testing.T) {
117143

118144
dataLen := int64(65 * 1024)
119145
reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z")
120-
req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime)
146+
req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher())
121147

122148
// (order of signed headers is different)
123149
expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class;x-amz-trailer,Signature=106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e"
@@ -145,7 +171,7 @@ func TestStreamingReader(t *testing.T) {
145171

146172
baseReader := io.NopCloser(bytes.NewReader(bytes.Repeat([]byte("a"), 65*1024)))
147173
req.Body = baseReader
148-
req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime)
174+
req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher())
149175

150176
b, err := io.ReadAll(req.Body)
151177
if err != nil {

0 commit comments

Comments
 (0)