Skip to content

Commit b80b2ad

Browse files
leif-scalityleif-scality
committed
CLDSRV-872: return stored checksum in PutObject response
1 parent e0566e0 commit b80b2ad

File tree

4 files changed

+193
-16
lines changed

4 files changed

+193
-16
lines changed

‎lib/api/objectPut.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,10 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) {
225225
if (storingResult) {
226226
// ETag's hex should always be enclosed in quotes
227227
responseHeaders.ETag = `"${storingResult.contentMD5}"`;
228+
if (storingResult.checksum) {
229+
const { checksumAlgorithm, checksumValue } = storingResult.checksum;
230+
responseHeaders[`x-amz-checksum-${checksumAlgorithm}`] = checksumValue;
231+
}
228232
}
229233
const vcfg = bucket.getVersioningConfiguration();
230234
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';

‎lib/services.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,7 @@ const services = {
333333
tags: md.getTags(),
334334
contentMD5,
335335
versionId,
336+
checksum: md.getChecksum(),
336337
});
337338
});
338339
},

‎tests/functional/raw-node/test/checksumPutObjectUploadPart.js‎

Lines changed: 183 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
const assert = require('assert');
22
const crypto = require('crypto');
33
const async = require('async');
4+
const { algorithms } = require('../../../../lib/api/apiUtils/integrity/validateChecksums');
45

56
const { makeS3Request } = require('../utils/makeRequest');
67
const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4');
@@ -38,7 +39,11 @@ function doPutRequest(url, headers, body, callback) {
3839
res => {
3940
let data = '';
4041
res.on('data', chunk => { data += chunk; });
41-
res.on('end', () => callback(null, { statusCode: res.statusCode, body: data }));
42+
res.on('end', () => callback(null, {
43+
statusCode: res.statusCode,
44+
body: data,
45+
headers: res.headers,
46+
}));
4247
}
4348
);
4449
req.on('error', callback);
@@ -141,16 +146,38 @@ const msgMalformedTrailer = 'The request contained trailing data that was not we
141146
const msgSdkMissingTrailer = 'x-amz-sdk-checksum-algorithm specified, but no corresponding' +
142147
' x-amz-checksum-* or x-amz-trailer headers were found.';
143148

149+
// Module-level variables for computed crc64nvme checksums (filled in before hook)
150+
let crc64nvmeOfTestContent2;
151+
let crc64nvmeOfTrailerContent;
152+
144153
// Create the 24 common protocol-scenario tests for a given URL factory.
145154
// urlFn() is called lazily at test runtime so that uploadId is available.
146-
function makeScenarioTests(urlFn) {
155+
// checkChecksumResponse: if true, assert x-amz-checksum-* response headers on 200 OK tests.
156+
function makeScenarioTests(urlFn, checkChecksumResponse = false) {
157+
before(async () => {
158+
if (!crc64nvmeOfTestContent2) {
159+
crc64nvmeOfTestContent2 = await algorithms.crc64nvme.digest(testContent2);
160+
}
161+
if (!crc64nvmeOfTrailerContent) {
162+
crc64nvmeOfTrailerContent = await algorithms.crc64nvme.digest(trailerContent);
163+
}
164+
});
165+
147166
itSkipIfAWS(
148167
'testS3PutNoChecksum: signed sha256 in x-amz-content-sha256, no x-amz-checksum header -> 200 OK',
149168
done => {
150169
doPutRequest(urlFn(), {
151170
'x-amz-content-sha256': testContent2Sha256Hex,
152171
'content-length': testContent2.length,
153-
}, testContent2, (err, res) => assertStatus(200)(err, res, done));
172+
}, testContent2, (err, res) => {
173+
assertStatus(200)(err, res, () => {
174+
if (checkChecksumResponse) {
175+
assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTestContent2,
176+
`expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTestContent2}`);
177+
}
178+
done();
179+
});
180+
});
154181
});
155182

156183
itSkipIfAWS(
@@ -161,7 +188,15 @@ function makeScenarioTests(urlFn) {
161188
'x-amz-sdk-checksum-algorithm': 'SHA256',
162189
'x-amz-checksum-sha256': testContent2Sha256B64,
163190
'content-length': testContent2.length,
164-
}, testContent2, (err, res) => assertStatus(200)(err, res, done));
191+
}, testContent2, (err, res) => {
192+
assertStatus(200)(err, res, () => {
193+
if (checkChecksumResponse) {
194+
assert.strictEqual(res.headers['x-amz-checksum-sha256'], testContent2Sha256B64,
195+
`expected x-amz-checksum-sha256: ${testContent2Sha256B64}`);
196+
}
197+
done();
198+
});
199+
});
165200
});
166201

167202
itSkipIfAWS(
@@ -184,7 +219,15 @@ function makeScenarioTests(urlFn) {
184219
'x-amz-sdk-checksum-algorithm': 'SHA256',
185220
'x-amz-checksum-sha256': testContent2Sha256B64,
186221
'content-length': testContent2.length,
187-
}, testContent2, (err, res) => assertStatus(200)(err, res, done));
222+
}, testContent2, (err, res) => {
223+
assertStatus(200)(err, res, () => {
224+
if (checkChecksumResponse) {
225+
assert.strictEqual(res.headers['x-amz-checksum-sha256'], testContent2Sha256B64,
226+
`expected x-amz-checksum-sha256: ${testContent2Sha256B64}`);
227+
}
228+
done();
229+
});
230+
});
188231
});
189232

190233
itSkipIfAWS(
@@ -208,7 +251,15 @@ function makeScenarioTests(urlFn) {
208251
'x-amz-trailer': 'x-amz-checksum-sha256',
209252
'x-amz-decoded-content-length': trailerContent.length,
210253
'content-length': Buffer.byteLength(body),
211-
}, body, (err, res) => assertStatus(200)(err, res, done));
254+
}, body, (err, res) => {
255+
assertStatus(200)(err, res, () => {
256+
if (checkChecksumResponse) {
257+
assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256,
258+
`expected x-amz-checksum-sha256: ${trailerContentSha256}`);
259+
}
260+
done();
261+
});
262+
});
212263
});
213264

214265
itSkipIfAWS(
@@ -295,7 +346,15 @@ function makeScenarioTests(urlFn) {
295346
'x-amz-trailer': 'x-amz-checksum-sha256',
296347
'x-amz-decoded-content-length': trailerContent.length,
297348
'content-length': Buffer.byteLength(body),
298-
}, body, (err, res) => assertStatus(200)(err, res, done));
349+
}, body, (err, res) => {
350+
assertStatus(200)(err, res, () => {
351+
if (checkChecksumResponse) {
352+
assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256,
353+
`expected x-amz-checksum-sha256: ${trailerContentSha256}`);
354+
}
355+
done();
356+
});
357+
});
299358
});
300359

301360
itSkipIfAWS(
@@ -308,7 +367,15 @@ function makeScenarioTests(urlFn) {
308367
'x-amz-sdk-checksum-algorithm': 'SHA256',
309368
'x-amz-decoded-content-length': trailerContent.length,
310369
'content-length': Buffer.byteLength(body),
311-
}, body, (err, res) => assertStatus(200)(err, res, done));
370+
}, body, (err, res) => {
371+
assertStatus(200)(err, res, () => {
372+
if (checkChecksumResponse) {
373+
assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256,
374+
`expected x-amz-checksum-sha256: ${trailerContentSha256}`);
375+
}
376+
done();
377+
});
378+
});
312379
});
313380

314381
itSkipIfAWS(
@@ -403,7 +470,15 @@ function makeScenarioTests(urlFn) {
403470
// no x-amz-trailer
404471
'x-amz-decoded-content-length': trailerContent.length,
405472
'content-length': Buffer.byteLength(body),
406-
}, body, (err, res) => assertStatus(200)(err, res, done));
473+
}, body, (err, res) => {
474+
assertStatus(200)(err, res, () => {
475+
if (checkChecksumResponse) {
476+
assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent,
477+
`expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`);
478+
}
479+
done();
480+
});
481+
});
407482
});
408483

409484
itSkipIfAWS(
@@ -417,7 +492,15 @@ function makeScenarioTests(urlFn) {
417492
// no x-amz-trailer
418493
'x-amz-decoded-content-length': trailerContent.length,
419494
'content-length': Buffer.byteLength(body),
420-
}, body, (err, res) => assertStatus(200)(err, res, done));
495+
}, body, (err, res) => {
496+
assertStatus(200)(err, res, () => {
497+
if (checkChecksumResponse) {
498+
assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent,
499+
`expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`);
500+
}
501+
done();
502+
});
503+
});
421504
});
422505

423506
itSkipIfAWS(
@@ -431,7 +514,15 @@ function makeScenarioTests(urlFn) {
431514
'content-md5': trailerContentMd5B64,
432515
'x-amz-decoded-content-length': trailerContent.length,
433516
'content-length': Buffer.byteLength(body),
434-
}, body, (err, res) => assertStatus(200)(err, res, done));
517+
}, body, (err, res) => {
518+
assertStatus(200)(err, res, () => {
519+
if (checkChecksumResponse) {
520+
assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256,
521+
`expected x-amz-checksum-sha256: ${trailerContentSha256}`);
522+
}
523+
done();
524+
});
525+
});
435526
});
436527

437528
itSkipIfAWS(
@@ -445,7 +536,15 @@ function makeScenarioTests(urlFn) {
445536
'x-amz-trailer': 'x-amz-checksum-sha256',
446537
'x-amz-decoded-content-length': trailerContent.length,
447538
'content-length': Buffer.byteLength(body),
448-
}, body, (err, res) => assertStatus(200)(err, res, done));
539+
}, body, (err, res) => {
540+
assertStatus(200)(err, res, () => {
541+
if (checkChecksumResponse) {
542+
assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256,
543+
`expected x-amz-checksum-sha256: ${trailerContentSha256}`);
544+
}
545+
done();
546+
});
547+
});
449548
});
450549
}
451550

@@ -557,7 +656,7 @@ describe('PutObject: trailer and checksum protocol scenarios', () => {
557656
});
558657
});
559658

560-
makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`);
659+
makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`, true);
561660

562661
});
563662

@@ -608,3 +707,74 @@ describe('UploadPart: trailer and checksum protocol scenarios', () => {
608707
() => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}`
609708
);
610709
});
710+
711+
describe('PutObject: checksum response header per algorithm', () => {
712+
const url = `http://localhost:8000/${bucket}/${objectKey}`;
713+
const body = testContent2;
714+
const sha256Hex = testContent2Sha256Hex;
715+
716+
let expectedCrc64nvme;
717+
718+
before(async () => {
719+
await new Promise((resolve, reject) =>
720+
makeS3Request({ method: 'PUT', authCredentials, bucket },
721+
err => (err ? reject(err) : resolve())));
722+
expectedCrc64nvme = await algorithms.crc64nvme.digest(body);
723+
});
724+
725+
after(done => {
726+
makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => {
727+
makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => {
728+
assert.ifError(err);
729+
done();
730+
});
731+
});
732+
});
733+
734+
const checksumAlgos = [
735+
{ name: 'crc32', computeExpected: () => algorithms.crc32.digest(body) },
736+
{ name: 'crc32c', computeExpected: () => algorithms.crc32c.digest(body) },
737+
{ name: 'crc64nvme', computeExpected: () => expectedCrc64nvme },
738+
{ name: 'sha1', computeExpected: () => algorithms.sha1.digest(body) },
739+
{ name: 'sha256', computeExpected: () => algorithms.sha256.digest(body) },
740+
];
741+
742+
for (const algo of checksumAlgos) {
743+
itSkipIfAWS(
744+
`returns x-amz-checksum-${algo.name} response header with correct value`,
745+
done => {
746+
const expectedValue = algo.computeExpected();
747+
const headerName = `x-amz-checksum-${algo.name}`;
748+
doPutRequest(url, {
749+
'x-amz-content-sha256': sha256Hex,
750+
[headerName]: expectedValue,
751+
'content-length': body.length,
752+
}, body, (err, res) => {
753+
assert.ifError(err);
754+
assert.strictEqual(res.statusCode, 200,
755+
`expected 200, got ${res.statusCode}: ${res.body}`);
756+
assert.strictEqual(res.headers[headerName], expectedValue,
757+
`expected ${headerName}: ${expectedValue}`);
758+
done();
759+
});
760+
}
761+
);
762+
}
763+
764+
itSkipIfAWS(
765+
'returns x-amz-checksum-crc64nvme response header when no checksum header is sent',
766+
done => {
767+
doPutRequest(url, {
768+
'x-amz-content-sha256': sha256Hex,
769+
'content-length': body.length,
770+
}, body, (err, res) => {
771+
assert.ifError(err);
772+
assert.strictEqual(res.statusCode, 200,
773+
`expected 200, got ${res.statusCode}: ${res.body}`);
774+
assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], expectedCrc64nvme,
775+
`expected x-amz-checksum-crc64nvme: ${expectedCrc64nvme}`);
776+
done();
777+
});
778+
}
779+
);
780+
});

‎tests/unit/api/objectPut.js‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -882,8 +882,9 @@ describe('objectPut API', () => {
882882

883883
bucketPut(authInfo, testPutBucketRequest, log, err => {
884884
assert.ifError(err);
885-
objectPut(authInfo, request, undefined, log, err => {
885+
objectPut(authInfo, request, undefined, log, (err, resHeaders) => {
886886
assert.ifError(err);
887+
assert.strictEqual(resHeaders['x-amz-checksum-sha256'], sha256Value);
887888
metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => {
888889
assert.ifError(err);
889890
assert(md.checksum, 'checksum should be set in metadata');
@@ -899,13 +900,14 @@ describe('objectPut API', () => {
899900
it('should store crc64nvme checksum in metadata when no checksum header is provided', done => {
900901
bucketPut(authInfo, testPutBucketRequest, log, err => {
901902
assert.ifError(err);
902-
objectPut(authInfo, testPutObjectRequest, undefined, log, err => {
903+
objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => {
903904
assert.ifError(err);
905+
assert(resHeaders['x-amz-checksum-crc64nvme'], 'crc64nvme response header should be set');
904906
metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => {
905907
assert.ifError(err);
906908
assert(md.checksum, 'checksum should be set in metadata');
907909
assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme');
908-
assert(md.checksum.checksumValue, 'checksumValue should be set');
910+
assert.strictEqual(md.checksum.checksumValue, resHeaders['x-amz-checksum-crc64nvme']);
909911
assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
910912
done();
911913
});

0 commit comments

Comments
 (0)