Skip to content

Commit 541a987

Browse files
afrankenafranken
committed
fix: Let S3Mock support ChecksumType.FULL_OBJECT for Multipart uploads
Fixes #2843
1 parent c1e3cae commit 541a987

File tree

11 files changed

+259
-37
lines changed

11 files changed

+259
-37
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ Version 5.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
151151
* Features and fixes
152152
* Get object with range now returns the same headers as non-range calls.
153153
* Docker: Copy "s3mock.jar" to "/opt/", run with absolute path reference to avoid issues when working directory is changed. (fixes #2827)
154+
* S3Mock supports ChecksumType.FULL_OBJECT for Multipart uploads (fixes #2843)
154155
* Refactorings
155156
* Use Jackson 3 annotations and mappers.
156157
* AWS has deprecated SDK for Java v1 and will remove support EOY 2025.

‎integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultipartIT.kt‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,174 @@ internal class MultipartIT : S3TestBase() {
187187
}
188188
}
189189

190+
@Test
191+
@S3VerifiedSuccess(year = 2025)
192+
fun testMultipartUpload_withChecksumType_COMPOSITE(testInfo: TestInfo) {
193+
val bucketName = givenBucket(testInfo)
194+
val initiateMultipartUploadResult =
195+
s3Client
196+
.createMultipartUpload {
197+
it.bucket(bucketName)
198+
it.key(UPLOAD_FILE_NAME)
199+
it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
200+
it.checksumType(ChecksumType.COMPOSITE)
201+
}
202+
val uploadId = initiateMultipartUploadResult.uploadId()
203+
val uploadPartResult =
204+
s3Client.uploadPart(
205+
{
206+
it.bucket(initiateMultipartUploadResult.bucket())
207+
it.key(initiateMultipartUploadResult.key())
208+
it.uploadId(uploadId)
209+
it.partNumber(1)
210+
it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
211+
it.contentLength(UPLOAD_FILE_LENGTH)
212+
},
213+
RequestBody.fromFile(UPLOAD_FILE),
214+
)
215+
216+
val checksum =
217+
DigestUtil.checksumMultipart(
218+
listOf(UPLOAD_FILE_PATH),
219+
DefaultChecksumAlgorithm.CRC32,
220+
)
221+
222+
s3Client
223+
.completeMultipartUpload {
224+
it.bucket(initiateMultipartUploadResult.bucket())
225+
it.key(initiateMultipartUploadResult.key())
226+
it.uploadId(initiateMultipartUploadResult.uploadId())
227+
it.checksumType(ChecksumType.COMPOSITE)
228+
it.multipartUpload {
229+
it.parts({
230+
it.eTag(uploadPartResult.eTag())
231+
it.partNumber(1)
232+
it.checksumCRC32(uploadPartResult.checksumCRC32())
233+
})
234+
}
235+
}.also {
236+
assertThat(it.checksumCRC32()).isEqualTo(checksum)
237+
}
238+
239+
val etag = "\"${DigestUtil.hexDigestMultipart(listOf(UPLOAD_FILE_PATH))}\""
240+
s3Client
241+
.getObject {
242+
it.bucket(initiateMultipartUploadResult.bucket())
243+
it.key(initiateMultipartUploadResult.key())
244+
it.checksumMode(ChecksumMode.ENABLED)
245+
}.use {
246+
assertThat(it.response().eTag()).isEqualTo(etag)
247+
assertThat(it.response().checksumCRC32()).isEqualTo(checksum)
248+
}
249+
}
250+
251+
@Test
252+
@S3VerifiedSuccess(year = 2025)
253+
fun testMultipartUpload_withChecksumType_throwsOn_DIFFERENT(testInfo: TestInfo) {
254+
val bucketName = givenBucket(testInfo)
255+
val initiateMultipartUploadResult =
256+
s3Client
257+
.createMultipartUpload {
258+
it.bucket(bucketName)
259+
it.key(UPLOAD_FILE_NAME)
260+
it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
261+
it.checksumType(ChecksumType.COMPOSITE)
262+
}
263+
val uploadId = initiateMultipartUploadResult.uploadId()
264+
val uploadPartResult =
265+
s3Client.uploadPart(
266+
{
267+
it.bucket(initiateMultipartUploadResult.bucket())
268+
it.key(initiateMultipartUploadResult.key())
269+
it.uploadId(uploadId)
270+
it.partNumber(1)
271+
it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
272+
it.contentLength(UPLOAD_FILE_LENGTH)
273+
},
274+
RequestBody.fromFile(UPLOAD_FILE),
275+
)
276+
277+
assertThatThrownBy {
278+
s3Client
279+
.completeMultipartUpload {
280+
it.bucket(initiateMultipartUploadResult.bucket())
281+
it.key(initiateMultipartUploadResult.key())
282+
it.uploadId(initiateMultipartUploadResult.uploadId())
283+
it.checksumType(ChecksumType.FULL_OBJECT) // intentionally different from creteMultipartUpload value
284+
it.multipartUpload {
285+
it.parts({
286+
it.eTag(uploadPartResult.eTag())
287+
it.partNumber(1)
288+
it.checksumCRC32(uploadPartResult.checksumCRC32())
289+
})
290+
}
291+
}
292+
}.isInstanceOf(AwsServiceException::class.java)
293+
.hasMessageContaining("Service: S3, Status Code: 400")
294+
}
295+
296+
@Test
297+
@S3VerifiedSuccess(year = 2025)
298+
fun testMultipartUpload_withChecksumType_FULL_OBJECT(testInfo: TestInfo) {
299+
val bucketName = givenBucket(testInfo)
300+
val initiateMultipartUploadResult =
301+
s3Client
302+
.createMultipartUpload {
303+
it.bucket(bucketName)
304+
it.key(UPLOAD_FILE_NAME)
305+
it.checksumAlgorithm(ChecksumAlgorithm.CRC64_NVME)
306+
it.checksumType(ChecksumType.FULL_OBJECT)
307+
}
308+
val uploadId = initiateMultipartUploadResult.uploadId()
309+
val uploadPartResult =
310+
s3Client.uploadPart(
311+
{
312+
it.bucket(initiateMultipartUploadResult.bucket())
313+
it.key(initiateMultipartUploadResult.key())
314+
it.uploadId(uploadId)
315+
it.partNumber(1)
316+
it.checksumAlgorithm(ChecksumAlgorithm.CRC64_NVME)
317+
it.contentLength(UPLOAD_FILE_LENGTH)
318+
},
319+
RequestBody.fromFile(UPLOAD_FILE),
320+
)
321+
322+
val checksum =
323+
DigestUtil.checksumFor(
324+
UPLOAD_FILE_PATH,
325+
DefaultChecksumAlgorithm.CRC64NVME,
326+
)
327+
328+
s3Client
329+
.completeMultipartUpload {
330+
it.bucket(initiateMultipartUploadResult.bucket())
331+
it.key(initiateMultipartUploadResult.key())
332+
it.uploadId(initiateMultipartUploadResult.uploadId())
333+
it.checksumType(ChecksumType.FULL_OBJECT)
334+
it.multipartUpload {
335+
it.parts({
336+
it.eTag(uploadPartResult.eTag())
337+
it.partNumber(1)
338+
it.checksumCRC64NVME(uploadPartResult.checksumCRC64NVME())
339+
})
340+
}
341+
}.also {
342+
assertThat(it.checksumCRC64NVME()).isEqualTo(checksum)
343+
}
344+
345+
val etag = "\"${DigestUtil.hexDigestMultipart(listOf(UPLOAD_FILE_PATH))}\""
346+
347+
s3Client
348+
.getObject {
349+
it.bucket(initiateMultipartUploadResult.bucket())
350+
it.key(initiateMultipartUploadResult.key())
351+
it.checksumMode(ChecksumMode.ENABLED)
352+
}.use {
353+
assertThat(it.response().eTag()).isEqualTo(etag)
354+
assertThat(it.response().checksumCRC64NVME()).isEqualTo(checksum)
355+
}
356+
}
357+
190358
/**
191359
* Tests if a multipart upload with the last part being smaller than 5MB works.
192360
*/

‎server/src/main/kotlin/com/adobe/testing/s3mock/S3Exception.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,14 @@ class S3Exception
6767
)
6868
}
6969

70+
fun completeRequestWrongChecksumMode(checksumMode: String): S3Exception {
71+
return S3Exception(
72+
HttpStatus.BAD_REQUEST.value(), BAD_REQUEST_CODE,
73+
("The upload was created using the $checksumMode checksum mode. " +
74+
"The complete request must use the same checksum mode.")
75+
)
76+
}
77+
7078
val NO_SUCH_UPLOAD_MULTIPART: S3Exception = S3Exception(
7179
HttpStatus.NOT_FOUND.value(), "NoSuchUpload",
7280
"The specified multipart upload does not exist. The upload ID might be invalid, or the "

‎server/src/main/kotlin/com/adobe/testing/s3mock/controller/MultipartController.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ import com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromHeader
6060
import com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromSdk
6161
import com.adobe.testing.s3mock.util.HeaderUtil.checksumFrom
6262
import com.adobe.testing.s3mock.util.HeaderUtil.checksumHeaderFrom
63+
import com.adobe.testing.s3mock.util.HeaderUtil.checksumTypeFrom
6364
import com.adobe.testing.s3mock.util.HeaderUtil.encryptionHeadersFrom
6465
import com.adobe.testing.s3mock.util.HeaderUtil.storeHeadersFrom
6566
import com.adobe.testing.s3mock.util.HeaderUtil.userMetadataFrom
@@ -449,6 +450,7 @@ class MultipartController(
449450
encryptionHeadersFrom(httpHeaders),
450451
locationWithEncodedKey,
451452
checksumFrom(httpHeaders),
453+
checksumTypeFrom(httpHeaders),
452454
checksumAlgorithmFromHeader(httpHeaders)
453455
)!!
454456
} else {

‎server/src/main/kotlin/com/adobe/testing/s3mock/dto/ChecksumType.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,12 @@ enum class ChecksumType @JsonCreator constructor(private val value: String) {
3333
}
3434

3535
companion object {
36-
fun fromString(value: String): ChecksumType? {
36+
fun fromString(value: String?): ChecksumType? {
3737
return when (value) {
3838
"composite" -> COMPOSITE
39+
"COMPOSITE" -> COMPOSITE
3940
"full_object" -> FULL_OBJECT
41+
"FULL_OBJECT" -> FULL_OBJECT
4042
else -> null
4143
}
4244
}

‎server/src/main/kotlin/com/adobe/testing/s3mock/service/MultipartService.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ open class MultipartService(private val bucketStore: BucketStore, private val mu
161161
encryptionHeaders: Map<String, String>,
162162
location: String,
163163
checksum: String?,
164+
checksumType: ChecksumType?,
164165
checksumAlgorithm: ChecksumAlgorithm?
165166
): CompleteMultipartUploadResult? {
166167
val bucketMetadata = bucketStore.getBucketMetadata(bucketName)
@@ -177,6 +178,7 @@ open class MultipartService(private val bucketStore: BucketStore, private val mu
177178
multipartUploadInfo,
178179
location,
179180
checksum,
181+
checksumType,
180182
checksumAlgorithm
181183
)
182184
}

‎server/src/main/kotlin/com/adobe/testing/s3mock/store/MultipartStore.kt‎

Lines changed: 51 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ open class MultipartStore(private val objectStore: ObjectStore, private val obje
189189
uploadInfo: MultipartUploadInfo?,
190190
location: String,
191191
checksum: String?,
192+
checksumType: ChecksumType?,
192193
checksumAlgorithm: ChecksumAlgorithm?
193194
): CompleteMultipartUploadResult {
194195
requireNotNull(uploadInfo) { "Unknown upload $uploadId" }
@@ -201,42 +202,42 @@ open class MultipartStore(private val objectStore: ObjectStore, private val obje
201202
toInputStream(partsPaths).use { input ->
202203
tempFile.outputStream().use { os ->
203204
input.transferTo(os)
204-
val checksumFor = validateChecksums(uploadInfo, parts, partsPaths, checksum, checksumAlgorithm)
205-
val etag = DigestUtil.hexDigestMultipart(partsPaths)
206-
val s3ObjectMetadata = objectStore.storeS3ObjectMetadata(
207-
bucket,
208-
id,
209-
key,
210-
uploadInfo.contentType,
211-
uploadInfo.storeHeaders,
212-
tempFile,
213-
uploadInfo.userMetadata,
214-
encryptionHeaders,
215-
etag,
216-
uploadInfo.tags,
217-
uploadInfo.checksumAlgorithm,
218-
checksumFor,
219-
uploadInfo.upload.owner,
220-
uploadInfo.storageClass,
221-
ChecksumType.COMPOSITE
222-
)
223-
// delete parts and update MultipartInfo
224-
partsPaths.forEach { runCatching { it.toFile().deleteRecursively() } }
225-
val completedUploadInfo = uploadInfo.complete()
226-
writeMetafile(bucket, completedUploadInfo)
227-
return CompleteMultipartUploadResult.from(
228-
location,
229-
completedUploadInfo.bucket,
230-
key,
231-
etag,
232-
completedUploadInfo,
233-
checksumFor,
234-
s3ObjectMetadata.checksumType,
235-
checksumAlgorithm,
236-
s3ObjectMetadata.versionId
237-
)
238205
}
239206
}
207+
val checksumFor = validateChecksums(uploadInfo, tempFile, parts, partsPaths, checksum, checksumType, checksumAlgorithm)
208+
val etag = DigestUtil.hexDigestMultipart(partsPaths)
209+
val s3ObjectMetadata = objectStore.storeS3ObjectMetadata(
210+
bucket,
211+
id,
212+
key,
213+
uploadInfo.contentType,
214+
uploadInfo.storeHeaders,
215+
tempFile,
216+
uploadInfo.userMetadata,
217+
encryptionHeaders,
218+
etag,
219+
uploadInfo.tags,
220+
uploadInfo.checksumAlgorithm,
221+
checksumFor,
222+
uploadInfo.upload.owner,
223+
uploadInfo.storageClass,
224+
checksumType
225+
)
226+
// delete parts and update MultipartInfo
227+
partsPaths.forEach { runCatching { it.toFile().deleteRecursively() } }
228+
val completedUploadInfo = uploadInfo.complete()
229+
writeMetafile(bucket, completedUploadInfo)
230+
return CompleteMultipartUploadResult.from(
231+
location,
232+
completedUploadInfo.bucket,
233+
key,
234+
etag,
235+
completedUploadInfo,
236+
checksumFor,
237+
s3ObjectMetadata.checksumType,
238+
checksumAlgorithm,
239+
s3ObjectMetadata.versionId
240+
)
240241
} catch (e: IOException) {
241242
throw IllegalStateException("Error finishing multipart upload bucket=$bucket, key=$key, id=$id, uploadId=$uploadId", e)
242243
} finally {
@@ -421,14 +422,24 @@ open class MultipartStore(private val objectStore: ObjectStore, private val obje
421422

422423
private fun validateChecksums(
423424
uploadInfo: MultipartUploadInfo,
425+
tempFile: Path,
424426
completedParts: List<CompletedPart>,
425427
partsPaths: List<Path>,
426428
checksum: String?,
429+
checksumType: ChecksumType?,
427430
checksumAlgorithm: ChecksumAlgorithm?
428431
): String? {
429432
val checksumToValidate = checksum ?: uploadInfo.checksum
430433
val checksumAlgorithmToValidate = checksumAlgorithm ?: uploadInfo.checksumAlgorithm
431-
val checksumFor = checksumFor(partsPaths, uploadInfo)
434+
if(checksumType != null && uploadInfo.checksumType != null && checksumType != uploadInfo.checksumType) {
435+
throw S3Exception.completeRequestWrongChecksumMode(uploadInfo.checksumType.name)
436+
}
437+
val checksumFor = if (uploadInfo.checksumType == ChecksumType.COMPOSITE) {
438+
checksumFor(partsPaths, uploadInfo)
439+
} else {
440+
checksumFor(tempFile, uploadInfo)
441+
}
442+
432443
if (checksumAlgorithmToValidate != null) {
433444
completedParts.forEach { part ->
434445
if (part.checksum(checksumAlgorithmToValidate) == null) {
@@ -451,6 +462,11 @@ open class MultipartStore(private val objectStore: ObjectStore, private val obje
451462
DigestUtil.checksumMultipart(paths, algo.toChecksumAlgorithm())
452463
}
453464

465+
private fun checksumFor(path: Path, uploadInfo: MultipartUploadInfo): String? =
466+
uploadInfo.checksumAlgorithm?.let { algo ->
467+
DigestUtil.checksumFor(path, algo.toChecksumAlgorithm())
468+
}
469+
454470
companion object {
455471
private val LOG: Logger = LoggerFactory.getLogger(MultipartStore::class.java)
456472
private const val PART_SUFFIX = ".part"

‎server/src/main/kotlin/com/adobe/testing/s3mock/store/S3ObjectMetadata.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ data class S3ObjectMetadata(
5353
val policy: AccessControlPolicy?,
5454
val versionId: String?,
5555
val deleteMarker: Boolean = false,
56-
val checksumType: ChecksumType? = ChecksumType.FULL_OBJECT
56+
val checksumType: ChecksumType?
5757
) {
5858
companion object {
5959
fun deleteMarker(metadata: S3ObjectMetadata, versionId: String?): S3ObjectMetadata =

0 commit comments

Comments
 (0)