Skip to content

Commit 9c3c373

Browse files
afrankenafranken
committed
fix: Let S3Mock support ChecksumType.FULL_OBJECT for Multipart uploads
Fixes #2843
1 parent c1e3cae commit 9c3c373

File tree

9 files changed

+202
-37
lines changed

9 files changed

+202
-37
lines changed

‎integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultipartIT.kt‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,126 @@ internal class MultipartIT : S3TestBase() {
187187
}
188188
}
189189

190+
@Test
191+
@S3VerifiedSuccess(year = 2025)
192+
fun testMultipartUpload_withChecksumType_COMPOSITE(testInfo: TestInfo) {
193+
val bucketName = givenBucket(testInfo)
194+
val initiateMultipartUploadResult =
195+
s3Client
196+
.createMultipartUpload {
197+
it.bucket(bucketName)
198+
it.key(UPLOAD_FILE_NAME)
199+
it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
200+
it.checksumType(ChecksumType.COMPOSITE)
201+
}
202+
val uploadId = initiateMultipartUploadResult.uploadId()
203+
val uploadPartResult =
204+
s3Client.uploadPart(
205+
{
206+
it.bucket(initiateMultipartUploadResult.bucket())
207+
it.key(initiateMultipartUploadResult.key())
208+
it.uploadId(uploadId)
209+
it.partNumber(1)
210+
it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
211+
it.contentLength(UPLOAD_FILE_LENGTH)
212+
},
213+
RequestBody.fromFile(UPLOAD_FILE),
214+
)
215+
216+
val checksum =
217+
DigestUtil.checksumMultipart(
218+
listOf(UPLOAD_FILE_PATH),
219+
DefaultChecksumAlgorithm.CRC32,
220+
)
221+
222+
s3Client.completeMultipartUpload {
223+
it.bucket(initiateMultipartUploadResult.bucket())
224+
it.key(initiateMultipartUploadResult.key())
225+
it.uploadId(initiateMultipartUploadResult.uploadId())
226+
it.checksumType(ChecksumType.COMPOSITE)
227+
it.multipartUpload {
228+
it.parts({
229+
it.eTag(uploadPartResult.eTag())
230+
it.partNumber(1)
231+
it.checksumCRC32(uploadPartResult.checksumCRC32())
232+
})
233+
}
234+
}.also {
235+
assertThat(it.checksumCRC32()).isEqualTo(checksum)
236+
}
237+
238+
val etag = "\"${DigestUtil.hexDigestMultipart(listOf(UPLOAD_FILE_PATH))}\""
239+
s3Client
240+
.getObject {
241+
it.bucket(initiateMultipartUploadResult.bucket())
242+
it.key(initiateMultipartUploadResult.key())
243+
it.checksumMode(ChecksumMode.ENABLED)
244+
}.use {
245+
assertThat(it.response().eTag()).isEqualTo(etag)
246+
assertThat(it.response().checksumCRC32()).isEqualTo(checksum)
247+
}
248+
}
249+
250+
@Test
251+
@S3VerifiedSuccess(year = 2025)
252+
fun testMultipartUpload_withChecksumType_FULL_OBJECT(testInfo: TestInfo) {
253+
val bucketName = givenBucket(testInfo)
254+
val initiateMultipartUploadResult =
255+
s3Client
256+
.createMultipartUpload {
257+
it.bucket(bucketName)
258+
it.key(UPLOAD_FILE_NAME)
259+
it.checksumAlgorithm(ChecksumAlgorithm.CRC64_NVME)
260+
it.checksumType(ChecksumType.FULL_OBJECT)
261+
}
262+
val uploadId = initiateMultipartUploadResult.uploadId()
263+
val uploadPartResult =
264+
s3Client.uploadPart(
265+
{
266+
it.bucket(initiateMultipartUploadResult.bucket())
267+
it.key(initiateMultipartUploadResult.key())
268+
it.uploadId(uploadId)
269+
it.partNumber(1)
270+
it.checksumAlgorithm(ChecksumAlgorithm.CRC64_NVME)
271+
it.contentLength(UPLOAD_FILE_LENGTH)
272+
},
273+
RequestBody.fromFile(UPLOAD_FILE),
274+
)
275+
276+
val checksum =
277+
DigestUtil.checksumFor(
278+
UPLOAD_FILE_PATH,
279+
DefaultChecksumAlgorithm.CRC64NVME,
280+
)
281+
s3Client.completeMultipartUpload {
282+
it.bucket(initiateMultipartUploadResult.bucket())
283+
it.key(initiateMultipartUploadResult.key())
284+
it.uploadId(initiateMultipartUploadResult.uploadId())
285+
it.checksumType(ChecksumType.FULL_OBJECT)
286+
it.multipartUpload {
287+
it.parts({
288+
it.eTag(uploadPartResult.eTag())
289+
it.partNumber(1)
290+
it.checksumCRC64NVME(uploadPartResult.checksumCRC64NVME())
291+
})
292+
}
293+
}.also {
294+
assertThat(it.checksumCRC64NVME()).isEqualTo(checksum)
295+
}
296+
297+
val etag = "\"${DigestUtil.hexDigestMultipart(listOf(UPLOAD_FILE_PATH))}\""
298+
299+
s3Client
300+
.getObject {
301+
it.bucket(initiateMultipartUploadResult.bucket())
302+
it.key(initiateMultipartUploadResult.key())
303+
it.checksumMode(ChecksumMode.ENABLED)
304+
}.use {
305+
assertThat(it.response().eTag()).isEqualTo(etag)
306+
assertThat(it.response().checksumCRC64NVME()).isEqualTo(checksum)
307+
}
308+
}
309+
190310
/**
191311
* Tests if a multipart upload with the last part being smaller than 5MB works.
192312
*/

‎server/src/main/kotlin/com/adobe/testing/s3mock/controller/MultipartController.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ import com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromHeader
6060
import com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromSdk
6161
import com.adobe.testing.s3mock.util.HeaderUtil.checksumFrom
6262
import com.adobe.testing.s3mock.util.HeaderUtil.checksumHeaderFrom
63+
import com.adobe.testing.s3mock.util.HeaderUtil.checksumTypeFrom
6364
import com.adobe.testing.s3mock.util.HeaderUtil.encryptionHeadersFrom
6465
import com.adobe.testing.s3mock.util.HeaderUtil.storeHeadersFrom
6566
import com.adobe.testing.s3mock.util.HeaderUtil.userMetadataFrom
@@ -449,6 +450,7 @@ class MultipartController(
449450
encryptionHeadersFrom(httpHeaders),
450451
locationWithEncodedKey,
451452
checksumFrom(httpHeaders),
453+
checksumTypeFrom(httpHeaders),
452454
checksumAlgorithmFromHeader(httpHeaders)
453455
)!!
454456
} else {

‎server/src/main/kotlin/com/adobe/testing/s3mock/dto/ChecksumType.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,12 @@ enum class ChecksumType @JsonCreator constructor(private val value: String) {
3333
}
3434

3535
companion object {
36-
fun fromString(value: String): ChecksumType? {
36+
fun fromString(value: String?): ChecksumType? {
3737
return when (value) {
3838
"composite" -> COMPOSITE
39+
"COMPOSITE" -> COMPOSITE
3940
"full_object" -> FULL_OBJECT
41+
"FULL_OBJECT" -> FULL_OBJECT
4042
else -> null
4143
}
4244
}

‎server/src/main/kotlin/com/adobe/testing/s3mock/service/MultipartService.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ open class MultipartService(private val bucketStore: BucketStore, private val mu
161161
encryptionHeaders: Map<String, String>,
162162
location: String,
163163
checksum: String?,
164+
checksumType: ChecksumType?,
164165
checksumAlgorithm: ChecksumAlgorithm?
165166
): CompleteMultipartUploadResult? {
166167
val bucketMetadata = bucketStore.getBucketMetadata(bucketName)
@@ -177,6 +178,7 @@ open class MultipartService(private val bucketStore: BucketStore, private val mu
177178
multipartUploadInfo,
178179
location,
179180
checksum,
181+
checksumType,
180182
checksumAlgorithm
181183
)
182184
}

‎server/src/main/kotlin/com/adobe/testing/s3mock/store/MultipartStore.kt‎

Lines changed: 51 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ open class MultipartStore(private val objectStore: ObjectStore, private val obje
189189
uploadInfo: MultipartUploadInfo?,
190190
location: String,
191191
checksum: String?,
192+
checksumType: ChecksumType?,
192193
checksumAlgorithm: ChecksumAlgorithm?
193194
): CompleteMultipartUploadResult {
194195
requireNotNull(uploadInfo) { "Unknown upload $uploadId" }
@@ -201,42 +202,42 @@ open class MultipartStore(private val objectStore: ObjectStore, private val obje
201202
toInputStream(partsPaths).use { input ->
202203
tempFile.outputStream().use { os ->
203204
input.transferTo(os)
204-
val checksumFor = validateChecksums(uploadInfo, parts, partsPaths, checksum, checksumAlgorithm)
205-
val etag = DigestUtil.hexDigestMultipart(partsPaths)
206-
val s3ObjectMetadata = objectStore.storeS3ObjectMetadata(
207-
bucket,
208-
id,
209-
key,
210-
uploadInfo.contentType,
211-
uploadInfo.storeHeaders,
212-
tempFile,
213-
uploadInfo.userMetadata,
214-
encryptionHeaders,
215-
etag,
216-
uploadInfo.tags,
217-
uploadInfo.checksumAlgorithm,
218-
checksumFor,
219-
uploadInfo.upload.owner,
220-
uploadInfo.storageClass,
221-
ChecksumType.COMPOSITE
222-
)
223-
// delete parts and update MultipartInfo
224-
partsPaths.forEach { runCatching { it.toFile().deleteRecursively() } }
225-
val completedUploadInfo = uploadInfo.complete()
226-
writeMetafile(bucket, completedUploadInfo)
227-
return CompleteMultipartUploadResult.from(
228-
location,
229-
completedUploadInfo.bucket,
230-
key,
231-
etag,
232-
completedUploadInfo,
233-
checksumFor,
234-
s3ObjectMetadata.checksumType,
235-
checksumAlgorithm,
236-
s3ObjectMetadata.versionId
237-
)
238205
}
239206
}
207+
val checksumFor = validateChecksums(uploadInfo, tempFile, parts, partsPaths, checksum, checksumType, checksumAlgorithm)
208+
val etag = DigestUtil.hexDigestMultipart(partsPaths)
209+
val s3ObjectMetadata = objectStore.storeS3ObjectMetadata(
210+
bucket,
211+
id,
212+
key,
213+
uploadInfo.contentType,
214+
uploadInfo.storeHeaders,
215+
tempFile,
216+
uploadInfo.userMetadata,
217+
encryptionHeaders,
218+
etag,
219+
uploadInfo.tags,
220+
uploadInfo.checksumAlgorithm,
221+
checksumFor,
222+
uploadInfo.upload.owner,
223+
uploadInfo.storageClass,
224+
checksumType
225+
)
226+
// delete parts and update MultipartInfo
227+
partsPaths.forEach { runCatching { it.toFile().deleteRecursively() } }
228+
val completedUploadInfo = uploadInfo.complete()
229+
writeMetafile(bucket, completedUploadInfo)
230+
return CompleteMultipartUploadResult.from(
231+
location,
232+
completedUploadInfo.bucket,
233+
key,
234+
etag,
235+
completedUploadInfo,
236+
checksumFor,
237+
s3ObjectMetadata.checksumType,
238+
checksumAlgorithm,
239+
s3ObjectMetadata.versionId
240+
)
240241
} catch (e: IOException) {
241242
throw IllegalStateException("Error finishing multipart upload bucket=$bucket, key=$key, id=$id, uploadId=$uploadId", e)
242243
} finally {
@@ -421,14 +422,24 @@ open class MultipartStore(private val objectStore: ObjectStore, private val obje
421422

422423
private fun validateChecksums(
423424
uploadInfo: MultipartUploadInfo,
425+
tempFile: Path,
424426
completedParts: List<CompletedPart>,
425427
partsPaths: List<Path>,
426428
checksum: String?,
429+
checksumType: ChecksumType?,
427430
checksumAlgorithm: ChecksumAlgorithm?
428431
): String? {
429432
val checksumToValidate = checksum ?: uploadInfo.checksum
430433
val checksumAlgorithmToValidate = checksumAlgorithm ?: uploadInfo.checksumAlgorithm
431-
val checksumFor = checksumFor(partsPaths, uploadInfo)
434+
if(checksumType != null && checksumType != uploadInfo.checksumType) {
435+
//throw S3Exception
436+
}
437+
val checksumFor = if (uploadInfo.checksumType == ChecksumType.COMPOSITE) {
438+
checksumFor(partsPaths, uploadInfo)
439+
} else {
440+
checksumFor(tempFile, uploadInfo)
441+
}
442+
432443
if (checksumAlgorithmToValidate != null) {
433444
completedParts.forEach { part ->
434445
if (part.checksum(checksumAlgorithmToValidate) == null) {
@@ -451,6 +462,11 @@ open class MultipartStore(private val objectStore: ObjectStore, private val obje
451462
DigestUtil.checksumMultipart(paths, algo.toChecksumAlgorithm())
452463
}
453464

465+
private fun checksumFor(path: Path, uploadInfo: MultipartUploadInfo): String? =
466+
uploadInfo.checksumAlgorithm?.let { algo ->
467+
DigestUtil.checksumFor(path, algo.toChecksumAlgorithm())
468+
}
469+
454470
companion object {
455471
private val LOG: Logger = LoggerFactory.getLogger(MultipartStore::class.java)
456472
private const val PART_SUFFIX = ".part"

‎server/src/main/kotlin/com/adobe/testing/s3mock/store/S3ObjectMetadata.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ data class S3ObjectMetadata(
5353
val policy: AccessControlPolicy?,
5454
val versionId: String?,
5555
val deleteMarker: Boolean = false,
56-
val checksumType: ChecksumType? = ChecksumType.FULL_OBJECT
56+
val checksumType: ChecksumType?
5757
) {
5858
companion object {
5959
fun deleteMarker(metadata: S3ObjectMetadata, versionId: String?): S3ObjectMetadata =

‎server/src/main/kotlin/com/adobe/testing/s3mock/util/HeaderUtil.kt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.adobe.testing.s3mock.util
1818

1919
import com.adobe.testing.s3mock.dto.ChecksumAlgorithm
20+
import com.adobe.testing.s3mock.dto.ChecksumType
2021
import com.adobe.testing.s3mock.dto.CopySource
2122
import com.adobe.testing.s3mock.dto.StorageClass
2223
import com.adobe.testing.s3mock.store.S3ObjectMetadata
@@ -27,6 +28,7 @@ import com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_CRC32C
2728
import com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_CRC64NVME
2829
import com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA1
2930
import com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA256
31+
import com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_TYPE
3032
import com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CONTENT_SHA256
3133
import com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE_VERSION_ID
3234
import com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_SDK_CHECKSUM_ALGORITHM
@@ -231,6 +233,13 @@ object HeaderUtil {
231233
else null
232234
}
233235

236+
@JvmStatic
237+
fun checksumTypeFrom(headers: HttpHeaders): ChecksumType? {
238+
return if (headers.containsHeader(X_AMZ_CHECKSUM_TYPE))
239+
ChecksumType.fromString(headers.getFirst(X_AMZ_CHECKSUM_TYPE))
240+
else null
241+
}
242+
234243
@JvmStatic
235244
fun checksumFrom(headers: HttpHeaders): String? {
236245
return when {

‎server/src/test/kotlin/com/adobe/testing/s3mock/controller/MultipartControllerTest.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ internal class MultipartControllerTest : BaseControllerTest() {
279279
anyOrNull(),
280280
any(),
281281
anyOrNull(),
282+
anyOrNull(),
282283
anyOrNull()
283284
)
284285
).thenReturn(result)
@@ -357,6 +358,7 @@ internal class MultipartControllerTest : BaseControllerTest() {
357358
anyOrNull(),
358359
any(),
359360
anyOrNull(),
361+
anyOrNull(),
360362
anyOrNull()
361363
)
362364
).thenReturn(result)
@@ -430,6 +432,7 @@ internal class MultipartControllerTest : BaseControllerTest() {
430432
anyOrNull(),
431433
any(),
432434
anyOrNull(),
435+
anyOrNull(),
433436
anyOrNull()
434437
)
435438
).thenReturn(result)

0 commit comments

Comments
 (0)