API consistency: Multipart API / DTOs

Checking AWS APIs for changes and S3Mock for implementations and tests.

Fixes #2340

Author: afranken

CHANGELOG.md

@@ -150,14 +150,20 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
     * Check AWS API for changes
     * Update S3Mock API / DTOs
     * Add tests for changed API / DTOs
-    * List Objects now returns "delimiter"
-    * List Objects V2 now accepts "fetch-owner" and returns "delimiter"
-    * List Buckets now accepts all parameters listed in AWS S3 API
+    * CreateBucket API now accepts "CreateBucketConfiguration" request body
+    * CompleteMultipartUpload API now accepts checksums and returns checksums
+    * ListObjects API now returns "delimiter"
+    * ListObjects V2 API now accepts "fetch-owner" and returns "delimiter"
+    * ListBuckets API now accepts parameters listed in AWS S3 API
+    * CreateMultipartUpload now accepts checksum headers and returns checksum and encryption headers
+    * CompleteMultipartUpload now accepts checksum headers and returns checksum and encryption headers
+      * Checksum validation on complete
 * Version updates (deliverable dependencies)
   * Bump spring-boot.version from 3.4.4 to 3.4.5
   * Bump testcontainers.version from 1.20.6 to 1.21.0
 * Version updates (build dependencies)
-  * Bump github/codeql-action from 3.28.15 to 3.28.16
+  * Bump github/codeql-action from 3.28.15 to 3.28.17
+  * Bump com.puppycrawl.tools:checkstyle from 10.23.0 to 10.23.1
 
 ## 4.1.1
 Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartIT.kt

@@ -37,6 +37,8 @@ import software.amazon.awssdk.services.s3.S3AsyncClient
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm
 import software.amazon.awssdk.services.s3.model.ChecksumMode
+import software.amazon.awssdk.services.s3.model.ChecksumType
+import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest
 import software.amazon.awssdk.services.s3.model.CompletedPart
 import software.amazon.awssdk.services.s3.model.ListPartsRequest
 import software.amazon.awssdk.services.s3.model.NoSuchBucketException
@@ -255,13 +257,9 @@ internal class MultiPartIT : S3TestBase() {
       .isEqualTo("${serviceEndpoint}/$bucketName/${UriUtils.encode(UPLOAD_FILE_NAME, StandardCharsets.UTF_8)}")
   }
 
-
-  /**
-   * Tests if a multipart upload with the last part being smaller than 5MB works.
-   */
   @Test
   @S3VerifiedSuccess(year = 2025)
-  fun testMultipartUpload_checksum(testInfo: TestInfo) {
+  fun `multipartupload send checksum in create and complete`(testInfo: TestInfo) {
     val bucketName = givenBucket(testInfo)
     val uploadFile = File(TEST_IMAGE_TIFF)
     //construct uploadfile >5MB
@@ -277,7 +275,10 @@ internal class MultiPartIT : S3TestBase() {
         it.bucket(bucketName)
         it.key(TEST_IMAGE_TIFF)
         it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
+        it.checksumType(ChecksumType.COMPOSITE)
       }
+    assertThat(initiateMultipartUploadResult.checksumAlgorithm()).isEqualTo(ChecksumAlgorithm.CRC32)
+    assertThat(initiateMultipartUploadResult.checksumType()).isEqualTo(ChecksumType.COMPOSITE)
     val uploadId = initiateMultipartUploadResult.uploadId()
     // upload part 1, <5MB
     val partResponse1 = s3Client.uploadPart(
@@ -289,7 +290,6 @@ internal class MultiPartIT : S3TestBase() {
         it.partNumber(1)
         it.contentLength(tempFile.length())
       },
-      //.lastPart(true)
       RequestBody.fromFile(tempFile),
     )
     val etag1 = partResponse1.eTag()
@@ -303,7 +303,6 @@ internal class MultiPartIT : S3TestBase() {
         it.partNumber(2)
         it.contentLength(uploadFile.length())
       },
-      //.lastPart(true)
       RequestBody.fromFile(uploadFile),
     )
     val etag2 = partResponse2.eTag()
@@ -354,11 +353,87 @@ internal class MultiPartIT : S3TestBase() {
       .isEqualTo("${serviceEndpoint}/$bucketName/${UriUtils.encode(TEST_IMAGE_TIFF, StandardCharsets.UTF_8)}")
   }
 
+  @Test
+  @S3VerifiedSuccess(year = 2025)
+  fun `multipartupload send checksum in create only`(testInfo: TestInfo) {
+    val bucketName = givenBucket(testInfo)
+    val uploadFile = File(TEST_IMAGE_TIFF)
+    //construct uploadfile >5MB
+    val tempFile = Files.newTemporaryFile().also {
+      (readStreamIntoByteArray(uploadFile.inputStream()) +
+        readStreamIntoByteArray(uploadFile.inputStream()) +
+        readStreamIntoByteArray(uploadFile.inputStream()))
+        .inputStream()
+        .copyTo(it.outputStream())
+    }
+
+    val initiateMultipartUploadResult = s3Client.createMultipartUpload {
+        it.bucket(bucketName)
+        it.key(TEST_IMAGE_TIFF)
+        it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
+      }
+    val uploadId = initiateMultipartUploadResult.uploadId()
+    // upload part 1, <5MB
+    val partResponse1 = s3Client.uploadPart(
+      {
+        it.bucket(initiateMultipartUploadResult.bucket())
+        it.key(initiateMultipartUploadResult.key())
+        it.uploadId(uploadId)
+        it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
+        it.partNumber(1)
+        it.contentLength(tempFile.length())
+      },
+      RequestBody.fromFile(tempFile),
+    )
+    val etag1 = partResponse1.eTag()
+    val checksum1 = partResponse1.checksumCRC32()
+    // upload part 2, <5MB
+    val partResponse2 = s3Client.uploadPart({
+        it.bucket(initiateMultipartUploadResult.bucket())
+        it.key(initiateMultipartUploadResult.key())
+        it.uploadId(uploadId)
+        it.checksumAlgorithm(ChecksumAlgorithm.CRC32)
+        it.partNumber(2)
+        it.contentLength(uploadFile.length())
+      },
+      RequestBody.fromFile(uploadFile),
+    )
+    val etag2 = partResponse2.eTag()
+    val checksum2 = partResponse2.checksumCRC32()
+    val localChecksum1 = DigestUtil.checksumFor(tempFile.toPath(), DefaultChecksumAlgorithm.CRC32)
+    assertThat(checksum1).isEqualTo(localChecksum1)
+    val localChecksum2 = DigestUtil.checksumFor(uploadFile.toPath(), DefaultChecksumAlgorithm.CRC32)
+    assertThat(checksum2).isEqualTo(localChecksum2)
+
+    assertThatThrownBy {
+      s3Client.completeMultipartUpload {
+        it.bucket(initiateMultipartUploadResult.bucket())
+        it.key(initiateMultipartUploadResult.key())
+        it.uploadId(initiateMultipartUploadResult.uploadId())
+        it.multipartUpload {
+          it.parts(
+            {
+              it.eTag(etag1)
+              it.partNumber(1)
+            },
+            {
+              it.eTag(etag2)
+              it.partNumber(2)
+            }
+          )
+        }
+      }
+    }.isInstanceOf(S3Exception::class.java)
+      .hasMessageContaining("Service: S3, Status Code: 400")
+      .hasMessageContaining("The upload was created using a crc32 checksum. The complete request must include the " +
+        "checksum for each part. It was missing for part 1 in the request.")
+  }
+
 
   @S3VerifiedSuccess(year = 2025)
   @ParameterizedTest
   @MethodSource(value = ["checksumAlgorithms"])
-  fun testUploadPart_checksumAlgorithm(checksumAlgorithm: software.amazon.awssdk.checksums.spi.ChecksumAlgorithm,
+  fun testUploadPart_checksumAlgorithm_initiate(checksumAlgorithm: software.amazon.awssdk.checksums.spi.ChecksumAlgorithm,
                                        testInfo: TestInfo) {
     val bucketName = givenBucket(testInfo)
     val uploadFile = File(UPLOAD_FILE_NAME)
@@ -377,7 +452,6 @@ internal class MultiPartIT : S3TestBase() {
         it.checksumAlgorithm(checksumAlgorithm.toAlgorithm())
         it.partNumber(1)
         it.contentLength(uploadFile.length()).build()
-        //.lastPart(true)
       },
       RequestBody.fromFile(uploadFile),
     ).also {
@@ -392,6 +466,45 @@ internal class MultiPartIT : S3TestBase() {
     }
   }
 
+  @S3VerifiedSuccess(year = 2025)
+  @ParameterizedTest
+  @MethodSource(value = ["checksumAlgorithms"])
+  fun testUploadPart_checksumAlgorithm_complete(
+    checksumAlgorithm: software.amazon.awssdk.checksums.spi.ChecksumAlgorithm,
+    testInfo: TestInfo
+  ) {
+    val bucketName = givenBucket(testInfo)
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val initiateMultipartUploadResult = s3Client.createMultipartUpload {
+      it.bucket(bucketName)
+      it.key(UPLOAD_FILE_NAME)
+    }
+    val uploadId = initiateMultipartUploadResult.uploadId()
+
+    s3Client.uploadPart({
+        it.bucket(initiateMultipartUploadResult.bucket())
+        it.key(initiateMultipartUploadResult.key())
+        it.uploadId(uploadId)
+        it.partNumber(1)
+        it.contentLength(uploadFile.length()).build()
+        //.lastPart(true)
+      },
+      RequestBody.fromFile(uploadFile),
+    )
+
+    assertThatThrownBy {
+      s3Client.completeMultipartUpload {
+        it.bucket(bucketName)
+        it.key(UPLOAD_FILE_NAME)
+        it.uploadId(uploadId)
+        it.checksumType(ChecksumType.COMPOSITE)
+        it.checksum("WRONG CHECKSUM", checksumAlgorithm.toAlgorithm())
+      }
+    }
+      .isInstanceOf(S3Exception::class.java)
+      .hasMessageContaining("Service: S3, Status Code: 400")
+  }
+
   @S3VerifiedSuccess(year = 2025)
   @ParameterizedTest
   @MethodSource(value = ["checksumAlgorithms"])
@@ -473,6 +586,19 @@ internal class MultiPartIT : S3TestBase() {
       else -> error("Unknown checksum algorithm")
     }
 
+  private fun CompleteMultipartUploadRequest.Builder.checksum(
+    checksum: String,
+    checksumAlgorithm: ChecksumAlgorithm
+  ): CompleteMultipartUploadRequest.Builder =
+    when (checksumAlgorithm) {
+      ChecksumAlgorithm.SHA1 -> this.checksumSHA1(checksum)
+      ChecksumAlgorithm.SHA256 -> this.checksumSHA256(checksum)
+      ChecksumAlgorithm.CRC32 -> this.checksumCRC32(checksum)
+      ChecksumAlgorithm.CRC32_C -> this.checksumCRC32C(checksum)
+      ChecksumAlgorithm.CRC64_NVME -> this.checksumCRC64NVME(checksum)
+      else -> error("Unknown checksum algorithm")
+    }
+
   @Test
   @S3VerifiedSuccess(year = 2025)
   fun testInitiateMultipartAndRetrieveParts(testInfo: TestInfo) {
@@ -1372,6 +1498,6 @@ internal class MultiPartIT : S3TestBase() {
   companion object {
     private const val NO_SUCH_BUCKET = "The specified bucket does not exist"
     private const val INVALID_PART_NUMBER = "Part number must be an integer between 1 and 10000, inclusive"
-    private const val INVALID_PART = "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tagSet might not have matched the part's entity tagSet."
+    private const val INVALID_PART = "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag may not match the part's entity tag."
   }
 }