Merge pull request #2345 from adobe/2218-do-not-store-aws-chunked-encoding

Do not store "aws-chunked" encoding

Author: afranken

CHANGELOG.md

@@ -11,7 +11,8 @@
 * [PLANNED - 5.x - RELEASE TBD](#planned---5x---release-tbd)
   * [Planned changes](#planned-changes)
 * [CURRENT - 4.x - THIS VERSION IS UNDER ACTIVE DEVELOPMENT](#current---4x---this-version-is-under-active-development)
-  * [4.1.1 - PLANNED](#411---planned)
+  * [4.2.0 - PLANNED](#420---planned)
+  * [4.1.1](#411)
   * [4.1.0](#410)
   * [4.0.0](#400)
 * [DEPRECATED - 3.x](#deprecated---3x)
@@ -137,18 +138,30 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
 
 **The current major version 4 will receive new features, dependency updates and bug fixes on a continuous basis.**
 
-## 4.1.1 - PLANNED
+## 4.2.0 - PLANNED
 Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
 
 * Features and fixes
-  * Content-Encoding: aws-chunked should not be stored (fixes #2218)
+  * Support checksum algorithm CRC64NVME (fixes #2334)
 * Refactorings
   * TBD
 * Version updates (deliverable dependencies)
   * TBD
 * Version updates (build dependencies)
   * TBD
 
+## 4.1.1
+Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
+
+* Features and fixes
+  * Content-Encoding: aws-chunked should not be stored (fixes #2218)
+* Refactorings
+  * none
+* Version updates (deliverable dependencies)
+  * none
+* Version updates (build dependencies)
+  * none
+
 ## 4.1.0
 Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
 

…ting/s3mock/its/AwsChunkedEndcodingIT.kt …sting/s3mock/its/AwsChunkedEncodingIT.ktintegration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingIT.kt renamed to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEncodingIT.kt integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingIT.kt renamed to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEncodingIT.kt

@@ -30,9 +30,9 @@ import java.io.InputStream
 /**
  * Chunked encoding with signing is only active in AWS SDK v2 when endpoint is http
  */
-internal class AwsChunkedEndcodingIT : S3TestBase() {
+internal class AwsChunkedEncodingIT : S3TestBase() {
 
-  private val s3Client = createS3Client(serviceEndpointHttp)
+  private val s3Client = createS3Client(serviceEndpointHttp, true)
 
   /**
    * Unfortunately the S3 API does not persist or return data that would let us verify if signed and chunked encoding
@@ -68,11 +68,11 @@ internal class AwsChunkedEndcodingIT : S3TestBase() {
     s3Client.getObject {
       it.bucket(bucket)
       it.key(UPLOAD_FILE_NAME)
-    }.also { getObjectResponse ->
-      assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
-      assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
-
-      getObjectResponse.response().checksumSHA256().also {
+    }.also {
+      assertThat(it.response().eTag()).isEqualTo(expectedEtag)
+      assertThat(it.response().contentLength()).isEqualTo(uploadFile.length())
+      assertThat(it.response().contentEncoding()).isNotEqualTo("aws-chunked")
+      it.response().checksumSHA256().also {
         assertThat(it).isNotBlank
         assertThat(it).isEqualTo(expectedChecksum)
       }
@@ -109,6 +109,35 @@ internal class AwsChunkedEndcodingIT : S3TestBase() {
     }.also {
       assertThat(it.response().eTag()).isEqualTo(expectedEtag)
       assertThat(it.response().contentLength()).isEqualTo(uploadFile.length())
+      assertThat(it.response().contentEncoding()).isNotEqualTo("aws-chunked")
+    }
+  }
+
+  @Test
+  @S3VerifiedFailure(
+    year = 2023,
+    reason = "Only works with http endpoints"
+  )
+  fun `put object creates correct content-encoding, get object returns content-encoding`(testInfo: TestInfo) {
+    val bucket = givenBucket(testInfo)
+    val uploadFile = File(UPLOAD_FILE_NAME)
+
+    val customEncoding = "my-custom-encoding"
+
+    s3Client.putObject(
+      {
+        it.bucket(bucket)
+        it.key(UPLOAD_FILE_NAME)
+        it.contentEncoding(customEncoding)
+      },
+      RequestBody.fromFile(uploadFile)
+    )
+
+    s3Client.getObject {
+      it.bucket(bucket)
+      it.key(UPLOAD_FILE_NAME)
+    }.also {
+      assertThat(it.response().contentEncoding()).isEqualTo(customEncoding)
     }
   }
 }

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt

@@ -227,7 +227,6 @@ internal abstract class S3TestBase {
   @AfterEach
   fun cleanupStores() {
     for (bucket in _s3Client.listBuckets().buckets()) {
-      if(bucket.name() == "testputandgetretention-545488000") {return}
       //Empty all buckets
       deleteMultipartUploads(bucket)
       deleteObjectsInBucket(bucket, isObjectLockEnabled(bucket))

server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java

@@ -24,8 +24,8 @@
 import static com.adobe.testing.s3mock.S3Exception.BAD_REQUEST_CONTENT;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_DECODED_CONTENT_LENGTH;
 import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromSdk;
-import static com.adobe.testing.s3mock.util.HeaderUtil.isChunked;
-import static com.adobe.testing.s3mock.util.HeaderUtil.isChunkedAndV4Signed;
+import static com.adobe.testing.s3mock.util.HeaderUtil.isChunkedEncoding;
+import static com.adobe.testing.s3mock.util.HeaderUtil.isV4Signed;
 
 import com.adobe.testing.s3mock.dto.ChecksumAlgorithm;
 import com.adobe.testing.s3mock.util.AbstractAwsInputStream;
@@ -93,9 +93,9 @@ public Pair<Path, String> toTempFile(InputStream inputStream) {
   private InputStream wrapStream(InputStream dataStream, HttpHeaders headers) {
     var lengthHeader = headers.getFirst(X_AMZ_DECODED_CONTENT_LENGTH);
     var length = lengthHeader == null ? -1 : Long.parseLong(lengthHeader);
-    if (isChunkedAndV4Signed(headers)) {
+    if (isV4Signed(headers)) {
       return new AwsChunkedDecodingChecksumInputStream(dataStream, length);
-    } else if (isChunked(headers)) {
+    } else if (isChunkedEncoding(headers)) {
       return new AwsUnsignedChunkedDecodingChecksumInputStream(dataStream, length);
     } else {
       return dataStream;

server/src/main/java/com/adobe/testing/s3mock/util/HeaderUtil.java

@@ -116,7 +116,8 @@ public static Map<String, String> storeHeadersFrom(HttpHeaders headers) {
         header -> (equalsIgnoreCase(header, HttpHeaders.EXPIRES)
             || equalsIgnoreCase(header, HttpHeaders.CONTENT_LANGUAGE)
             || equalsIgnoreCase(header, HttpHeaders.CONTENT_DISPOSITION)
-            || equalsIgnoreCase(header, HttpHeaders.CONTENT_ENCODING)
+            || (equalsIgnoreCase(header, HttpHeaders.CONTENT_ENCODING)
+                && !isOnlyChunkedEncoding(headers))
             || equalsIgnoreCase(header, HttpHeaders.CACHE_CONTROL)
         ));
   }
@@ -152,19 +153,35 @@ && isNotBlank(entry.getValue().get(0))) {
         .collect(Collectors.toMap(SimpleEntry::getKey, SimpleEntry::getValue));
   }
 
-  public static boolean isChunkedAndV4Signed(HttpHeaders headers) {
+  public static boolean isV4Signed(HttpHeaders headers) {
     var sha256Header = headers.getFirst(X_AMZ_CONTENT_SHA256);
     return sha256Header != null
         && (sha256Header.equals(STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD)
             || sha256Header.equals(STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD_TRAILER));
   }
 
-  public static boolean isChunked(HttpHeaders headers) {
+  public static boolean isChunkedEncoding(HttpHeaders headers) {
     var contentEncodingHeaders = headers.get(HttpHeaders.CONTENT_ENCODING);
     return (contentEncodingHeaders != null
         && (contentEncodingHeaders.contains(AWS_CHUNKED)));
   }
 
+  /**
+   * Check if aws-chunked is the only "Content-Encoding" header.
+   * <quote>
+   *   If aws-chunked is the only value that you pass in the content-encoding header, S3 considers
+   *   the content-encoding header empty and does not return this header when your retrieve the
+   *   object.
+   * </quote>
+   *  See <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html">API</a>
+   */
+  private static boolean isOnlyChunkedEncoding(HttpHeaders headers) {
+    var contentEncodingHeaders = headers.get(HttpHeaders.CONTENT_ENCODING);
+    return (contentEncodingHeaders != null
+        && contentEncodingHeaders.size() == 1
+        && (contentEncodingHeaders.contains(AWS_CHUNKED)));
+  }
+
   public static MediaType mediaTypeFrom(final String contentType) {
     try {
       return MediaType.parseMediaType(contentType);