diff --git a/CHANGELOG.md b/CHANGELOG.md index ee7e31c16..0e125a00f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ * [PLANNED - 5.x - RELEASE TBD](#planned---5x---release-tbd) * [Planned changes](#planned-changes) * [CURRENT - 4.x - THIS VERSION IS UNDER ACTIVE DEVELOPMENT](#current---4x---this-version-is-under-active-development) - * [4.1.1 - PLANNED](#411---planned) + * [4.2.0 - PLANNED](#420---planned) + * [4.1.1](#411) * [4.1.0](#410) * [4.0.0](#400) * [DEPRECATED - 3.x](#deprecated---3x) @@ -137,11 +138,11 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav **The current major version 4 will receive new features, dependency updates and bug fixes on a continuous basis.** -## 4.1.1 - PLANNED +## 4.2.0 - PLANNED Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration. * Features and fixes - * Content-Encoding: aws-chunked should not be stored (fixes #2218) + * Support checksum algorithm CRC64NVME (fixes #2334) * Refactorings * TBD * Version updates (deliverable dependencies) @@ -149,6 +150,18 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav * Version updates (build dependencies) * TBD +## 4.1.1 +Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration. + +* Features and fixes + * Content-Encoding: aws-chunked should not be stored (fixes #2218) +* Refactorings + * none +* Version updates (deliverable dependencies) + * none +* Version updates (build dependencies) + * none + ## 4.1.0 Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration. diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEncodingIT.kt similarity index 74% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingIT.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEncodingIT.kt index 77756dceb..789190af7 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingIT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEncodingIT.kt @@ -30,9 +30,9 @@ import java.io.InputStream /** * Chunked encoding with signing is only active in AWS SDK v2 when endpoint is http */ -internal class AwsChunkedEndcodingIT : S3TestBase() { +internal class AwsChunkedEncodingIT : S3TestBase() { - private val s3Client = createS3Client(serviceEndpointHttp) + private val s3Client = createS3Client(serviceEndpointHttp, true) /** * Unfortunately the S3 API does not persist or return data that would let us verify if signed and chunked encoding @@ -68,11 +68,11 @@ internal class AwsChunkedEndcodingIT : S3TestBase() { s3Client.getObject { it.bucket(bucket) it.key(UPLOAD_FILE_NAME) - }.also { getObjectResponse -> - assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag) - assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length()) - - getObjectResponse.response().checksumSHA256().also { + }.also { + assertThat(it.response().eTag()).isEqualTo(expectedEtag) + assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) + assertThat(it.response().contentEncoding()).isNotEqualTo("aws-chunked") + it.response().checksumSHA256().also { assertThat(it).isNotBlank assertThat(it).isEqualTo(expectedChecksum) } @@ -109,6 +109,35 @@ internal class AwsChunkedEndcodingIT : S3TestBase() { }.also { assertThat(it.response().eTag()).isEqualTo(expectedEtag) assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) + assertThat(it.response().contentEncoding()).isNotEqualTo("aws-chunked") + } + } + + @Test + @S3VerifiedFailure( + year = 2023, + reason = "Only works with http endpoints" + ) + fun `put object creates correct content-encoding, get object returns content-encoding`(testInfo: TestInfo) { + val bucket = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + + val customEncoding = "my-custom-encoding" + + s3Client.putObject( + { + it.bucket(bucket) + it.key(UPLOAD_FILE_NAME) + it.contentEncoding(customEncoding) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.getObject { + it.bucket(bucket) + it.key(UPLOAD_FILE_NAME) + }.also { + assertThat(it.response().contentEncoding()).isEqualTo(customEncoding) } } } diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt index 36e7c69d2..84b1dd5bb 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt @@ -227,7 +227,6 @@ internal abstract class S3TestBase { @AfterEach fun cleanupStores() { for (bucket in _s3Client.listBuckets().buckets()) { - if(bucket.name() == "testputandgetretention-545488000") {return} //Empty all buckets deleteMultipartUploads(bucket) deleteObjectsInBucket(bucket, isObjectLockEnabled(bucket)) diff --git a/server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java b/server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java index 6fb63d70d..3b94ca1ae 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java +++ b/server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java @@ -24,8 +24,8 @@ import static com.adobe.testing.s3mock.S3Exception.BAD_REQUEST_CONTENT; import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_DECODED_CONTENT_LENGTH; import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromSdk; -import static com.adobe.testing.s3mock.util.HeaderUtil.isChunked; -import static com.adobe.testing.s3mock.util.HeaderUtil.isChunkedAndV4Signed; +import static com.adobe.testing.s3mock.util.HeaderUtil.isChunkedEncoding; +import static com.adobe.testing.s3mock.util.HeaderUtil.isV4Signed; import com.adobe.testing.s3mock.dto.ChecksumAlgorithm; import com.adobe.testing.s3mock.util.AbstractAwsInputStream; @@ -93,9 +93,9 @@ public Pair toTempFile(InputStream inputStream) { private InputStream wrapStream(InputStream dataStream, HttpHeaders headers) { var lengthHeader = headers.getFirst(X_AMZ_DECODED_CONTENT_LENGTH); var length = lengthHeader == null ? -1 : Long.parseLong(lengthHeader); - if (isChunkedAndV4Signed(headers)) { + if (isV4Signed(headers)) { return new AwsChunkedDecodingChecksumInputStream(dataStream, length); - } else if (isChunked(headers)) { + } else if (isChunkedEncoding(headers)) { return new AwsUnsignedChunkedDecodingChecksumInputStream(dataStream, length); } else { return dataStream; diff --git a/server/src/main/java/com/adobe/testing/s3mock/util/HeaderUtil.java b/server/src/main/java/com/adobe/testing/s3mock/util/HeaderUtil.java index 73611f4d6..8ecf507c0 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/util/HeaderUtil.java +++ b/server/src/main/java/com/adobe/testing/s3mock/util/HeaderUtil.java @@ -116,7 +116,8 @@ public static Map storeHeadersFrom(HttpHeaders headers) { header -> (equalsIgnoreCase(header, HttpHeaders.EXPIRES) || equalsIgnoreCase(header, HttpHeaders.CONTENT_LANGUAGE) || equalsIgnoreCase(header, HttpHeaders.CONTENT_DISPOSITION) - || equalsIgnoreCase(header, HttpHeaders.CONTENT_ENCODING) + || (equalsIgnoreCase(header, HttpHeaders.CONTENT_ENCODING) + && !isOnlyChunkedEncoding(headers)) || equalsIgnoreCase(header, HttpHeaders.CACHE_CONTROL) )); } @@ -152,19 +153,35 @@ && isNotBlank(entry.getValue().get(0))) { .collect(Collectors.toMap(SimpleEntry::getKey, SimpleEntry::getValue)); } - public static boolean isChunkedAndV4Signed(HttpHeaders headers) { + public static boolean isV4Signed(HttpHeaders headers) { var sha256Header = headers.getFirst(X_AMZ_CONTENT_SHA256); return sha256Header != null && (sha256Header.equals(STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD) || sha256Header.equals(STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD_TRAILER)); } - public static boolean isChunked(HttpHeaders headers) { + public static boolean isChunkedEncoding(HttpHeaders headers) { var contentEncodingHeaders = headers.get(HttpHeaders.CONTENT_ENCODING); return (contentEncodingHeaders != null && (contentEncodingHeaders.contains(AWS_CHUNKED))); } + /** + * Check if aws-chunked is the only "Content-Encoding" header. + * + * If aws-chunked is the only value that you pass in the content-encoding header, S3 considers + * the content-encoding header empty and does not return this header when your retrieve the + * object. + * + * See API + */ + private static boolean isOnlyChunkedEncoding(HttpHeaders headers) { + var contentEncodingHeaders = headers.get(HttpHeaders.CONTENT_ENCODING); + return (contentEncodingHeaders != null + && contentEncodingHeaders.size() == 1 + && (contentEncodingHeaders.contains(AWS_CHUNKED))); + } + public static MediaType mediaTypeFrom(final String contentType) { try { return MediaType.parseMediaType(contentType);