Merge pull request #1837 from adobe/1818-correctly-detect-eol-in-chunked-uploads

Correctly detect EOL in chunked uploads

Author: afranken

CHANGELOG.md

@@ -134,7 +134,7 @@ Version 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
 * Version updates
   * TBD
 
-## 3.7.3 - PLANNED
+## 3.7.3
 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
 
 * Features and fixes

integration-tests/src/test/resources/sampleFile_large.txt

@@ -43,6 +43,16 @@ public static ChecksumAlgorithm fromString(String value) {
     };
   }
 
+  public static ChecksumAlgorithm fromHeader(String value) {
+    return switch (value) {
+      case "x-amz-checksum-sha256" -> SHA256;
+      case "x-amz-checksum-sha1" -> SHA1;
+      case "x-amz-checksum-crc32" -> CRC32;
+      case "x-amz-checksum-crc32c" -> CRC32C;
+      default -> null;
+    };
+  }
+
   public Algorithm toAlgorithm() {
     return switch (this) {
       case CRC32 -> Algorithm.CRC32;

server/src/main/java/com/adobe/testing/s3mock/dto/ChecksumAlgorithm.java

@@ -24,7 +24,9 @@
 import static com.adobe.testing.s3mock.S3Exception.NOT_MODIFIED;
 import static com.adobe.testing.s3mock.S3Exception.NO_SUCH_KEY;
 import static com.adobe.testing.s3mock.S3Exception.PRECONDITION_FAILED;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_DECODED_CONTENT_LENGTH;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_TRAILER;
 import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromSdk;
 import static com.adobe.testing.s3mock.util.HeaderUtil.isChunked;
 import static com.adobe.testing.s3mock.util.HeaderUtil.isChunkedAndV4Signed;
@@ -37,6 +39,7 @@
 import com.adobe.testing.s3mock.dto.Delete;
 import com.adobe.testing.s3mock.dto.DeleteResult;
 import com.adobe.testing.s3mock.dto.DeletedS3Object;
+import com.adobe.testing.s3mock.dto.Error;
 import com.adobe.testing.s3mock.dto.LegalHold;
 import com.adobe.testing.s3mock.dto.Owner;
 import com.adobe.testing.s3mock.dto.Retention;
@@ -45,9 +48,7 @@
 import com.adobe.testing.s3mock.store.BucketStore;
 import com.adobe.testing.s3mock.store.ObjectStore;
 import com.adobe.testing.s3mock.store.S3ObjectMetadata;
-import com.adobe.testing.s3mock.util.AbstractAwsInputStream;
-import com.adobe.testing.s3mock.util.AwsChunkedDecodingChecksumInputStream;
-import com.adobe.testing.s3mock.util.AwsUnsignedChunkedDecodingChecksumInputStream;
+import com.adobe.testing.s3mock.util.AwsChunkedInputStream;
 import com.adobe.testing.s3mock.util.DigestUtil;
 import java.io.IOException;
 import java.io.InputStream;
@@ -161,7 +162,7 @@ public DeleteResult deleteObjects(String bucketName, Delete delete) {
         response.addDeletedObject(DeletedS3Object.from(object));
       } catch (IllegalStateException e) {
         response.addError(
-            new com.adobe.testing.s3mock.dto.Error("InternalError",
+            new Error("InternalError",
                 object.key(),
                 "We encountered an internal error. Please try again.",
                 object.versionId()));
@@ -266,13 +267,13 @@ public void verifyRetention(Retention retention) {
 
   public Pair<Path, String> toTempFile(InputStream inputStream, HttpHeaders httpHeaders) {
     try {
-      var tempFile = Files.createTempFile("tempObject", "");
+      var tempFile = Files.createTempFile("ObjectService", "toTempFile");
       try (OutputStream os = Files.newOutputStream(tempFile)) {
         InputStream wrappedStream = wrapStream(inputStream, httpHeaders);
         wrappedStream.transferTo(os);
         ChecksumAlgorithm algorithmFromSdk = checksumAlgorithmFromSdk(httpHeaders);
         if (algorithmFromSdk != null
-            && wrappedStream instanceof AbstractAwsInputStream awsInputStream) {
+            && wrappedStream instanceof AwsChunkedInputStream awsInputStream) {
           return Pair.of(tempFile, awsInputStream.getChecksum());
         }
         return Pair.of(tempFile, null);
@@ -291,11 +292,11 @@ public void verifyChecksum(Path path, String checksum, ChecksumAlgorithm checksu
 
   InputStream wrapStream(InputStream dataStream, HttpHeaders headers) {
     var lengthHeader = headers.getFirst(X_AMZ_DECODED_CONTENT_LENGTH);
+    var trailHeader = headers.getOrEmpty(X_AMZ_TRAILER);
+    var hasChecksum = trailHeader.stream().anyMatch(h -> h.contains(X_AMZ_CHECKSUM));
     var length = lengthHeader == null ? -1 : Long.parseLong(lengthHeader);
-    if (isChunkedAndV4Signed(headers)) {
-      return new AwsChunkedDecodingChecksumInputStream(dataStream, length);
-    } else if (isChunked(headers)) {
-      return new AwsUnsignedChunkedDecodingChecksumInputStream(dataStream, length);
+    if (isChunkedAndV4Signed(headers) || isChunked(headers)) {
+      return new AwsChunkedInputStream(dataStream, length, hasChecksum);
     } else {
       return dataStream;
     }