blobstore: checksum support for mpu

blob/blob-ali/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>org.wiremock</groupId>
             <artifactId>wiremock</artifactId>
-            <version>3.13.2</version>
+            <version>3.12.1</version>
             <scope>test</scope>
         </dependency>
         <dependency>

blob/blob-aws/pom.xml

@@ -18,7 +18,7 @@
         <dependency>
             <groupId>org.wiremock</groupId>
             <artifactId>wiremock</artifactId>
-            <version>3.13.2</version>
+            <version>3.12.1</version>
             <scope>test</scope>
         </dependency>
         <dependency>

blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java

@@ -44,6 +44,7 @@
 import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
 import software.amazon.awssdk.core.ResponseInputStream;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import software.amazon.awssdk.retries.StandardRetryStrategy;
@@ -420,32 +421,44 @@ public CreateMultipartUploadRequest toCreateMultipartUploadRequest(
     }
     // else: no SSE headers; S3 uses bucket default encryption
 
+    if (request.isChecksumEnabled()) {
+      builder.checksumAlgorithm(CRC32_C);
+    }
+
     return builder.build();
   }
 
   public UploadPartRequest toUploadPartRequest(MultipartUpload mpu, MultipartPart mpp) {
-    return UploadPartRequest.builder()
+    UploadPartRequest.Builder builder = UploadPartRequest.builder()
         .bucket(getBucket())
         .key(mpu.getKey())
         .uploadId(mpu.getId())
         .partNumber(mpp.getPartNumber())
-        .contentLength(mpp.getContentLength())
-        .build();
+        .contentLength(mpp.getContentLength());
+
+    if (!StringUtils.isEmpty(mpp.getChecksumValue())) {
+      builder.checksumAlgorithm(CRC32_C);
+      builder.checksumCRC32C(mpp.getChecksumValue());
+    }
+
+    return builder.build();
   }
 
   public CompleteMultipartUploadRequest toCompleteMultipartUploadRequest(
       MultipartUpload mpu, List<UploadPartResponse> parts) {
 
-    List<CompletedPart> completedParts =
-        parts.stream()
-            .sorted(Comparator.comparingInt(UploadPartResponse::getPartNumber))
-            .map(
-                part ->
-                    CompletedPart.builder()
-                        .partNumber(part.getPartNumber())
-                        .eTag(part.getEtag())
-                        .build())
-            .collect(Collectors.toList());
+    List<CompletedPart> completedParts = parts.stream()
+        .sorted(Comparator.comparingInt(UploadPartResponse::getPartNumber))
+        .map(part -> {
+          CompletedPart.Builder partBuilder = CompletedPart.builder()
+              .partNumber(part.getPartNumber())
+              .eTag(part.getEtag());
+          if (StringUtils.isNotEmpty(part.getChecksumValue())) {
+            partBuilder.checksumCRC32C(part.getChecksumValue());
+          }
+          return partBuilder.build();
+        })
+        .collect(Collectors.toList());
 
     return CompleteMultipartUploadRequest.builder()
         .bucket(getBucket())
@@ -645,26 +658,33 @@ public CopyResponse toCopyResponse(String destKey, CopyObjectResponse response)
         .build();
   }
 
-  public MultipartUpload toMultipartUpload(
-      MultipartUploadRequest request, CreateMultipartUploadResponse response) {
+  public MultipartUpload toMultipartUpload(MultipartUploadRequest request,
+                                           CreateMultipartUploadResponse response) {
     return MultipartUpload.builder()
         .bucket(response.bucket())
         .key(response.key())
         .id(response.uploadId())
         .metadata(request.getMetadata())
         .tags(request.getTags())
         .kmsKeyId(request.getKmsKeyId())
+        .checksumEnabled(request.isChecksumEnabled())
         .build();
   }
 
   public UploadPartResponse toUploadPartResponse(
       MultipartPart part, software.amazon.awssdk.services.s3.model.UploadPartResponse response) {
-    return new UploadPartResponse(part.getPartNumber(), response.eTag(), part.getContentLength());
+    return new UploadPartResponse(
+        part.getPartNumber(), response.eTag(), part.getContentLength(),
+        response.checksumCRC32C());
   }
 
   public MultipartUploadResponse toMultipartUploadResponse(
       CompleteMultipartUploadResponse response) {
-    return new MultipartUploadResponse(response.eTag());
+    String checksumValue = null;
+    if (response.checksumCRC32C() != null) {
+      checksumValue = response.checksumCRC32C();
+    }
+    return new MultipartUploadResponse(response.eTag(), checksumValue);
   }
 
   /**

blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsBlobStoreTest.java

@@ -86,6 +86,7 @@
 import software.amazon.awssdk.services.s3.S3ClientBuilder;
 import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.AbortMultipartUploadResponse;
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse;
 import software.amazon.awssdk.services.s3.model.CompletedPart;
@@ -1097,13 +1098,82 @@ void testDoAbortMultipartUpload() {
     assertEquals("mpu-id", actualRequest.uploadId());
   }
 
+  @Test
+  void testMultipartUploadWithChecksum() {
+    // Step 1: Initiate multipart upload with checksum enabled
+    CreateMultipartUploadResponse mockCreateResponse = mock(CreateMultipartUploadResponse.class);
+    doReturn("bucket-1").when(mockCreateResponse).bucket();
+    doReturn("object-1").when(mockCreateResponse).key();
+    doReturn("mpu-id").when(mockCreateResponse).uploadId();
+    when(mockS3Client.createMultipartUpload((CreateMultipartUploadRequest) any())).thenReturn(
+        mockCreateResponse);
+
+    MultipartUploadRequest request = new MultipartUploadRequest.Builder()
+        .withKey("object-1")
+        .withChecksumEnabled(true)
+        .build();
+
+    MultipartUpload mpu = aws.initiateMultipartUpload(request);
+
+    // Verify checksumAlgorithm is set on create request
+    ArgumentCaptor<CreateMultipartUploadRequest> createCaptor =
+        ArgumentCaptor.forClass(CreateMultipartUploadRequest.class);
+    verify(mockS3Client, times(1)).createMultipartUpload(createCaptor.capture());
+    assertEquals(ChecksumAlgorithm.CRC32_C, createCaptor.getValue().checksumAlgorithm());
+    assertTrue(mpu.isChecksumEnabled());
+
+    // Step 2: Upload part with checksum
+    UploadPartResponse mockUploadPartResponse = mock(UploadPartResponse.class);
+    doReturn("part-etag").when(mockUploadPartResponse).eTag();
+    doReturn("AAAAAA==").when(mockUploadPartResponse).checksumCRC32C();
+    doReturn(mockUploadPartResponse).when(mockS3Client)
+        .uploadPart(any(UploadPartRequest.class), any(RequestBody.class));
+
+    byte[] content = "This is test data".getBytes(StandardCharsets.UTF_8);
+    MultipartPart part = new MultipartPart(1, content, "AAAAAA==");
+
+    var uploadPartResp = aws.uploadMultipartPart(mpu, part);
+
+    // Verify checksumCRC32C is set on upload part request
+    ArgumentCaptor<UploadPartRequest> uploadCaptor =
+        ArgumentCaptor.forClass(UploadPartRequest.class);
+    verify(mockS3Client, times(1)).uploadPart(uploadCaptor.capture(), any(RequestBody.class));
+    assertEquals(ChecksumAlgorithm.CRC32_C, uploadCaptor.getValue().checksumAlgorithm());
+    assertEquals("AAAAAA==", uploadCaptor.getValue().checksumCRC32C());
+
+    // Verify checksum in upload part response
+    assertEquals("AAAAAA==", uploadPartResp.getChecksumValue());
+
+    // Step 3: Complete multipart upload with checksum
+    CompleteMultipartUploadResponse mockCompleteResponse =
+        mock(CompleteMultipartUploadResponse.class);
+    doReturn("complete-etag").when(mockCompleteResponse).eTag();
+    doReturn("composite-checksum==").when(mockCompleteResponse).checksumCRC32C();
+    doReturn(mockCompleteResponse).when(mockS3Client)
+        .completeMultipartUpload((CompleteMultipartUploadRequest) any());
+
+    List<com.salesforce.multicloudj.blob.driver.UploadPartResponse> parts =
+        List.of(new com.salesforce.multicloudj.blob.driver.UploadPartResponse(1, "part-etag",
+            content.length, "AAAAAA=="));
+
+    MultipartUploadResponse completeResp = aws.completeMultipartUpload(mpu, parts);
+
+    // Verify checksumCRC32C is set on completed part
+    ArgumentCaptor<CompleteMultipartUploadRequest> completeCaptor =
+        ArgumentCaptor.forClass(CompleteMultipartUploadRequest.class);
+    verify(mockS3Client, times(1)).completeMultipartUpload(completeCaptor.capture());
+    List<CompletedPart> completedParts = completeCaptor.getValue().multipartUpload().parts();
+    assertEquals("AAAAAA==", completedParts.get(0).checksumCRC32C());
+
+    // Verify composite checksum in response
+    assertEquals("composite-checksum==", completeResp.getChecksumValue());
+  }
+
   @Test
   void testDoGetTags() {
     GetObjectTaggingResponse mockResponse = mock(GetObjectTaggingResponse.class);
-    List<Tag> tags =
-        List.of(
-            Tag.builder().key("key1").value("value1").build(),
-            Tag.builder().key("key2").value("value2").build());
+    List<Tag> tags = List.of(Tag.builder().key("key1").value("value1").build(),
+        Tag.builder().key("key2").value("value2").build());
     doReturn(tags).when(mockResponse).tagSet();
     doReturn(mockResponse).when(mockS3Client).getObjectTagging((GetObjectTaggingRequest) any());
 
@@ -1213,7 +1283,7 @@ void testDoDoesObjectExist() {
   void testDoDoesBucketExist() {
     HeadBucketResponse mockResponse = mock(HeadBucketResponse.class);
     when(mockS3Client.headBucket(
-            ArgumentMatchers.<java.util.function.Consumer<HeadBucketRequest.Builder>>any()))
+        ArgumentMatchers.<java.util.function.Consumer<HeadBucketRequest.Builder>>any()))
         .thenReturn(mockResponse);
 
     boolean result = aws.doDoesBucketExist();

blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testmultipartupload_withchecksum-delete-0.json

@@ -0,0 +1,20 @@
+{
+  "id" : "a779b0a1-022c-450c-bd6f-5e6041047169",
+  "name" : "AwsBlobStoreIT_testMultipartUpload_withChecksum-DELETE-0",
+  "request" : {
+    "url" : "/chameleon-jcloud/conformance-tests/multipart-withChecksum",
+    "method" : "DELETE"
+  },
+  "response" : {
+    "status" : 204,
+    "headers" : {
+      "Server" : "AmazonS3",
+      "x-amz-request-id" : "DDK10V547NXC6TPY",
+      "x-amz-id-2" : "4Pt7gMty3ZC7zWWnuq+ghaHDRu1Q4k25buEL6aEakFwESV2B91gsx6WUWqA9c+TiaHdxH1o3kO8=",
+      "Date" : "Mon, 09 Mar 2026 18:31:47 GMT"
+    }
+  },
+  "uuid" : "a779b0a1-022c-450c-bd6f-5e6041047169",
+  "persistent" : true,
+  "insertionIndex" : 598
+}

blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testmultipartupload_withchecksum-head-1.json

@@ -0,0 +1,26 @@
+{
+  "id" : "f794524f-b1d5-4b37-ad36-0f7ad79b3638",
+  "name" : "AwsBlobStoreIT_testMultipartUpload_withChecksum-HEAD-1",
+  "request" : {
+    "url" : "/chameleon-jcloud/conformance-tests/multipart-withChecksum",
+    "method" : "HEAD"
+  },
+  "response" : {
+    "status" : 200,
+    "headers" : {
+      "Accept-Ranges" : "bytes",
+      "x-amz-meta-key1" : "value1",
+      "Server" : "AmazonS3",
+      "ETag" : "\"63a814bc1f352d8149b1dd9f3747e89e-2\"",
+      "Last-Modified" : "Mon, 09 Mar 2026 18:31:37 GMT",
+      "x-amz-request-id" : "DDK6G2ZVBH6SKR9J",
+      "x-amz-server-side-encryption" : "AES256",
+      "x-amz-id-2" : "u4rU0eGuH02xa5BG6xwucQYz6+b3B4KUkYo2YG76foWuyQWOpIdFE4tLhn89cQ1Za/l9ROPb8ZY=",
+      "Date" : "Mon, 09 Mar 2026 18:31:47 GMT",
+      "Content-Type" : "binary/octet-stream"
+    }
+  },
+  "uuid" : "f794524f-b1d5-4b37-ad36-0f7ad79b3638",
+  "persistent" : true,
+  "insertionIndex" : 599
+}

blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testmultipartupload_withchecksum-post-2.json

@@ -0,0 +1,38 @@
+{
+  "id" : "95d09e13-e409-431c-bb5b-578818b95b47",
+  "name" : "AwsBlobStoreIT_testMultipartUpload_withChecksum-POST-2",
+  "request" : {
+    "urlPath" : "/chameleon-jcloud/conformance-tests/multipart-withChecksum",
+    "method" : "POST",
+    "headers" : {
+      "X-Query-Param-Count" : {
+        "equalTo" : "1"
+      }
+    },
+    "queryParameters" : {
+      "uploadId" : {
+        "hasExactly" : [ {
+          "equalTo" : "AmDIoyx_Brtnc8MngJfF8Ldb8Co6txGSCKT4anxsRG9Bk_lQW8.tNcmszBvSa01u_VKyvYmwrwKCfpwcVZWh57hliCXojDAohhoPAJsUFAbwaNGVhvvXmGpB7LPxgD4KpBrCGwHbT9gD_JZobR14JmE5ytpE7jOBE951v5SeNtPgknM_j.CfIqazLQpJzIBIgwxt8tB.CbU3M6ToIqjD8Q--"
+        } ]
+      }
+    },
+    "bodyPatterns" : [ {
+      "equalToXml" : "<?xml version=\"1.0\" encoding=\"UTF-8\"?><CompleteMultipartUpload xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Part><ETag>&quot;2b5c91ad399409ec9b409b66f5bb00fb&quot;</ETag><ChecksumCRC32C>OGE8Qg==</ChecksumCRC32C><PartNumber>1</PartNumber></Part><Part><ETag>&quot;267c038cc2ec32ffd0ee2512ff5da5a5&quot;</ETag><ChecksumCRC32C>RCmziw==</ChecksumCRC32C><PartNumber>2</PartNumber></Part></CompleteMultipartUpload>"
+    } ]
+  },
+  "response" : {
+    "status" : 200,
+    "body" : "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\n<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Location>https://s3.us-west-2.amazonaws.com/chameleon-jcloud/conformance-tests%2Fmultipart-withChecksum</Location><Bucket>chameleon-jcloud</Bucket><Key>conformance-tests/multipart-withChecksum</Key><ETag>\"63a814bc1f352d8149b1dd9f3747e89e-2\"</ETag><ChecksumCRC32C>gDcnRA==-2</ChecksumCRC32C><ChecksumType>COMPOSITE</ChecksumType></CompleteMultipartUploadResult>",
+    "headers" : {
+      "Server" : "AmazonS3",
+      "x-amz-request-id" : "DDK8YFWABC3D0TAR",
+      "x-amz-server-side-encryption" : "AES256",
+      "x-amz-id-2" : "8Ngcx6C3AE/MnZTFCQJD71i6vKdsI1Q3Ri8vrEUt5YCtFEMgf1nQwPzQaEfJP88zjVpJw7nKIBc=",
+      "Date" : "Mon, 09 Mar 2026 18:31:47 GMT",
+      "Content-Type" : "application/xml"
+    }
+  },
+  "uuid" : "95d09e13-e409-431c-bb5b-578818b95b47",
+  "persistent" : true,
+  "insertionIndex" : 600
+}

blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testmultipartupload_withchecksum-post-5.json

@@ -0,0 +1,36 @@
+{
+  "id" : "f464dca9-31a9-448d-b049-afefc0dbfd34",
+  "name" : "AwsBlobStoreIT_testMultipartUpload_withChecksum-POST-5",
+  "request" : {
+    "urlPath" : "/chameleon-jcloud/conformance-tests/multipart-withChecksum",
+    "method" : "POST",
+    "headers" : {
+      "X-Query-Param-Count" : {
+        "equalTo" : "1"
+      }
+    },
+    "queryParameters" : {
+      "uploads" : {
+        "hasExactly" : [ {
+          "equalTo" : ""
+        } ]
+      }
+    }
+  },
+  "response" : {
+    "status" : 200,
+    "base64Body" : "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPEluaXRpYXRlTXVsdGlwYXJ0VXBsb2FkUmVzdWx0IHhtbG5zPSJodHRwOi8vczMuYW1hem9uYXdzLmNvbS9kb2MvMjAwNi0wMy0wMS8iPjxCdWNrZXQ+Y2hhbWVsZW9uLWpjbG91ZDwvQnVja2V0PjxLZXk+Y29uZm9ybWFuY2UtdGVzdHMvbXVsdGlwYXJ0LXdpdGhDaGVja3N1bTwvS2V5PjxVcGxvYWRJZD5odzNoMDNJNFEyZWdNbWlpRlFBdEM2RnM5YnVNZDh3ZHlMV3M2ZU9mNmprLjdmS2RhTWFyb29NRzQzbXBTZEZCLnFvemw4Smh3NDFGOU5faVRHNlFmbk1DdmZSaFNKUGE2T29iNHFndzFfOFBPUXZUUUNkNEVpSDlnVzZXNzh0MXN3TWlWVldMSFV0QUZJNjRSR1pPXy5RMUZJem9WOFNKdktESzdOWlZZQnZ2a0FFbE8yT0lrZEp6Yk9ydW1qa1dpWl9Fc1BlTG0wQXV3Y3ouOE8xYWdRLS08L1VwbG9hZElkPjwvSW5pdGlhdGVNdWx0aXBhcnRVcGxvYWRSZXN1bHQ+",
+    "headers" : {
+      "Server" : "AmazonS3",
+      "x-amz-checksum-algorithm" : "CRC32C",
+      "x-amz-checksum-type" : "COMPOSITE",
+      "x-amz-request-id" : "V0WDZJAT39CJR61M",
+      "x-amz-server-side-encryption" : "AES256",
+      "x-amz-id-2" : "Sp3hzSucQji6jc1oHfQFeJLuQgouuaCsJMdwcisA2hmrBq2J3by5736h3sEPxrVnKLoQlHq+PTk=",
+      "Date" : "Fri, 06 Mar 2026 17:43:37 GMT"
+    }
+  },
+  "uuid" : "f464dca9-31a9-448d-b049-afefc0dbfd34",
+  "persistent" : true,
+  "insertionIndex" : 6
+}

blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testmultipartupload_withchecksum-post-6.json

@@ -0,0 +1,36 @@
+{
+  "id" : "b9d3c7c9-26d0-4fcb-ba86-389e232a3892",
+  "name" : "AwsBlobStoreIT_testMultipartUpload_withChecksum-POST-6",
+  "request" : {
+    "urlPath" : "/chameleon-jcloud/conformance-tests/multipart-withChecksum",
+    "method" : "POST",
+    "headers" : {
+      "X-Query-Param-Count" : {
+        "equalTo" : "1"
+      }
+    },
+    "queryParameters" : {
+      "uploads" : {
+        "hasExactly" : [ {
+          "equalTo" : ""
+        } ]
+      }
+    }
+  },
+  "response" : {
+    "status" : 200,
+    "base64Body" : "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPEluaXRpYXRlTXVsdGlwYXJ0VXBsb2FkUmVzdWx0IHhtbG5zPSJodHRwOi8vczMuYW1hem9uYXdzLmNvbS9kb2MvMjAwNi0wMy0wMS8iPjxCdWNrZXQ+Y2hhbWVsZW9uLWpjbG91ZDwvQnVja2V0PjxLZXk+Y29uZm9ybWFuY2UtdGVzdHMvbXVsdGlwYXJ0LXdpdGhDaGVja3N1bTwvS2V5PjxVcGxvYWRJZD5BbURJb3l4X0JydG5jOE1uZ0pmRjhMZGI4Q282dHhHU0NLVDRhbnhzUkc5QmtfbFFXOC50TmNtc3pCdlNhMDF1X1ZLeXZZbXdyd0tDZnB3Y1ZaV2g1N2hsaUNYb2pEQW9oaG9QQUpzVUZBYndhTkdWaHZ2WG1HcEI3TFB4Z0Q0S3BCckNHd0hiVDlnRF9KWm9iUjE0Sm1FNXl0cEU3ak9CRTk1MXY1U2VOdFBna25NX2ouQ2ZJcWF6TFFwSnpJQklnd3h0OHRCLkNiVTNNNlRvSXFqRDhRLS08L1VwbG9hZElkPjwvSW5pdGlhdGVNdWx0aXBhcnRVcGxvYWRSZXN1bHQ+",
+    "headers" : {
+      "Server" : "AmazonS3",
+      "x-amz-checksum-algorithm" : "CRC32C",
+      "x-amz-checksum-type" : "COMPOSITE",
+      "x-amz-request-id" : "WYNT8AK9BKFQD5VV",
+      "x-amz-server-side-encryption" : "AES256",
+      "x-amz-id-2" : "g2fQ1L4As/blF/V0cb3qIPw3aCSKN9tNBpQN7B1NabibWSTrHwAoku07Se19iAQ+8fwXVle7lbk=",
+      "Date" : "Mon, 09 Mar 2026 18:31:37 GMT"
+    }
+  },
+  "uuid" : "b9d3c7c9-26d0-4fcb-ba86-389e232a3892",
+  "persistent" : true,
+  "insertionIndex" : 604
+}