Support POST Object requests

CHANGELOG.md

@@ -12,7 +12,7 @@
   * [Planned changes](#planned-changes)
 * [CURRENT - 4.x - THIS VERSION IS UNDER ACTIVE DEVELOPMENT](#current---4x---this-version-is-under-active-development)
   * [4.1.1 - PLANNED](#411---planned)
-  * [4.1.0 - PLANNED](#410---planned)
+  * [4.1.0](#410)
   * [4.0.0](#400)
 * [DEPRECATED - 3.x](#deprecated---3x)
   * [3.12.0](#3120)
@@ -149,7 +149,7 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
 * Version updates (build dependencies)
   * TBD
 
-## 4.1.0 - PLANNED
+## 4.1.0
 Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
 
 * Features and fixes
@@ -160,8 +160,11 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
     * These were corner cases where error messages were incorrect, or proper validations were missing.
   * Migrate all integration tests to AWS SDK v2, remove AWS SDK v1 tests from the integration-tests module
 * Version updates (deliverable dependencies)
-  * TBD
+  * Bump aws-v2.version from 2.31.17 to 2.31.25
+  * Bump aws.sdk.kotlin:s3-jvm from 1.4.59 to 1.4.67
+  * Bump commons-io:commons-io from 2.18.0 to 2.19.0
 * Version updates (build dependencies)
+  * Bump step-security/harden-runner from 2.11.1 to 2.12.0
   * Bump actions/setup-java from 4.7.0 to 4.7.1
 
 ## 4.0.0

build-config/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>4.0.1-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>s3mock-build-config</artifactId>

docker/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>4.0.1-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>s3mock-docker</artifactId>

integration-tests/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>4.0.1-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>s3mock-integration-tests</artifactId>
@@ -123,6 +123,31 @@
       <artifactId>s3-jvm</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpmime</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <!--
+        Workaround, AWS SDK for Java v2 currently does not support presigned URLs for POST requests.
+        see https://github.com/aws/aws-sdk-java-v2/issues/1493
+       -->
+      <groupId>tel.schich</groupId>
+      <artifactId>aws-s3-post-object-presigner</artifactId>
+      <version>0.1.1</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PresignedUrlIT.kt

@@ -18,13 +18,17 @@ package com.adobe.testing.s3mock.its
 import com.adobe.testing.s3mock.dto.InitiateMultipartUploadResult
 import com.adobe.testing.s3mock.util.DigestUtil
 import org.apache.http.HttpHeaders
+import org.apache.http.HttpHeaders.CONTENT_TYPE
 import org.apache.http.HttpStatus
 import org.apache.http.client.methods.HttpDelete
 import org.apache.http.client.methods.HttpGet
 import org.apache.http.client.methods.HttpPost
 import org.apache.http.client.methods.HttpPut
+import org.apache.http.entity.ByteArrayEntity
+import org.apache.http.entity.ContentType
 import org.apache.http.entity.FileEntity
 import org.apache.http.entity.StringEntity
+import org.apache.http.entity.mime.MultipartEntityBuilder
 import org.apache.http.impl.client.CloseableHttpClient
 import org.apache.http.message.BasicHeader
 import org.assertj.core.api.Assertions.assertThat
@@ -34,6 +38,8 @@ import software.amazon.awssdk.core.sync.RequestBody
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.services.s3.model.CompletedPart
 import software.amazon.awssdk.services.s3.presigner.S3Presigner
+import tel.schich.awss3postobjectpresigner.S3PostObjectPresigner
+import tel.schich.awss3postobjectpresigner.S3PostObjectRequest
 import java.io.File
 import java.nio.file.Files
 import java.nio.file.Path
@@ -44,6 +50,62 @@ internal class PresignedUrlIT : S3TestBase() {
   private val httpClient: CloseableHttpClient = createHttpClient()
   private val s3Client: S3Client = createS3Client()
   private val s3Presigner: S3Presigner = createS3Presigner()
+  private val s3PostObjectPresigner: S3PostObjectPresigner = createS3PostObjectPresigner()
+
+  @Test
+  @S3VerifiedFailure(year = 2025,
+    reason = "S3PostObjectPresigner does not create working presigned URLs.")
+  fun testPresignedUrl_postObject_largeFile(testInfo: TestInfo) {
+    val key = randomName
+    val bucketName = givenBucket(testInfo)
+
+    val presignedUrlString = s3PostObjectPresigner.presignPost(
+      S3PostObjectRequest
+        .builder()
+        .bucket(bucketName)
+        .expiration(Duration.ofMinutes(1L))
+        .build()
+    ).uri().toString()
+
+    assertThat(presignedUrlString).isNotBlank()
+
+    val randomMBytes = randomMBytes(20)
+    HttpPost(presignedUrlString).apply {
+        this.entity = MultipartEntityBuilder.create()
+          .addTextBody("key", key)
+          .addTextBody(CONTENT_TYPE, "application/octet-stream")
+          //.addTextBody(X_AMZ_STORAGE_CLASS, "INTELLIGENT_TIERING")
+          .addTextBody("tagging", "<Tagging><TagSet><Tag><Key>Tag Name</Key><Value>Tag Value</Value></Tag></TagSet></Tagging>")
+          .addBinaryBody("file", randomMBytes.inputStream(), ContentType.APPLICATION_OCTET_STREAM, key)
+          .build()
+      }.also { post ->
+      httpClient.execute(
+        post
+      ).use {
+        assertThat(it.statusLine.statusCode).isEqualTo(HttpStatus.SC_OK)
+        val expectedEtag = "\"${DigestUtil.hexDigest(randomMBytes.inputStream())}\""
+        val actualEtag = it.getFirstHeader(HttpHeaders.ETAG).value
+        assertThat(actualEtag).isEqualTo(expectedEtag)
+      }
+    }
+
+    s3Client.getObject {
+      it.bucket(bucketName)
+      it.key(key)
+    }.use {
+      val expectedEtag = "\"${DigestUtil.hexDigest(randomMBytes.inputStream())}\""
+      val actualEtag = "\"${DigestUtil.hexDigest(it)}\""
+      assertThat(actualEtag).isEqualTo(expectedEtag)
+    }
+    s3Client.getObjectTagging {
+      it.bucket(bucketName)
+      it.key(key)
+    }.also {
+      assertThat(it.tagSet()).hasSize(1)
+      assertThat(it.tagSet()[0].key()).isEqualTo("Tag Name")
+      assertThat(it.tagSet()[0].value()).isEqualTo("Tag Value")
+    }
+  }
 
   @Test
   @S3VerifiedSuccess(year = 2025)
@@ -180,6 +242,43 @@ internal class PresignedUrlIT : S3TestBase() {
     }
   }
 
+  @Test
+  @S3VerifiedSuccess(year = 2025)
+  fun testPresignedUrl_putObject_largeFile(testInfo: TestInfo) {
+    val key = randomName
+    val bucketName = givenBucket(testInfo)
+
+    val presignedUrlString = s3Presigner.presignPutObject {
+      it.putObjectRequest {
+        it.bucket(bucketName)
+        it.key(key)
+      }
+      it.signatureDuration(Duration.ofMinutes(1L))
+    }.url().toString()
+
+    assertThat(presignedUrlString).isNotBlank()
+
+    val randomMBytes = randomMBytes(20)
+    HttpPut(presignedUrlString).apply {
+      this.entity = ByteArrayEntity(randomMBytes)
+    }.also { put ->
+      httpClient.execute(
+        put
+      ).use {
+        assertThat(it.statusLine.statusCode).isEqualTo(HttpStatus.SC_OK)
+      }
+    }
+
+    s3Client.getObject {
+      it.bucket(bucketName)
+      it.key(key)
+    }.use {
+      val expectedEtag = "\"${DigestUtil.hexDigest(randomMBytes.inputStream())}\""
+      val actualEtag = "\"${DigestUtil.hexDigest(it)}\""
+      assertThat(actualEtag).isEqualTo(expectedEtag)
+    }
+  }
+
   @Test
   @S3VerifiedSuccess(year = 2025)
   fun testPresignedUrl_createMultipartUpload(testInfo: TestInfo) {

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt

@@ -55,6 +55,7 @@ import software.amazon.awssdk.services.s3.model.UploadPartResponse
 import software.amazon.awssdk.services.s3.presigner.S3Presigner
 import software.amazon.awssdk.transfer.s3.S3TransferManager
 import software.amazon.awssdk.utils.AttributeMap
+import tel.schich.awss3postobjectpresigner.S3PostObjectPresigner
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
 import java.io.File
@@ -209,6 +210,17 @@ internal abstract class S3TestBase {
       .build()
   }
 
+  protected fun createS3PostObjectPresigner(endpoint: String = serviceEndpoint): S3PostObjectPresigner {
+    return S3PostObjectPresigner.builder()
+      .region(Region.of(s3Region))
+      .credentialsProvider(
+        StaticCredentialsProvider.create(AwsBasicCredentials.create(s3AccessKeyId, s3SecretAccessKey))
+      )
+      .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
+      .endpointOverride(URI.create(endpoint))
+      .build()
+  }
+
   /**
    * Deletes all existing buckets.
    */
@@ -467,7 +479,7 @@ internal abstract class S3TestBase {
     return randomMBytes(_5MB.toInt())
   }
 
-  private fun randomMBytes(size: Int): ByteArray {
+  protected fun randomMBytes(size: Int): ByteArray {
     val bytes = ByteArray(size)
     Random.nextBytes(bytes)
     return bytes

pom.xml

@@ -21,7 +21,7 @@
 
   <groupId>com.adobe.testing</groupId>
   <artifactId>s3mock-parent</artifactId>
-  <version>4.0.1-SNAPSHOT</version>
+  <version>4.1.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <name>S3Mock - Parent</name>
@@ -89,26 +89,32 @@
   </distributionManagement>
 
   <properties>
+    <java.version>17</java.version>
+    <kotlin.version>2.1.20</kotlin.version>
+    <kotlin.compiler.jvmTarget>${java.version}</kotlin.compiler.jvmTarget>
+    <kotlin.compiler.apiVersion>2.1</kotlin.compiler.apiVersion>
+    <kotlin.compiler.languageVersion>2.1</kotlin.compiler.languageVersion>
+
     <aws-v2.version>2.31.25</aws-v2.version>
 
     <aws.version>1.12.782</aws.version>
 
     <aws-kotlin.version>1.4.67</aws-kotlin.version>
 
-    <checkstyle.version>10.23.0</checkstyle.version>
     <commons-codec.version>1.15</commons-codec.version>
     <commons-io.version>2.19.0</commons-io.version>
+    <httpclient.version>4.5.14</httpclient.version>
+    <httpmime.version>4.5.14</httpmime.version>
+    <httpcore.version>4.4.16</httpcore.version>
+
     <docker-builder.image.name>s3mock-buildx</docker-builder.image.name>
     <docker-maven-plugin.version>0.46.0</docker-maven-plugin.version>
-
     <docker.image.name>adobe/s3mock</docker.image.name>
-    <java.version>17</java.version>
+
     <junit-jupiter.version>5.7.2</junit-jupiter.version>
     <junit.version>4.13.2</junit.version>
-    <kotlin.version>2.1.20</kotlin.version>
-    <kotlin.compiler.jvmTarget>${java.version}</kotlin.compiler.jvmTarget>
-    <kotlin.compiler.apiVersion>2.1</kotlin.compiler.apiVersion>
-    <kotlin.compiler.languageVersion>2.1</kotlin.compiler.languageVersion>
+
+    <checkstyle.version>10.23.0</checkstyle.version>
     <license-maven-plugin-git.version>5.0.0</license-maven-plugin-git.version>
     <maven-checkstyle-plugin.version>3.6.0</maven-checkstyle-plugin.version>
     <maven-clean-plugin.version>3.4.1</maven-clean-plugin.version>
@@ -124,6 +130,7 @@
     <maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
     <maven-source-plugin.version>3.3.1</maven-source-plugin.version>
     <maven-surefire-plugin.version>3.5.3</maven-surefire-plugin.version>
+    <exec-maven-plugin.version>3.5.0</exec-maven-plugin.version>
     <mockito-kotlin.version>5.4.0</mockito-kotlin.version>
     <nexus-staging-maven-plugin.version>1.7.0</nexus-staging-maven-plugin.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -280,6 +287,26 @@
         <artifactId>s3-jvm</artifactId>
         <version>${aws-kotlin.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.httpcomponents</groupId>
+        <artifactId>httpclient</artifactId>
+        <version>${httpclient.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.httpcomponents</groupId>
+        <artifactId>httpcore</artifactId>
+        <version>${httpcore.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.httpcomponents</groupId>
+        <artifactId>httpmime</artifactId>
+        <version>${httpmime.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.mockito</groupId>
+        <artifactId>mockito-core</artifactId>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -554,7 +581,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>3.5.0</version>
+          <version>${exec-maven-plugin.version}</version>
         </plugin>
         <plugin>
           <!--

server/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>4.0.1-SNAPSHOT</version>
+    <version>4.1.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>s3mock</artifactId>

server/src/main/java/com/adobe/testing/s3mock/BucketController.java

@@ -16,7 +16,6 @@
 
 package com.adobe.testing.s3mock;
 
-import static com.adobe.testing.s3mock.S3Exception.NOT_FOUND_BUCKET_VERSIONING_CONFIGURATION;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_BUCKET_OBJECT_LOCK_ENABLED;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_OBJECT_OWNERSHIP;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.CONTINUATION_TOKEN;
@@ -49,7 +48,6 @@
 import com.adobe.testing.s3mock.dto.ObjectLockConfiguration;
 import com.adobe.testing.s3mock.dto.VersioningConfiguration;
 import com.adobe.testing.s3mock.service.BucketService;
-import com.adobe.testing.s3mock.store.BucketMetadata;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.CrossOrigin;