API consistency: Bucket API / DTOs

Checking AWS APIs for changes and S3Mock for implementations and tests.

Fixes #2340

Author: afranken

CHANGELOG.md

@@ -150,6 +150,9 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
     * Check AWS API for changes
     * Update S3Mock API / DTOs
     * Add tests for changed API / DTOs
+    * List Objects now returns "delimiter"
+    * List Objects V2 now accepts "fetch-owner" and returns "delimiter"
+    * List Buckets now accepts all parameters listed in AWS S3 API
 * Version updates (deliverable dependencies)
   * Bump spring-boot.version from 3.4.4 to 3.4.5
   * Bump testcontainers.version from 1.20.6 to 1.21.0

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketIT.kt

@@ -26,13 +26,16 @@ import software.amazon.awssdk.awscore.exception.AwsServiceException
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.services.s3.model.AbortIncompleteMultipartUpload
 import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration
+import software.amazon.awssdk.services.s3.model.BucketType
 import software.amazon.awssdk.services.s3.model.BucketVersioningStatus
+import software.amazon.awssdk.services.s3.model.DataRedundancy
 import software.amazon.awssdk.services.s3.model.DeleteBucketRequest
 import software.amazon.awssdk.services.s3.model.ExpirationStatus
 import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationRequest
 import software.amazon.awssdk.services.s3.model.LifecycleExpiration
 import software.amazon.awssdk.services.s3.model.LifecycleRule
 import software.amazon.awssdk.services.s3.model.LifecycleRuleFilter
+import software.amazon.awssdk.services.s3.model.LocationType
 import software.amazon.awssdk.services.s3.model.MFADelete
 import software.amazon.awssdk.services.s3.model.MFADeleteStatus
 import software.amazon.awssdk.services.s3.model.NoSuchBucketException
@@ -68,6 +71,35 @@ internal class BucketIT : S3TestBase() {
     }
   }
 
+  @Test
+  @S3VerifiedSuccess(year = 2025)
+  fun `creating a bucket with configuration is successful`(testInfo: TestInfo) {
+    val bucketName = bucketName(testInfo)
+    val createBucketResponse = s3Client.createBucket {
+      it.bucket(bucketName)
+      it.createBucketConfiguration {
+        it.locationConstraint("ap-southeast-5")
+        it.bucket {
+          it.dataRedundancy(DataRedundancy.SINGLE_AVAILABILITY_ZONE)
+          it.type(BucketType.DIRECTORY)
+        }
+        it.location {
+          it.name("SomeName")
+          it.type(LocationType.AVAILABILITY_ZONE)
+        }
+      }
+    }
+    assertThat(createBucketResponse.sdkHttpResponse().statusCode()).isEqualTo(200)
+    assertThat(createBucketResponse.location()).isEqualTo("/$bucketName")
+
+    val bucketCreated = s3Client.waiter().waitUntilBucketExists { it.bucket(bucketName) }
+    val bucketCreatedResponse = bucketCreated.matched().response().get()
+    assertThat(bucketCreatedResponse).isNotNull
+
+    //does not throw exception if bucket exists.
+    s3Client.headBucket { it.bucket(bucketName) }
+  }
+
   @Test
   @S3VerifiedSuccess(year = 2025)
   fun `deleting a non-empty bucket fails`(testInfo: TestInfo) {
@@ -94,20 +126,113 @@ internal class BucketIT : S3TestBase() {
     // and account for a clock-skew in the Docker container of up to a minute.
     val creationDate = Instant.now().minus(1, ChronoUnit.MINUTES)
 
-    s3Client.listBuckets{
+    s3Client.listBuckets { }.also {
+      assertThat(it.hasBuckets()).isTrue
+      it.buckets().also {
+        assertThat(it.size).isEqualTo(5)
+        assertThat(it.map { b -> b.name() }).containsExactly(
+          // the default buckets
+          "bucket-a",
+          "bucket-b",
+          // the buckets we created in this test
+          "${bucketName}-1",
+          "${bucketName}-2",
+          "${bucketName}-3"
+        )
+        assertThat(it[2].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[3].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[4].creationDate()).isAfterOrEqualTo(creationDate)
+      }
+      assertThat(it.prefix()).isNull()
+      assertThat(it.continuationToken()).isNull()
+      assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store")
+      assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
+    }
+  }
+
+  @Test
+  @S3VerifiedFailure(year = 2025,
+    reason = "Default owner does not exist in S3.")
+  fun `creating and listing multiple buckets limiting by prefix is successful`(testInfo: TestInfo) {
+    val bucketName = bucketName(testInfo)
+    givenBucket("${bucketName}-1")
+    givenBucket("${bucketName}-2")
+    givenBucket("${bucketName}-3")
+    // the returned creation date might strip off the millisecond-part, resulting in rounding down
+    // and account for a clock-skew in the Docker container of up to a minute.
+    val creationDate = Instant.now().minus(1, ChronoUnit.MINUTES)
+
+    s3Client.listBuckets {
       it.prefix(bucketName)
     }.also {
       assertThat(it.hasBuckets()).isTrue
-      //TODO: ListBuckets API currently ignores the prefix argument, see #2340
-      it.buckets()
-        .filter { b -> b.name().startsWith(bucketName) }.also { filteredBuckets ->
-          assertThat(filteredBuckets.size).isEqualTo(3)
-          assertThat(filteredBuckets.map { b -> b.name() })
-            .containsExactlyInAnyOrder("${bucketName}-1", "${bucketName}-2", "${bucketName}-3")
-          assertThat(filteredBuckets[0].creationDate()).isAfterOrEqualTo(creationDate)
-          assertThat(filteredBuckets[1].creationDate()).isAfterOrEqualTo(creationDate)
-          assertThat(filteredBuckets[2].creationDate()).isAfterOrEqualTo(creationDate)
-        }
+      it.buckets().also {
+        assertThat(it.size).isEqualTo(3)
+        assertThat(it.map { b -> b.name() }).containsExactly(
+          // the buckets we created in this test
+          "${bucketName}-1",
+          "${bucketName}-2",
+          "${bucketName}-3"
+        )
+        assertThat(it[0].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[1].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[2].creationDate()).isAfterOrEqualTo(creationDate)
+      }
+      assertThat(it.prefix()).isEqualTo(bucketName)
+      assertThat(it.continuationToken()).isNull()
+      assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store")
+      assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
+    }
+  }
+
+  @Test
+  @S3VerifiedFailure(year = 2025,
+    reason = "Default owner does not exist in S3.")
+  fun `creating and listing multiple buckets limiting by maxBuckets is successful`(testInfo: TestInfo) {
+    val bucketName = bucketName(testInfo)
+    givenBucket("${bucketName}-1")
+    givenBucket("${bucketName}-2")
+    givenBucket("${bucketName}-3")
+    // the returned creation date might strip off the millisecond-part, resulting in rounding down
+    // and account for a clock-skew in the Docker container of up to a minute.
+    val creationDate = Instant.now().minus(1, ChronoUnit.MINUTES)
+
+    val continuationToken = s3Client.listBuckets {
+      it.maxBuckets(4)
+    }.also {
+      assertThat(it.hasBuckets()).isTrue
+      it.buckets().also {
+        assertThat(it.size).isEqualTo(4)
+        assertThat(it.map { b -> b.name() }).containsExactly(
+          // the default buckets
+          "bucket-a",
+          "bucket-b",
+          // the buckets we created in this test
+          "${bucketName}-1",
+          "${bucketName}-2"
+        )
+        assertThat(it[2].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[3].creationDate()).isAfterOrEqualTo(creationDate)
+      }
+      assertThat(it.prefix()).isNull()
+      assertThat(it.continuationToken()).isNotNull
+      assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store")
+      assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
+    }.continuationToken()
+
+    s3Client.listBuckets {
+      it.continuationToken(continuationToken)
+    }.also {
+      assertThat(it.hasBuckets()).isTrue
+      it.buckets().also {
+        assertThat(it.size).isEqualTo(1)
+        assertThat(it.map { b -> b.name() }).containsExactly(
+          "${bucketName}-3"
+        )
+        assertThat(it[0].creationDate()).isAfterOrEqualTo(creationDate)
+      }
+      assertThat(it.prefix()).isNull()
+      assertThat(it.continuationToken()).isNull()
       assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store")
       assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
     }
@@ -121,7 +246,7 @@ internal class BucketIT : S3TestBase() {
       assertThat(it.buckets())
         .hasSize(2)
         .extracting("name")
-        .containsExactlyInAnyOrder(INITIAL_BUCKET_NAMES.first(), INITIAL_BUCKET_NAMES.last())
+        .containsExactly(INITIAL_BUCKET_NAMES.first(), INITIAL_BUCKET_NAMES.last())
     }
   }