diff --git a/CHANGELOG.md b/CHANGELOG.md index 68258f4df..4ce874bef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,11 +156,13 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav * Support Browser-Based Uploads Using POST (fixes #2200) * https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-UsingHTTPPOST.html * Refactorings - * TBD + * Validate all integration tests against S3, fix S3Mock where necessary + * These were corner cases where error messages were incorrect, or proper validations were missing. + * Migrate all integration tests to AWS SDK v2, remove AWS SDK v1 tests from the integration-tests module * Version updates (deliverable dependencies) * TBD * Version updates (build dependencies) - * TBD + * Bump actions/setup-java from 4.7.0 to 4.7.1 ## 4.0.0 Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration. @@ -170,6 +172,7 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav * Allow overriding headers in head object * Implement If-(Un)modified-Since handling (fixes #829) * Close all InputStreams and OutputStreams + * Checksums are returned for MultipartUploads as part of the response body * Add AWS SDK V1 deprecation notice * AWS has deprecated SDK for Java v1, and will remove support EOY 2025. * S3Mock will remove usage of Java v1 early 2026. @@ -178,7 +181,7 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav * "FROM" in Dockerfile did not match "as" * Delete files on shutdown using a `DisposableBean` instead of `File#deleteOnExit()` * Version updates (deliverable dependencies) - * Bump spring-boot.version from 3.3.5 to 3.4.4 + * Bump spring-boot.version from 3.3.3 to 3.4.4 * Jackson 2.18.2 to 2.17.2 (remove override, use Spring-Boot supplied version) * Bump aws-v2.version from 2.29.29 to 2.31.17 * Bump aws.version from 1.12.779 to 1.12.780 diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 242092ce3..d9ce93857 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -48,21 +48,6 @@ - - com.amazonaws - aws-java-sdk-core - test - - - com.amazonaws - aws-java-sdk-s3 - test - - - org.apache.commons - commons-lang3 - test - org.assertj assertj-core diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AclITV2.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AclIT.kt similarity index 85% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AclITV2.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AclIT.kt index 05d304169..a1dd28209 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AclITV2.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AclIT.kt @@ -27,26 +27,26 @@ import software.amazon.awssdk.services.s3.model.ObjectOwnership import software.amazon.awssdk.services.s3.model.Permission.FULL_CONTROL import software.amazon.awssdk.services.s3.model.Type.CANONICAL_USER -internal class AclITV2 : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() +internal class AclIT : S3TestBase() { + private val s3Client: S3Client = createS3Client() @Test - @S3VerifiedSuccess(year = 2024) - fun testPutCannedAcl_OK(testInfo: TestInfo) { + @S3VerifiedSuccess(year = 2025) + fun `put canned ACL returns OK, get ACL returns the ACL`(testInfo: TestInfo) { val sourceKey = UPLOAD_FILE_NAME val bucketName = bucketName(testInfo) //create bucket that sets ownership to non-default to allow setting ACLs. - s3ClientV2.createBucket { + s3Client.createBucket { it.bucket(bucketName) it.objectOwnership(ObjectOwnership.OBJECT_WRITER) }.also { assertThat(it.sdkHttpResponse().isSuccessful).isTrue() } - givenObjectV2(bucketName, sourceKey) + givenObject(bucketName, sourceKey) - s3ClientV2.putObjectAcl { + s3Client.putObjectAcl { it.bucket(bucketName) it.key(sourceKey) it.acl(ObjectCannedACL.PRIVATE) @@ -54,7 +54,7 @@ internal class AclITV2 : S3TestBase() { assertThat(it.sdkHttpResponse().isSuccessful).isTrue() } - s3ClientV2.getObjectAcl { + s3Client.getObjectAcl { it.bucket(bucketName) it.key(sourceKey) }.also { @@ -69,11 +69,11 @@ internal class AclITV2 : S3TestBase() { @Test @S3VerifiedFailure(year = 2022, reason = "Owner and Grantee not available on test AWS account.") - fun testGetAcl_noAcl(testInfo: TestInfo) { + fun `get ACL returns canned 'private' ACL`(testInfo: TestInfo) { val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey) + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) - val acl = s3ClientV2.getObjectAcl { + val acl = s3Client.getObjectAcl { it.bucket(bucketName) it.key(sourceKey) } @@ -100,16 +100,16 @@ internal class AclITV2 : S3TestBase() { @Test @S3VerifiedFailure(year = 2022, reason = "Owner and Grantee not available on test AWS account.") - fun testPutAndGetAcl(testInfo: TestInfo) { + fun `put ACL returns OK, get ACL returns the ACL`(testInfo: TestInfo) { val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey) + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) val userId = "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2ab" val userName = "John Doe" val granteeId = "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2ef" val granteeName = "Jane Doe" val granteeEmail = "jane@doe.com" - s3ClientV2.putObjectAcl { + s3Client.putObjectAcl { it.bucket(bucketName) it.key(sourceKey) it.accessControlPolicy { @@ -129,7 +129,7 @@ internal class AclITV2 : S3TestBase() { } } - val acl = s3ClientV2.getObjectAcl { + val acl = s3Client.getObjectAcl { it.bucket(bucketName) it.key(sourceKey) } diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingITV2.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingIT.kt similarity index 72% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingITV2.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingIT.kt index 04fda0540..77756dceb 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingITV2.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingIT.kt @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Adobe. + * Copyright 2017-2025 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,8 +23,6 @@ import org.junit.jupiter.api.TestInfo import software.amazon.awssdk.core.checksums.Algorithm import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm -import software.amazon.awssdk.services.s3.model.GetObjectRequest -import software.amazon.awssdk.services.s3.model.PutObjectRequest import java.io.File import java.io.FileInputStream import java.io.InputStream @@ -32,9 +30,9 @@ import java.io.InputStream /** * Chunked encoding with signing is only active in AWS SDK v2 when endpoint is http */ -internal class AwsChunkedEndcodingITV2 : S3TestBase() { +internal class AwsChunkedEndcodingIT : S3TestBase() { - private val s3ClientV2 = createS3ClientV2(serviceEndpointHttp) + private val s3Client = createS3Client(serviceEndpointHttp) /** * Unfortunately the S3 API does not persist or return data that would let us verify if signed and chunked encoding @@ -46,19 +44,19 @@ internal class AwsChunkedEndcodingITV2 : S3TestBase() { year = 2023, reason = "Only works with http endpoints" ) - fun testPutObject_checksum(testInfo: TestInfo) { - val bucket = givenBucketV2(testInfo) + fun `put object with checksum returns correct checksum, get object returns checksum`(testInfo: TestInfo) { + val bucket = givenBucket(testInfo) val uploadFile = File(UPLOAD_FILE_NAME) val uploadFileIs: InputStream = FileInputStream(uploadFile) val expectedEtag = "\"${DigestUtil.hexDigest(uploadFileIs)}\"" val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), Algorithm.SHA256) - val putObjectResponse = s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucket) - .key(UPLOAD_FILE_NAME) - .checksumAlgorithm(ChecksumAlgorithm.SHA256) - .build(), + val putObjectResponse = s3Client.putObject( + { + it.bucket(bucket) + it.key(UPLOAD_FILE_NAME) + it.checksumAlgorithm(ChecksumAlgorithm.SHA256) + }, RequestBody.fromFile(uploadFile) ) @@ -67,12 +65,10 @@ internal class AwsChunkedEndcodingITV2 : S3TestBase() { assertThat(it).isEqualTo(expectedChecksum) } - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucket) - .key(UPLOAD_FILE_NAME) - .build() - ).also { getObjectResponse -> + s3Client.getObject { + it.bucket(bucket) + it.key(UPLOAD_FILE_NAME) + }.also { getObjectResponse -> assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag) assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length()) @@ -93,26 +89,24 @@ internal class AwsChunkedEndcodingITV2 : S3TestBase() { year = 2023, reason = "Only works with http endpoints" ) - fun testPutObject_etagCreation(testInfo: TestInfo) { - val bucket = givenBucketV2(testInfo) + fun `put object creates correct etag, get object returns etag`(testInfo: TestInfo) { + val bucket = givenBucket(testInfo) val uploadFile = File(UPLOAD_FILE_NAME) val uploadFileIs: InputStream = FileInputStream(uploadFile) val expectedEtag = "\"${DigestUtil.hexDigest(uploadFileIs)}\"" - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucket) - .key(UPLOAD_FILE_NAME) - .build(), + s3Client.putObject( + { + it.bucket(bucket) + it.key(UPLOAD_FILE_NAME) + }, RequestBody.fromFile(uploadFile) ) - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucket) - .key(UPLOAD_FILE_NAME) - .build() - ).also { + s3Client.getObject { + it.bucket(bucket) + it.key(UPLOAD_FILE_NAME) + }.also { assertThat(it.response().eTag()).isEqualTo(expectedEtag) assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) } diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketIT.kt similarity index 54% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketIT.kt index b32b1df1e..2b90bf618 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV2IT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketIT.kt @@ -27,46 +27,41 @@ import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.AbortIncompleteMultipartUpload import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration import software.amazon.awssdk.services.s3.model.BucketVersioningStatus -import software.amazon.awssdk.services.s3.model.CreateBucketRequest -import software.amazon.awssdk.services.s3.model.DeleteBucketLifecycleRequest import software.amazon.awssdk.services.s3.model.DeleteBucketRequest import software.amazon.awssdk.services.s3.model.ExpirationStatus import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationRequest -import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest -import software.amazon.awssdk.services.s3.model.HeadBucketRequest import software.amazon.awssdk.services.s3.model.LifecycleExpiration import software.amazon.awssdk.services.s3.model.LifecycleRule import software.amazon.awssdk.services.s3.model.LifecycleRuleFilter import software.amazon.awssdk.services.s3.model.MFADelete import software.amazon.awssdk.services.s3.model.MFADeleteStatus import software.amazon.awssdk.services.s3.model.NoSuchBucketException -import software.amazon.awssdk.services.s3.model.PutBucketLifecycleConfigurationRequest +import java.time.Instant +import java.time.temporal.ChronoUnit import java.util.concurrent.TimeUnit /** * Test the application using the AmazonS3 SDK V2. */ -internal class BucketV2IT : S3TestBase() { +internal class BucketIT : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() + private val s3Client: S3Client = createS3Client() @Test - @S3VerifiedSuccess(year = 2024) - fun createAndDeleteBucket(testInfo: TestInfo) { + @S3VerifiedSuccess(year = 2025) + fun `creating and deleting a bucket is successful`(testInfo: TestInfo) { val bucketName = bucketName(testInfo) - s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()) + s3Client.createBucket { it.bucket(bucketName) } - val bucketCreated = s3ClientV2.waiter() - .waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build()) + val bucketCreated = s3Client.waiter().waitUntilBucketExists { it.bucket(bucketName) } val bucketCreatedResponse = bucketCreated.matched().response().get() assertThat(bucketCreatedResponse).isNotNull //does not throw exception if bucket exists. - s3ClientV2.headBucket(HeadBucketRequest.builder().bucket(bucketName).build()) + s3Client.headBucket { it.bucket(bucketName) } - s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()) - val bucketDeleted = s3ClientV2.waiter() - .waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build()) + s3Client.deleteBucket { it.bucket(bucketName) } + val bucketDeleted = s3Client.waiter().waitUntilBucketNotExists { it.bucket(bucketName) } bucketDeleted.matched().exception().get().also { assertThat(it).isNotNull assertThat(it).isInstanceOf(NoSuchBucketException::class.java) @@ -74,20 +69,77 @@ internal class BucketV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) - fun getBucketLocation(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val bucketLocation = s3ClientV2.getBucketLocation(GetBucketLocationRequest.builder().bucket(bucketName).build()) + @S3VerifiedSuccess(year = 2025) + fun `deleting a non-empty bucket fails`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + givenObject(bucketName, UPLOAD_FILE_NAME) + assertThatThrownBy { s3Client.deleteBucket { it.bucket(bucketName) } } + .isInstanceOf(AwsServiceException::class.java) + .hasMessageContaining("Service: S3, Status Code: 409") + .asInstanceOf(InstanceOfAssertFactories.type(AwsServiceException::class.java)) + .extracting(AwsServiceException::awsErrorDetails) + .extracting(AwsErrorDetails::errorCode) + .isEqualTo("BucketNotEmpty") + } + + @Test + @S3VerifiedFailure(year = 2025, + reason = "Default owner does not exist in S3.") + fun `creating and listing multiple buckets is successful`(testInfo: TestInfo) { + val bucketName = bucketName(testInfo) + givenBucket("${bucketName}-1") + givenBucket("${bucketName}-2") + givenBucket("${bucketName}-3") + // the returned creation date might strip off the millisecond-part, resulting in rounding down + // and account for a clock-skew in the Docker container of up to a minute. + val creationDate = Instant.now().minus(1, ChronoUnit.MINUTES) + + s3Client.listBuckets{ + it.prefix(bucketName) + }.also { + assertThat(it.hasBuckets()).isTrue + //TODO: ListBuckets API currently ignores the prefix argument, see #2340 + it.buckets() + .filter { b -> b.name().startsWith(bucketName) }.also { filteredBuckets -> + assertThat(filteredBuckets.size).isEqualTo(3) + assertThat(filteredBuckets.map { b -> b.name() }) + .containsExactlyInAnyOrder("${bucketName}-1", "${bucketName}-2", "${bucketName}-3") + assertThat(filteredBuckets[0].creationDate()).isAfterOrEqualTo(creationDate) + assertThat(filteredBuckets[1].creationDate()).isAfterOrEqualTo(creationDate) + assertThat(filteredBuckets[2].creationDate()).isAfterOrEqualTo(creationDate) + } + assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store") + assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be") + } + } + + @Test + @S3VerifiedFailure(year = 2025, + reason = "Default buckets do not exist in S3.") + fun `default buckets were created`(testInfo: TestInfo) { + s3Client.listBuckets().also { + assertThat(it.buckets()) + .hasSize(2) + .extracting("name") + .containsExactlyInAnyOrder(INITIAL_BUCKET_NAMES.first(), INITIAL_BUCKET_NAMES.last()) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `get bucket location returns a result`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val bucketLocation = s3Client.getBucketLocation { it.bucket(bucketName) } assertThat(bucketLocation.locationConstraint().toString()).isEqualTo("eu-west-1") } @Test - @S3VerifiedSuccess(year = 2024) - fun getDefaultBucketVersioning(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) + @S3VerifiedSuccess(year = 2025) + fun `by default, bucket versioning is turned off`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) - s3ClientV2.getBucketVersioning { + s3Client.getBucketVersioning { it.bucket(bucketName) }.also { assertThat(it.status()).isNull() @@ -96,17 +148,17 @@ internal class BucketV2IT : S3TestBase() { } @Test - @S3VerifiedTodo - fun putAndGetBucketVersioning(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - s3ClientV2.putBucketVersioning { + @S3VerifiedSuccess(year = 2025) + fun `put bucket versioning works, get bucket versioning is returned correctly`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - s3ClientV2.getBucketVersioning { + s3Client.getBucketVersioning { it.bucket(bucketName) }.also { assertThat(it.status()).isEqualTo(BucketVersioningStatus.ENABLED) @@ -114,30 +166,30 @@ internal class BucketV2IT : S3TestBase() { } @Test - @S3VerifiedTodo - fun putAndGetBucketVersioning_suspended(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - s3ClientV2.putBucketVersioning { + @S3VerifiedSuccess(year = 2025) + fun `put bucket versioning works, suspending as well, get bucket versioning is returned correctly`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - s3ClientV2.getBucketVersioning { + s3Client.getBucketVersioning { it.bucket(bucketName) }.also { assertThat(it.status()).isEqualTo(BucketVersioningStatus.ENABLED) } - s3ClientV2.putBucketVersioning { + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.SUSPENDED) } } - s3ClientV2.getBucketVersioning { + s3Client.getBucketVersioning { it.bucket(bucketName) }.also { assertThat(it.status()).isEqualTo(BucketVersioningStatus.SUSPENDED) @@ -146,9 +198,9 @@ internal class BucketV2IT : S3TestBase() { @Test @S3VerifiedFailure(year = 2024, reason = "No real Mfa value") - fun putAndGetBucketVersioning_mfa(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - s3ClientV2.putBucketVersioning { + fun `put bucket versioning with mfa works, get bucket versioning is returned correctly`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + s3Client.putBucketVersioning { it.bucket(bucketName) it.mfa("fakeMfaValue") it.versioningConfiguration { @@ -157,7 +209,7 @@ internal class BucketV2IT : S3TestBase() { } } - s3ClientV2.getBucketVersioning { + s3Client.getBucketVersioning { it.bucket(bucketName) }.also { assertThat(it.status()).isEqualTo(BucketVersioningStatus.ENABLED) @@ -166,19 +218,18 @@ internal class BucketV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) - fun duplicateBucketCreation(testInfo: TestInfo) { + @S3VerifiedSuccess(year = 2025) + fun `duplicate bucket creation returns the correct error`(testInfo: TestInfo) { val bucketName = bucketName(testInfo) - s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()) + s3Client.createBucket { it.bucket(bucketName) } - val bucketCreated = s3ClientV2.waiter() - .waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build()) + val bucketCreated = s3Client.waiter().waitUntilBucketExists { it.bucket(bucketName) } bucketCreated.matched().response().get().also { assertThat(it).isNotNull } assertThatThrownBy { - s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()) + s3Client.createBucket { it.bucket(bucketName) } } .isInstanceOf(AwsServiceException::class.java) .hasMessageContaining("Service: S3, Status Code: 409") @@ -187,9 +238,8 @@ internal class BucketV2IT : S3TestBase() { .extracting(AwsErrorDetails::errorCode) .isEqualTo("BucketAlreadyOwnedByYou") - s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()) - val bucketDeleted = s3ClientV2.waiter() - .waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build()) + s3Client.deleteBucket { it.bucket(bucketName) } + val bucketDeleted = s3Client.waiter().waitUntilBucketNotExists { it.bucket(bucketName) } bucketDeleted.matched().exception().get().also { assertThat(it).isNotNull @@ -198,27 +248,25 @@ internal class BucketV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) - fun duplicateBucketDeletion(testInfo: TestInfo) { + @S3VerifiedSuccess(year = 2025) + fun `duplicate bucket deletion returns the correct error`(testInfo: TestInfo) { val bucketName = bucketName(testInfo) - s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()) + s3Client.createBucket { it.bucket(bucketName) } - val bucketCreated = s3ClientV2.waiter() - .waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build()) + val bucketCreated = s3Client.waiter().waitUntilBucketExists { it.bucket(bucketName) } bucketCreated.matched().response().get().also { assertThat(it).isNotNull } - s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()) - val bucketDeleted = s3ClientV2.waiter() - .waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build()) + s3Client.deleteBucket { it.bucket(bucketName) } + val bucketDeleted = s3Client.waiter().waitUntilBucketNotExists { it.bucket(bucketName) } bucketDeleted.matched().exception().get().also { assertThat(it).isNotNull assertThat(it).isInstanceOf(NoSuchBucketException::class.java) } assertThatThrownBy { - s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()) + s3Client.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()) } .isInstanceOf(AwsServiceException::class.java) .hasMessageContaining("Service: S3, Status Code: 404") @@ -229,20 +277,17 @@ internal class BucketV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) - fun getBucketLifecycle_notFound(testInfo: TestInfo) { + @S3VerifiedSuccess(year = 2025) + fun `get bucket lifecycle returns error if not set`(testInfo: TestInfo) { val bucketName = bucketName(testInfo) - s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()) + s3Client.createBucket { it.bucket(bucketName) } - val bucketCreated = s3ClientV2.waiter() - .waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build()) + val bucketCreated = s3Client.waiter().waitUntilBucketExists { it.bucket(bucketName) } val bucketCreatedResponse = bucketCreated.matched().response()!!.get() assertThat(bucketCreatedResponse).isNotNull assertThatThrownBy { - s3ClientV2.getBucketLifecycleConfiguration( - GetBucketLifecycleConfigurationRequest.builder().bucket(bucketName).build() - ) + s3Client.getBucketLifecycleConfiguration { it.bucket(bucketName) } } .isInstanceOf(AwsServiceException::class.java) .hasMessageContaining("Service: S3, Status Code: 404") @@ -253,13 +298,12 @@ internal class BucketV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) - fun putGetDeleteBucketLifecycle(testInfo: TestInfo) { + @S3VerifiedSuccess(year = 2025) + fun `put bucket lifecycle is successful, get bucket lifecycle returns the lifecycle, delete is successful`(testInfo: TestInfo) { val bucketName = bucketName(testInfo) - s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()) + s3Client.createBucket { it.bucket(bucketName) } - val bucketCreated = s3ClientV2.waiter() - .waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build()) + val bucketCreated = s3Client.waiter().waitUntilBucketExists { it.bucket(bucketName) } bucketCreated.matched().response()!!.get().also { assertThat(it).isNotNull } @@ -288,28 +332,16 @@ internal class BucketV2IT : S3TestBase() { ) .build() - s3ClientV2.putBucketLifecycleConfiguration( - PutBucketLifecycleConfigurationRequest - .builder() - .bucket(bucketName) - .lifecycleConfiguration( - configuration - ) - .build() - ) - - s3ClientV2.getBucketLifecycleConfiguration( - GetBucketLifecycleConfigurationRequest - .builder() - .bucket(bucketName) - .build() - ).also { + s3Client.putBucketLifecycleConfiguration { + it.bucket(bucketName) + it.lifecycleConfiguration(configuration) + } + + s3Client.getBucketLifecycleConfiguration { it.bucket(bucketName) }.also { assertThat(it.rules()[0]).isEqualTo(configuration.rules()[0]) } - s3ClientV2.deleteBucketLifecycle( - DeleteBucketLifecycleRequest.builder().bucket(bucketName).build() - ).also { + s3Client.deleteBucketLifecycle { it.bucket(bucketName) }.also { assertThat(it.sdkHttpResponse().statusCode()).isEqualTo(204) } @@ -319,7 +351,7 @@ internal class BucketV2IT : S3TestBase() { TimeUnit.SECONDS.sleep(3) assertThatThrownBy { - s3ClientV2.getBucketLifecycleConfiguration( + s3Client.getBucketLifecycleConfiguration( GetBucketLifecycleConfigurationRequest.builder().bucket(bucketName).build() ) } diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV1IT.kt deleted file mode 100644 index 85d7db33f..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketV1IT.kt +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.adobe.testing.s3mock.its - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.AmazonS3Exception -import com.amazonaws.services.s3.model.Bucket -import com.amazonaws.services.s3.model.HeadBucketRequest -import com.amazonaws.services.s3.model.PutObjectRequest -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import java.io.File -import java.util.Date -import java.util.stream.Collectors - -/** - * Test the application using the AmazonS3 SDK V1. - */ -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class BucketV1IT : S3TestBase() { - - private val s3Client: AmazonS3 = createS3ClientV1() - - @Test - @S3VerifiedFailure(year = 2022, - reason = "BucketOwner does not match owner in S3") - fun testCreateBucketAndListAllBuckets(testInfo: TestInfo) { - val bucketName = bucketName(testInfo) - val bucket = s3Client.createBucket(bucketName) - // the returned creation date might strip off the millisecond-part, resulting in rounding down - // and account for a clock-skew in the Docker container of up to a minute. - val creationDate = Date(System.currentTimeMillis() / 1000 * 1000 - 60000) - assertThat(bucket.name).isEqualTo(bucketName) - - val buckets = s3Client.listBuckets().stream() - .filter { b: Bucket -> bucketName == b.name } - .collect(Collectors.toList()) - assertThat(buckets).hasSize(1) - - val createdBucket = buckets[0] - assertThat(createdBucket.creationDate).isAfterOrEqualTo(creationDate) - - createdBucket.owner.also { - assertThat(it.displayName).isEqualTo("s3-mock-file-store") - assertThat(it.id).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be") - } - } - - @Test - @S3VerifiedFailure(year = 2022, - reason = "Default buckets do not exist in S3.") - fun testDefaultBucketCreation() { - val buckets = s3Client.listBuckets() - val bucketNames = buckets.stream() - .map { obj: Bucket -> obj.name } - .filter { o: String? -> INITIAL_BUCKET_NAMES.contains(o) } - .collect(Collectors.toSet()) - assertThat(bucketNames) - .containsAll(INITIAL_BUCKET_NAMES) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCreateAndDeleteBucket(testInfo: TestInfo) { - val bucketName = bucketName(testInfo) - s3Client.createBucket(bucketName) - s3Client.headBucket(HeadBucketRequest(bucketName)) - s3Client.deleteBucket(bucketName) - - s3Client.doesBucketExistV2(bucketName).also { - assertThat(it).isFalse - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testFailureDeleteNonEmptyBucket(testInfo: TestInfo) { - val bucketName = bucketName(testInfo) - s3Client.createBucket(bucketName) - val uploadFile = File(UPLOAD_FILE_NAME) - s3Client.putObject(PutObjectRequest(bucketName, UPLOAD_FILE_NAME, uploadFile)) - - assertThatThrownBy { s3Client.deleteBucket(bucketName) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 409; Error Code: BucketNotEmpty") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testBucketDoesExistV2_ok(testInfo: TestInfo) { - val bucketName = bucketName(testInfo) - s3Client.createBucket(bucketName) - - s3Client.doesBucketExistV2(bucketName).also { - assertThat(it).isTrue - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testBucketDoesExistV2_failure(testInfo: TestInfo) { - val bucketName = bucketName(testInfo) - - val doesBucketExist = s3Client.doesBucketExistV2(bucketName) - assertThat(doesBucketExist).isFalse - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun duplicateBucketCreation(testInfo: TestInfo) { - val bucketName = bucketName(testInfo) - s3Client.createBucket(bucketName) - - assertThatThrownBy { - s3Client.createBucket(bucketName) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Service: Amazon S3; Status Code: 409; " + - "Error Code: BucketAlreadyOwnedByYou;") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun duplicateBucketDeletion(testInfo: TestInfo) { - val bucketName = bucketName(testInfo) - s3Client.createBucket(bucketName) - - s3Client.deleteBucket(bucketName) - - assertThatThrownBy { - s3Client.deleteBucket(bucketName) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Service: Amazon S3; Status Code: 404; Error Code: NoSuchBucket;") - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ConcurrencyIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ConcurrencyIT.kt index 693751185..1ce6bc9be 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ConcurrencyIT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ConcurrencyIT.kt @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Adobe. + * Copyright 2017-2025 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,26 +21,25 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInfo import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest -import software.amazon.awssdk.services.s3.model.GetObjectRequest -import software.amazon.awssdk.services.s3.model.PutObjectRequest import java.util.concurrent.Callable import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger internal class ConcurrencyIT : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() + private val s3Client: S3Client = createS3Client() /** * Test that there are no inconsistencies when multiple threads PUT, GET and DELETE objects in * the same bucket. */ @Test - @S3VerifiedFailure(year = 2022, - reason = "No need to test S3 concurrency.") - fun concurrentBucketPutGetAndDeletes(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) + @S3VerifiedFailure( + year = 2022, + reason = "No need to test S3 concurrency." + ) + fun `concurrent bucket puts, gets and deletes are successful`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) val runners = mutableListOf() val pool = Executors.newFixedThreadPool(100) for (i in 1..100) { @@ -61,33 +60,26 @@ internal class ConcurrencyIT : S3TestBase() { inner class Runner(val bucketName: String, val key: String) : Callable { override fun call(): Boolean { LATCH.countDown() - s3ClientV2.putObject( - PutObjectRequest - .builder() - .bucket(bucketName) - .key(key) - .build(), RequestBody.empty() + s3Client.putObject( + { + it.bucket(bucketName) + it.key(key) + }, RequestBody.empty() ).also { assertThat(it.eTag()).isNotBlank } - s3ClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(key) - .build() - ).also { + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + }.also { assertThat(it.response().eTag()).isNotBlank } - s3ClientV2.deleteObject( - DeleteObjectRequest - .builder() - .bucket(bucketName) - .key(key) - .build() - ).also { + s3Client.deleteObject { + it.bucket(bucketName) + it.key(key) + }.also { assertThat(it.deleteMarker()).isTrue } DONE.incrementAndGet() diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectIT.kt new file mode 100644 index 000000000..19ade5642 --- /dev/null +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectIT.kt @@ -0,0 +1,537 @@ +/* + * Copyright 2017-2025 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adobe.testing.s3mock.its + +import com.adobe.testing.s3mock.S3Exception.PRECONDITION_FAILED +import com.adobe.testing.s3mock.util.DigestUtil +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.awaitility.Awaitility.await +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo +import software.amazon.awssdk.core.async.AsyncRequestBody +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.MetadataDirective +import software.amazon.awssdk.services.s3.model.S3Exception +import software.amazon.awssdk.services.s3.model.ServerSideEncryption +import software.amazon.awssdk.services.s3.model.StorageClass +import java.io.File +import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit.SECONDS +import java.util.concurrent.Executors + +/** + * Test the application using the AmazonS3 SDK V2. + */ +internal class CopyObjectIT : S3TestBase() { + + private val s3Client: S3Client = createS3Client() + private val transferManager = createTransferManager() + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object succeeds and object can be retrieved`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, putObjectResult) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + }.copyObjectResult().eTag().also { + assertThat(it).isEqualTo(putObjectResult.eTag()) + } + + s3Client.getObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.use { + val copiedDigest = DigestUtil.hexDigest(it) + assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object with key needing escaping succeeds and object can be retrieved`(testInfo: TestInfo) { + val sourceKey = charsSpecialKey() + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + val putObjectResult = s3Client.putObject( + { + it.bucket(bucketName) + it.key(sourceKey) + }, RequestBody.fromFile(uploadFile) + ) + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + }.copyObjectResult().eTag().also { + assertThat(it).isEqualTo(putObjectResult.eTag()) + } + + s3Client.getObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.use { + val copiedDigest = DigestUtil.hexDigest(it) + assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object with if match succeeds and object can be retrieved`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, putObjectResult) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + val matchingEtag = putObjectResult.eTag() + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + it.copySourceIfMatch(matchingEtag) + }.copyObjectResult().eTag().also { + assertThat(it).isEqualTo(putObjectResult.eTag()) + } + + s3Client.getObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.use { + val copiedDigest = DigestUtil.hexDigest(it) + assertThat("\"$copiedDigest\"").isEqualTo(matchingEtag) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object with if nonematch succeeds and object can be retrieved`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, putObjectResult) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val noneMatchingEtag = "\"${randomName}\"" + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + it.copySourceIfNoneMatch(noneMatchingEtag) + }.copyObjectResult().eTag().also { + assertThat(it).isEqualTo(putObjectResult.eTag()) + } + + s3Client.getObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.use { + val copiedDigest = DigestUtil.hexDigest(it) + assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object fails with non matching if match`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val noneMatchingEtag = "\"${randomName}\"" + + assertThatThrownBy { + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + it.copySourceIfMatch(noneMatchingEtag) + } + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 412") + .hasMessageContaining(PRECONDITION_FAILED.message) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object fails with non matching if nonematch`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, putObjectResult) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val matchingEtag = putObjectResult.eTag() + + assertThatThrownBy { + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + it.copySourceIfNoneMatch(matchingEtag) + } + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 412") + .hasMessageContaining(PRECONDITION_FAILED.message) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object succeeds with same bucket and key with REPLACE and changing metadata`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val sourceKey = UPLOAD_FILE_NAME + val putObjectResult = s3Client.putObject( + { + it.bucket(bucketName) + it.key(sourceKey) + it.metadata(mapOf("test-key" to "test-value")) + }, + RequestBody.fromFile(uploadFile) + ) + val sourceLastModified = s3Client.headObject { + it.bucket(bucketName) + it.key(sourceKey) + }.lastModified() + + await("wait until source object is 5 seconds old").until { + sourceLastModified.plusSeconds(5).isBefore(Instant.now()) + } + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(bucketName) + it.destinationKey(sourceKey) + it.metadata(mapOf("test-key2" to "test-value2")) + it.metadataDirective(MetadataDirective.REPLACE) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(sourceKey) + }.use { + val response = it.response() + val copiedObjectMetadata = response.metadata() + assertThat(copiedObjectMetadata["test-key2"]).isEqualTo("test-value2") + assertThat(copiedObjectMetadata["test-key"]).isNull() + + val length = response.contentLength() + assertThat(length).isEqualTo(uploadFile.length()) + + val copiedDigest = DigestUtil.hexDigest(it) + assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) + + //we waited for 5 seconds above, so last modified dates should be about 5 seconds apart + val between = Duration.between(sourceLastModified, response.lastModified()) + assertThat(between).isCloseTo(Duration.of(5, SECONDS), Duration.of(1, SECONDS)) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object fails with same bucket and key without changing metadata`(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val sourceKey = UPLOAD_FILE_NAME + + s3Client.putObject( + { + it.bucket(bucketName) + it.key(sourceKey) + it.metadata(mapOf("test-key" to "test-value")) + }, + RequestBody.fromFile(uploadFile) + ) + + val sourceLastModified = s3Client.headObject { + it.bucket(bucketName) + it.key(sourceKey) + }.lastModified() + + await("wait until source object is 5 seconds old").until { + sourceLastModified.plusSeconds(5).isBefore(Instant.now()) + } + + assertThatThrownBy { + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(bucketName) + it.destinationKey(sourceKey) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 400") + .hasMessageContaining("This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object succeeds with source metadata`(testInfo: TestInfo) { + + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val sourceKey = UPLOAD_FILE_NAME + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey/withSourceUserMetadata" + + val metadata = mapOf("test-key2" to "test-value2") + + val putObjectResult = s3Client.putObject( + { + it.bucket(bucketName) + it.key(sourceKey) + it.metadata(metadata) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + } + + s3Client.getObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.use { + val copiedDigest = DigestUtil.hexDigest(it) + assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) + assertThat(it.response().metadata()).isEqualTo(metadata) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object succeeds with new metadata`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, putObjectResult) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey/withNewUserMetadata" + + val metadata = mapOf("test-key2" to "test-value2") + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + it.metadata(metadata) + it.metadataDirective(MetadataDirective.REPLACE) + } + + s3Client.getObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.use { + val copiedDigest = DigestUtil.hexDigest(it) + assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) + assertThat(it.response().metadata()).isEqualTo(metadata) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object succeeds with new storageclass`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + s3Client.putObject( + { + it.bucket(bucketName) + it.key(sourceKey) + it.storageClass(StorageClass.REDUCED_REDUNDANCY) + }, + RequestBody.fromFile(uploadFile) + ) + + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + //must set storage class other than "STANDARD" to it gets applied. + .storageClass(StorageClass.STANDARD_IA) + } + + s3Client.getObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.use { + assertThat(it.response().storageClass()).isEqualTo(StorageClass.STANDARD_IA) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object succeeds with overwriting stored headers`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + s3Client.putObject( + { + it.bucket(bucketName) + it.key(sourceKey) + it.contentDisposition("") + }, + RequestBody.fromFile(uploadFile) + ) + + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + it.metadataDirective(MetadataDirective.REPLACE) + it.contentDisposition("attachment") + } + + s3Client.getObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.use { + assertThat(it.response().contentDisposition()).isEqualTo("attachment") + } + } + + @Test + @S3VerifiedFailure(year = 2025, + reason = "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm") + fun `copy object succeeds with encryption`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + it.sseCustomerKey(TEST_ENC_KEY_ID) + } + + s3Client.headObject { + it.bucket(destinationBucketName) + it.key(destinationKey) + }.also { + assertThat(it.eTag()).isEqualTo(putObjectResponse.eTag()) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object fails with wrong encryption key`(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + assertThatThrownBy { + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + it.serverSideEncryption(ServerSideEncryption.AWS_KMS) + it.ssekmsKeyId(TEST_WRONG_KEY_ID) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 400") + .hasMessageContaining("Invalid keyId 'key-ID-WRONGWRONGWRONG'") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object fails with non existing source key`(testInfo: TestInfo) { + val sourceKey = randomName + val bucketName = givenBucket(testInfo) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + assertThatThrownBy { + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 404") + .hasMessageContaining("The specified key does not exist.") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun `copy object with transfermanager succeeds`(testInfo: TestInfo) { + //content larger than default part threshold of 8MiB + val contentLen = 20 * _1MB + val sourceKey = UPLOAD_FILE_NAME + val bucketName = givenBucket(testInfo) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + val upload = transferManager.upload { + it.putObjectRequest { + it.key(sourceKey) + it.bucket(bucketName) + } + it.requestBody(AsyncRequestBody.fromInputStream(randomInputStream(contentLen), + contentLen.toLong(), + Executors.newFixedThreadPool(10))) + }.completionFuture().join() + + transferManager.copy { + it.copyObjectRequest { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + } + }.completionFuture().join().also { + assertThat(it.response().copyObjectResult().eTag()).isEqualTo(upload.response().eTag()) + } + } +} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectV1IT.kt deleted file mode 100644 index 9ddda54fb..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectV1IT.kt +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.adobe.testing.s3mock.its - -import com.adobe.testing.s3mock.util.DigestUtil -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.AmazonS3Exception -import com.amazonaws.services.s3.model.CopyObjectRequest -import com.amazonaws.services.s3.model.MetadataDirective -import com.amazonaws.services.s3.model.ObjectMetadata -import com.amazonaws.services.s3.model.PutObjectRequest -import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams -import com.amazonaws.services.s3.transfer.TransferManager -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import java.io.File -import java.io.FileInputStream -import java.util.UUID - -/** - * Test the application using the AmazonS3 SDK V1. - */ -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class CopyObjectV1IT : S3TestBase() { - - private val s3Client: AmazonS3 = createS3ClientV1() - private val transferManagerV1: TransferManager = createTransferManagerV1() - - /** - * Puts an Object; Copies that object to a new bucket; Downloads the object from the new bucket; - * compares checksums of original and copied object. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObject(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey).also { - s3Client.copyObject(it) - } - - s3Client.getObject(destinationBucketName, destinationKey).use { - val copiedDigest = DigestUtil.hexDigest(it.objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_successMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey) - val matchingEtag = "\"${putObjectResult.eTag}\"" - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey) - .withMatchingETagConstraint(matchingEtag).also { - s3Client.copyObject(it) - } - - s3Client.getObject(destinationBucketName, destinationKey).use { - val copiedDigest = DigestUtil.hexDigest(it.objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_successNoneMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey) - val nonMatchingEtag = "\"${randomName}\"" - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey) - .withNonmatchingETagConstraint(nonMatchingEtag).also { - s3Client.copyObject(it) - } - - s3Client.getObject(destinationBucketName, destinationKey).use { - val copiedDigest = DigestUtil.hexDigest(it.objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_failureMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV1(testInfo, sourceKey) - val nonMatchingEtag = "\"${randomName}\"" - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey) - .withMatchingETagConstraint(nonMatchingEtag).also { - s3Client.copyObject(it) - } - - assertThatThrownBy { - s3Client.getObject(destinationBucketName, destinationKey) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Service: Amazon S3; Status Code: 404; Error Code: NoSuchKey;") - .hasMessageContaining("The specified key does not exist.") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_failureNoneMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey) - val matchingEtag = "\"${putObjectResult.eTag}\"" - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey) - .withNonmatchingETagConstraint(matchingEtag).also { - s3Client.copyObject(it) - } - - assertThatThrownBy { - s3Client.getObject(destinationBucketName, destinationKey) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Service: Amazon S3; Status Code: 404; Error Code: NoSuchKey;") - .hasMessageContaining("The specified key does not exist.") - } - - /** - * Puts an Object; Copies that object to the same bucket and the same key; - * Downloads the object; compares checksums of original and copied object. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObjectToSameKey(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - val objectMetadata = ObjectMetadata().apply { - this.userMetadata = mapOf("test-key" to "test-value") - } - val putObjectResult = PutObjectRequest(bucketName, sourceKey, uploadFile).withMetadata(objectMetadata).let { - s3Client.putObject(it) - } - - CopyObjectRequest(bucketName, sourceKey, bucketName, sourceKey).apply { - this.newObjectMetadata = ObjectMetadata().apply { - this.userMetadata = mapOf("test-key1" to "test-value1") - } - }.also { - s3Client.copyObject(it) - } - - s3Client.getObject(bucketName, sourceKey).use { - val copiedObjectMetadata = it.objectMetadata - assertThat(copiedObjectMetadata.userMetadata["test-key"]).isNull() - assertThat(copiedObjectMetadata.userMetadata["test-key1"]).isEqualTo("test-value1") - - val objectContent = it.objectContent - val copiedDigest = DigestUtil.hexDigest(objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - } - } - - /** - * Puts an Object; Copies that object with REPLACE directive to the same bucket and the same key; - * Downloads the object; compares checksums of original and copied object. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObjectWithReplaceToSameKey(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - val objectMetadata = ObjectMetadata().apply { - this.userMetadata = mapOf("test-key" to "test-value") - } - val putObjectResult = PutObjectRequest(bucketName, sourceKey, uploadFile).withMetadata(objectMetadata).let { - s3Client.putObject(it) - } - - val replaceObjectMetadata = ObjectMetadata().apply { - this.userMetadata = mapOf("test-key2" to "test-value2") - } - CopyObjectRequest() - .withSourceBucketName(bucketName) - .withSourceKey(sourceKey) - .withDestinationBucketName(bucketName) - .withDestinationKey(sourceKey) - .withMetadataDirective(MetadataDirective.REPLACE) - .withNewObjectMetadata(replaceObjectMetadata) - .also { - s3Client.copyObject(it) - } - - - s3Client.getObject(bucketName, sourceKey).use { - val copiedObjectMetadata = it.objectMetadata - assertThat(copiedObjectMetadata.userMetadata["test-key"]).isNullOrEmpty() - assertThat(copiedObjectMetadata.userMetadata["test-key2"]).isEqualTo("test-value2") - - val objectContent = it.objectContent - val copiedDigest = DigestUtil.hexDigest(objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - } - } - - /** - * Puts an Object; Copies that object to a new bucket with new user metadata; Downloads the - * object from the new bucket; - * compares checksums of original and copied object; compares copied object user metadata with - * the new user metadata specified during copy request. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObjectWithNewUserMetadata(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey/withNewUserMetadata" - val objectMetadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - } - - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey).apply { - this.newObjectMetadata = objectMetadata - }.also { - s3Client.copyObject(it) - } - - s3Client.getObject(destinationBucketName, destinationKey).use { - val copiedDigest = DigestUtil.hexDigest(it.objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - assertThat(it.objectMetadata.userMetadata).isEqualTo(objectMetadata.userMetadata) - } - } - - /** - * Puts an Object with some user metadata; Copies that object to a new bucket. - * Downloads the object from the new bucket; - * compares checksums of original and copied object; compares copied object user metadata with - * the source object user metadata; - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObjectWithSourceUserMetadata(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey/withSourceObjectUserMetadata" - val sourceObjectMetadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - } - val putObjectResult = PutObjectRequest(bucketName, sourceKey, uploadFile).apply { - this.metadata = sourceObjectMetadata - }.let { - s3Client.putObject(it) - } - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey).also { - s3Client.copyObject(it) - } - s3Client.getObject(destinationBucketName, destinationKey).use { - val copiedDigest = DigestUtil.hexDigest(it.objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - assertThat(it.objectMetadata.userMetadata).isEqualTo(sourceObjectMetadata.userMetadata) - } - } - - /** - * Copy an object to a key needing URL escaping. - * - * @see .shouldCopyObject - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObjectToKeyNeedingEscaping(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/some escape-worthy characters $@ $sourceKey" - val putObjectResult = s3Client.putObject(PutObjectRequest(bucketName, sourceKey, uploadFile)) - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey).also { - s3Client.copyObject(it) - } - - s3Client.getObject(destinationBucketName, destinationKey).use { - val copiedDigest = DigestUtil.hexDigest(it.objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - assertThat(it.objectMetadata.contentLength).isEqualTo(uploadFile.length()) - } - } - - /** - * Copy an object from a key needing URL escaping. - * - * @see .shouldCopyObject - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObjectFromKeyNeedingEscaping(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = "some escape-worthy characters $@ $UPLOAD_FILE_NAME" - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - val putObjectResult = s3Client.putObject(PutObjectRequest(bucketName, sourceKey, uploadFile)) - CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey).also { - s3Client.copyObject(it) - } - - s3Client.getObject(destinationBucketName, destinationKey).use { - val copiedDigest = DigestUtil.hexDigest(it.objectContent) - assertThat(copiedDigest).isEqualTo(putObjectResult.eTag) - } - } - - /** - * Puts an Object; Copies that object to a new bucket; Downloads the object from the new bucket; - * compares checksums of original and copied object. - */ - @Test - @S3VerifiedFailure(year = 2022, - reason = "No KMS configuration for AWS test account") - fun shouldCopyObjectEncrypted(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - s3Client.putObject(PutObjectRequest(bucketName, sourceKey, uploadFile)) - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - val copyObjectResult = CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey) - .apply { - this.sseAwsKeyManagementParams = SSEAwsKeyManagementParams(TEST_ENC_KEY_ID) - }.let { - s3Client.copyObject(it) - } - s3Client.getObjectMetadata(destinationBucketName, destinationKey).also { - assertThat(it.contentLength).isEqualTo(uploadFile.length()) - } - - val uploadDigest = FileInputStream(uploadFile).let { - DigestUtil.hexDigest(TEST_ENC_KEY_ID, it) - } - assertThat(copyObjectResult.eTag).isEqualTo(uploadDigest) - } - - /** - * Tests that an object won't be copied with wrong encryption Key. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldNotObjectCopyWithWrongEncryptionKey(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV1(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf$sourceKey" - val copyObjectRequest = CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey).apply { - this.sseAwsKeyManagementParams = SSEAwsKeyManagementParams(TEST_WRONG_KEY_ID) - } - - assertThatThrownBy { s3Client.copyObject(copyObjectRequest) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 400; Error Code: KMS.NotFoundException") - } - - /** - * Tests that a copy request for a non-existing object throws the correct error. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldThrowNoSuchKeyOnCopyForNonExistingKey(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val sourceKey = randomName - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf$sourceKey" - val copyObjectRequest = CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey) - - assertThatThrownBy { s3Client.copyObject(copyObjectRequest) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 404; Error Code: NoSuchKey") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun multipartCopy() { - //content larger than default part threshold of 5MiB - val contentLen = 10 * _1MB - val objectMetadata = ObjectMetadata().apply { - this.contentLength = contentLen.toLong() - } - val assumedSourceKey = UUID.randomUUID().toString() - val sourceBucket = givenRandomBucketV1() - val targetBucket = givenRandomBucketV1() - val upload = transferManagerV1 - .upload( - sourceBucket, assumedSourceKey, - randomInputStream(contentLen), objectMetadata - ) - - val uploadResult = upload.waitForUploadResult().also { - assertThat(it.key).isEqualTo(assumedSourceKey) - } - - val assumedDestinationKey = UUID.randomUUID().toString() - transferManagerV1.copy( - sourceBucket, assumedSourceKey, targetBucket, - assumedDestinationKey - ).waitForCopyResult().also { - assertThat(it.destinationKey).isEqualTo(assumedDestinationKey) - assertThat(uploadResult.eTag).isEqualTo(it.eTag) - } - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectV2IT.kt deleted file mode 100644 index b2e8ec57c..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CopyObjectV2IT.kt +++ /dev/null @@ -1,429 +0,0 @@ -/* - * Copyright 2017-2024 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.adobe.testing.s3mock.its - -import com.adobe.testing.s3mock.S3Exception.PRECONDITION_FAILED -import com.adobe.testing.s3mock.util.DigestUtil -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.awaitility.Awaitility.await -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.CopyObjectRequest -import software.amazon.awssdk.services.s3.model.GetObjectRequest -import software.amazon.awssdk.services.s3.model.HeadObjectRequest -import software.amazon.awssdk.services.s3.model.MetadataDirective -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.S3Exception -import software.amazon.awssdk.services.s3.model.StorageClass -import java.io.File -import java.time.Duration -import java.time.Instant -import java.time.temporal.ChronoUnit.SECONDS - -/** - * Test the application using the AmazonS3 SDK V2. - */ -internal class CopyObjectV2IT : S3TestBase() { - - private val s3ClientV2: S3Client = createS3ClientV2() - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - - s3ClientV2.copyObject(CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .build()) - - s3ClientV2.getObject(GetObjectRequest - .builder() - .bucket(destinationBucketName) - .key(destinationKey) - .build() - ).use { - val copiedDigest = DigestUtil.hexDigest(it) - assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_successMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - - val matchingEtag = putObjectResult.eTag() - s3ClientV2.copyObject(CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .copySourceIfMatch(matchingEtag) - .build()) - - s3ClientV2.getObject(GetObjectRequest - .builder() - .bucket(destinationBucketName) - .key(destinationKey) - .build() - ).use { - val copiedDigest = DigestUtil.hexDigest(it) - assertThat("\"$copiedDigest\"").isEqualTo(matchingEtag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_successNoneMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val noneMatchingEtag = "\"${randomName}\"" - s3ClientV2.copyObject(CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .copySourceIfNoneMatch(noneMatchingEtag) - .build()) - - s3ClientV2.getObject(GetObjectRequest - .builder() - .bucket(destinationBucketName) - .key(destinationKey) - .build() - ).use { - val copiedDigest = DigestUtil.hexDigest(it) - assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_failureMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val noneMatchingEtag = "\"${randomName}\"" - - assertThatThrownBy { - s3ClientV2.copyObject(CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .copySourceIfMatch(noneMatchingEtag) - .build()) - } - .isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 412") - .hasMessageContaining(PRECONDITION_FAILED.message) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_failureNoneMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val matchingEtag = putObjectResult.eTag() - - assertThatThrownBy { - s3ClientV2.copyObject(CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .copySourceIfNoneMatch(matchingEtag) - .build()) - } - .isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 412") - .hasMessageContaining(PRECONDITION_FAILED.message) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObjectToSameBucketAndKey(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - val putObjectResult = s3ClientV2.putObject(PutObjectRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .metadata(mapOf("test-key" to "test-value")) - .build(), - RequestBody.fromFile(uploadFile) - ) - val sourceLastModified = s3ClientV2.headObject( - HeadObjectRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build() - ).lastModified() - - await("wait until source object is 5 seconds old").until { - sourceLastModified.plusSeconds(5).isBefore(Instant.now()) - } - - s3ClientV2.copyObject( - CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(bucketName) - .destinationKey(sourceKey) - .metadata(mapOf("test-key2" to "test-value2")) - .metadataDirective(MetadataDirective.REPLACE) - .build() - ) - - s3ClientV2.getObject(GetObjectRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build() - ).use { - val response = it.response() - val copiedObjectMetadata = response.metadata() - assertThat(copiedObjectMetadata["test-key2"]).isEqualTo("test-value2") - assertThat(copiedObjectMetadata["test-key"]).isNull() - - val length = response.contentLength() - assertThat(length).isEqualTo(uploadFile.length()) - - val copiedDigest = DigestUtil.hexDigest(it) - assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) - - //we waited for 5 seconds above, so last modified dates should be about 5 seconds apart - val between = Duration.between(sourceLastModified, response.lastModified()) - assertThat(between).isCloseTo(Duration.of(5, SECONDS), Duration.of(1, SECONDS)) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObjectToSameBucketAndKey_throws(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - s3ClientV2.putObject(PutObjectRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .metadata(mapOf("test-key" to "test-value")) - .build(), - RequestBody.fromFile(uploadFile) - ) - val sourceLastModified = s3ClientV2.headObject( - HeadObjectRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build() - ).lastModified() - - await("wait until source object is 5 seconds old").until { - sourceLastModified.plusSeconds(5).isBefore(Instant.now()) - } - - assertThatThrownBy { - s3ClientV2.copyObject( - CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(bucketName) - .destinationKey(sourceKey) - .build() - ) - }.isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 400") - .hasMessageContaining("This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObjectWithNewMetadata(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey/withNewUserMetadata" - - val metadata = mapOf("test-key2" to "test-value2") - s3ClientV2.copyObject( - CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .metadata(metadata) - .metadataDirective(MetadataDirective.REPLACE) - .build() - ) - - s3ClientV2.getObject(GetObjectRequest - .builder() - .bucket(destinationBucketName) - .key(destinationKey) - .build() - ).use { - val copiedDigest = DigestUtil.hexDigest(it) - assertThat("\"$copiedDigest\"").isEqualTo(putObjectResult.eTag()) - assertThat(it.response().metadata()).isEqualTo(metadata) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testCopyObject_storageClass(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(sourceKey) - .storageClass(StorageClass.REDUCED_REDUNDANCY) - .build(), - RequestBody.fromFile(uploadFile) - ) - - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - - s3ClientV2.copyObject(CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - //must set storage class other than "STANDARD" to it gets applied. - .storageClass(StorageClass.STANDARD_IA) - .build()) - - s3ClientV2.getObject(GetObjectRequest - .builder() - .bucket(destinationBucketName) - .key(destinationKey) - .build() - ).use { - assertThat(it.response().storageClass()).isEqualTo(StorageClass.STANDARD_IA) - } - } - - @Test - @S3VerifiedTodo - fun testCopyObject_overwriteStoreHeader(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(sourceKey) - .contentDisposition("") - .build(), - RequestBody.fromFile(uploadFile) - ) - - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - - s3ClientV2.copyObject(CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .metadataDirective(MetadataDirective.REPLACE) - .contentDisposition("attachment") - .build()) - - s3ClientV2.getObject(GetObjectRequest - .builder() - .bucket(destinationBucketName) - .key(destinationKey) - .build() - ).use { - assertThat(it.response().contentDisposition()).isEqualTo("attachment") - } - } - - @Test - @S3VerifiedTodo - fun testCopyObject_encrypted(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(sourceKey) - .build(), - RequestBody.fromFile(uploadFile) - ) - - val destinationBucketName = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - - s3ClientV2.copyObject(CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .sseCustomerKey(TEST_ENC_KEY_ID) - .build() - ) - - s3ClientV2.headObject(HeadObjectRequest - .builder() - .bucket(destinationBucketName) - .key(destinationKey) - .build() - ).also { - assertThat(it.contentLength()).isEqualTo(uploadFile.length()) - } - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CorsV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CorsIT.kt similarity index 95% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CorsV2IT.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CorsIT.kt index 7a8998b68..a409b8b73 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CorsV2IT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CorsIT.kt @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Adobe. + * Copyright 2017-2025 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,14 +32,14 @@ import java.util.UUID /** * Test the application using the AmazonS3 SDK V2. */ -internal class CorsV2IT : S3TestBase() { +internal class CorsIT : S3TestBase() { private val httpClient: CloseableHttpClient = createHttpClient() @Test @S3VerifiedFailure(year = 2024, reason = "No credentials sent in plain HTTP request") fun testPutObject_cors(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) val optionsRequest = HttpOptions("$serviceEndpoint/${bucketName}/testObjectName").apply { this.addHeader("Origin", "http://localhost/") } @@ -66,7 +66,7 @@ internal class CorsV2IT : S3TestBase() { @S3VerifiedFailure(year = 2024, reason = "No credentials sent in plain HTTP request") fun testGetBucket_cors(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) val httpOptions = HttpOptions("$serviceEndpoint/$targetBucket").apply { this.addHeader(BasicHeader("Origin", "http://someurl.com")) this.addHeader(BasicHeader("Access-Control-Request-Method", "GET")) diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CrtAsyncV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CrtAsyncIT.kt similarity index 52% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CrtAsyncV2IT.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CrtAsyncIT.kt index 7911cd846..a30f07ece 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CrtAsyncV2IT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/CrtAsyncIT.kt @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Adobe. + * Copyright 2017-2025 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,48 +25,37 @@ import org.springframework.web.util.UriUtils import software.amazon.awssdk.core.async.AsyncRequestBody import software.amazon.awssdk.core.async.AsyncResponseTransformer import software.amazon.awssdk.services.s3.S3AsyncClient -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest -import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload -import software.amazon.awssdk.services.s3.model.CompletedPart -import software.amazon.awssdk.services.s3.model.CreateBucketRequest -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest -import software.amazon.awssdk.services.s3.model.GetObjectRequest -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.UploadPartRequest import software.amazon.awssdk.transfer.s3.S3TransferManager import software.amazon.awssdk.transfer.s3.model.DownloadRequest -import software.amazon.awssdk.transfer.s3.model.UploadRequest import java.io.ByteArrayInputStream import java.io.File import java.io.FileInputStream import java.io.InputStream import java.nio.charset.StandardCharsets -internal class CrtAsyncV2IT : S3TestBase() { +internal class CrtAsyncIT : S3TestBase() { - private val autoS3CrtAsyncClientV2: S3AsyncClient = createAutoS3CrtAsyncClientV2() - private val transferManagerV2: S3TransferManager = createTransferManagerV2() + private val autoS3CrtAsyncClient: S3AsyncClient = createAutoS3CrtAsyncClient() + private val transferManager: S3TransferManager = createTransferManager() @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testPutObject_etagCreation(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) val uploadFileIs: InputStream = FileInputStream(uploadFile) val expectedEtag = "\"${DigestUtil.hexDigest(uploadFileIs)}\"" val bucketName = randomName - autoS3CrtAsyncClientV2 - .createBucket(CreateBucketRequest - .builder() - .bucket(bucketName) - .build() - ).join() - - val putObjectResponse = autoS3CrtAsyncClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build(), + autoS3CrtAsyncClient + .createBucket { + it.bucket(bucketName) + }.join() + + val putObjectResponse = autoS3CrtAsyncClient.putObject( + { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }, AsyncRequestBody.fromFile(uploadFile) ).join() @@ -77,29 +66,28 @@ internal class CrtAsyncV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testPutGetObject_successWithMatchingEtag(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) val bucketName = randomName - autoS3CrtAsyncClientV2 - .createBucket(CreateBucketRequest - .builder() - .bucket(bucketName) - .build() - ).join() - - val eTag = autoS3CrtAsyncClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName).key(UPLOAD_FILE_NAME).build(), + autoS3CrtAsyncClient + .createBucket { + it.bucket(bucketName) + }.join() + + val eTag = autoS3CrtAsyncClient.putObject( + { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }, AsyncRequestBody.fromFile(uploadFile) ).join().eTag() - autoS3CrtAsyncClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build(), + autoS3CrtAsyncClient.getObject( + { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }, AsyncResponseTransformer.toBytes() ).join().also { assertThat(it.response().eTag()).isEqualTo(eTag) @@ -108,69 +96,57 @@ internal class CrtAsyncV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testMultipartUpload(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) val uploadFile = File(UPLOAD_FILE_NAME) val objectMetadata = mapOf(Pair("key", "value")) - val createMultipartUploadResponseCompletableFuture = autoS3CrtAsyncClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest.builder().bucket(bucketName).key(UPLOAD_FILE_NAME) - .metadata(objectMetadata).build() - ) + val createMultipartUploadResponseCompletableFuture = autoS3CrtAsyncClient + .createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.metadata(objectMetadata) + } val initiateMultipartUploadResult = createMultipartUploadResponseCompletableFuture.join() val uploadId = initiateMultipartUploadResult.uploadId() // upload part 1, >5MB val randomBytes = randomBytes() val partETag = uploadPart(bucketName, UPLOAD_FILE_NAME, uploadId, 1, randomBytes) // upload part 2, <5MB - val uploadPartResponse = autoS3CrtAsyncClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .partNumber(2) - .contentLength(uploadFile.length()).build(), - //.lastPart(true) + val uploadPartResponse = autoS3CrtAsyncClient.uploadPart( + { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.partNumber(2) + it.contentLength(uploadFile.length()) + //it.lastPart(true) + }, AsyncRequestBody.fromFile(uploadFile), ).join() - val completeMultipartUploadResponse = autoS3CrtAsyncClientV2.completeMultipartUpload( - CompleteMultipartUploadRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(initiateMultipartUploadResult.uploadId()) - .multipartUpload( - CompletedMultipartUpload - .builder() - .parts( - CompletedPart - .builder() - .eTag(partETag) - .partNumber(1) - .build(), - CompletedPart - .builder() - .eTag(uploadPartResponse.eTag()) - .partNumber(2) - .build() - ) - .build() - ) - .build() - ).join() + val completeMultipartUploadResponse = autoS3CrtAsyncClient.completeMultipartUpload { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(initiateMultipartUploadResult.uploadId()) + it.multipartUpload { + it.parts( + { + it.eTag(partETag) + it.partNumber(1) + }, + { + it.eTag(uploadPartResponse.eTag()) + it.partNumber(2) + }) + } + }.join() // Verify only 1st and 3rd counts - val getObjectResponse = autoS3CrtAsyncClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build(), - AsyncResponseTransformer.toBytes() - ).join() + val getObjectResponse = autoS3CrtAsyncClient.getObject({ + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }, AsyncResponseTransformer.toBytes()).join() val uploadFileBytes = readStreamIntoByteArray(uploadFile.inputStream()) (DigestUtils.md5(randomBytes) + DigestUtils.md5(uploadFileBytes)).also { @@ -197,31 +173,31 @@ internal class CrtAsyncV2IT : S3TestBase() { partNumber: Int, randomBytes: ByteArray ): String { - return autoS3CrtAsyncClientV2 + return autoS3CrtAsyncClient .uploadPart( - UploadPartRequest.builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .partNumber(partNumber) - .contentLength(randomBytes.size.toLong()).build(), + { + it.bucket(bucketName) + it.key(key) + it.uploadId(uploadId) + it.partNumber(partNumber) + it.contentLength(randomBytes.size.toLong()) + }, AsyncRequestBody.fromBytes(randomBytes) ).join() .eTag() } @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testStreamUploadOfUnknownSize(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) val body = AsyncRequestBody.forBlockingInputStream(null) - val putObjectResponseFuture = autoS3CrtAsyncClientV2.putObject( - PutObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build(), + val putObjectResponseFuture = autoS3CrtAsyncClient.putObject( + { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }, body ) @@ -230,12 +206,11 @@ internal class CrtAsyncV2IT : S3TestBase() { putObjectResponseFuture.join() - val getObjectResponse = autoS3CrtAsyncClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build(), + val getObjectResponse = autoS3CrtAsyncClient.getObject( + { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }, AsyncResponseTransformer.toBytes() ).join() @@ -249,42 +224,33 @@ internal class CrtAsyncV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testStreamUploadOfUnknownSize_transferManager(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) val body = AsyncRequestBody.forBlockingInputStream(null) - val upload = transferManagerV2 - .upload( - UploadRequest - .builder() - .requestBody(body) - .putObjectRequest( - PutObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) - .build() - ) + val upload = transferManager + .upload { + it.requestBody(body) + it.putObjectRequest { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + } + } val randomBytes = randomBytes() body.writeInputStream(ByteArrayInputStream(randomBytes)) upload.completionFuture().join() - val download = transferManagerV2 + val download = transferManager .download( DownloadRequest .builder() - .getObjectRequest( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) + .getObjectRequest { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + } .responseTransformer(AsyncResponseTransformer.toBytes()) .build() ).completionFuture().join().result() diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ErrorResponsesV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ErrorResponsesV1IT.kt deleted file mode 100644 index 0900ba47d..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ErrorResponsesV1IT.kt +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest -import com.amazonaws.services.s3.model.AmazonS3Exception -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest -import com.amazonaws.services.s3.model.CopyObjectRequest -import com.amazonaws.services.s3.model.DeleteObjectsRequest -import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion -import com.amazonaws.services.s3.model.GetObjectRequest -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest -import com.amazonaws.services.s3.model.ObjectMetadata -import com.amazonaws.services.s3.model.PutObjectRequest -import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams -import com.amazonaws.services.s3.model.UploadPartRequest -import com.amazonaws.services.s3.transfer.TransferManager -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import java.io.File -import java.util.UUID - -/** - * Test the application using the AmazonS3 SDK V1. - * Verifies S3 Mocks Error Responses. - */ -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class ErrorResponsesV1IT : S3TestBase() { - - private val s3Client: AmazonS3 = createS3ClientV1() - private val transferManagerV1: TransferManager = createTransferManagerV1() - - @Test - @S3VerifiedSuccess(year = 2024) - fun getObject_noSuchKey(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val getObjectRequest = GetObjectRequest(bucketName, NON_EXISTING_KEY) - assertThatThrownBy { s3Client.getObject(getObjectRequest) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_KEY) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun getObject_noSuchKey_startingSlash(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val getObjectRequest = GetObjectRequest(bucketName, "/$NON_EXISTING_KEY") - assertThatThrownBy { s3Client.getObject(getObjectRequest) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_KEY) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun putObject_noSuchBucket() { - val uploadFile = File(UPLOAD_FILE_NAME) - assertThatThrownBy { - s3Client.putObject( - PutObjectRequest( - randomName, - UPLOAD_FILE_NAME, - uploadFile - ) - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun putObjectEncrypted_noSuchBucket() { - val uploadFile = File(UPLOAD_FILE_NAME) - PutObjectRequest(randomName, UPLOAD_FILE_NAME, uploadFile).apply { - this.sseAwsKeyManagementParams = SSEAwsKeyManagementParams(TEST_ENC_KEY_ID) - } - assertThatThrownBy { - s3Client.putObject( - PutObjectRequest( - randomName, - UPLOAD_FILE_NAME, - uploadFile - ) - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun copyObjectToNonExistingDestination_noSuchBucket(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val destinationBucketName = randomName - val destinationKey = "copyOf/$sourceKey" - val copyObjectRequest = CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey) - assertThatThrownBy { s3Client.copyObject(copyObjectRequest) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun copyObjectEncryptedToNonExistingDestination_noSuchBucket(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV1(testInfo, sourceKey) - val destinationBucketName = randomName - val destinationKey = "copyOf/$sourceKey" - val copyObjectRequest = CopyObjectRequest(bucketName, sourceKey, destinationBucketName, destinationKey).apply { - this.sseAwsKeyManagementParams = SSEAwsKeyManagementParams(TEST_ENC_KEY_ID) - } - assertThatThrownBy { s3Client.copyObject(copyObjectRequest) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun getObjectMetadata_noSuchBucket() { - assertThatThrownBy { - s3Client.getObjectMetadata( - randomName, - UPLOAD_FILE_NAME - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(STATUS_CODE_404) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun deleteFrom_noSuchBucket() { - assertThatThrownBy { - s3Client.deleteObject( - randomName, - UPLOAD_FILE_NAME - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun deleteObject_nonExistent_OK(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - s3Client.deleteObject(bucketName, randomName) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun batchDeleteObjects_noSuchBucket() { - val multiObjectDeleteRequest = DeleteObjectsRequest(randomName).apply { - this.keys = listOf(KeyVersion("1_$UPLOAD_FILE_NAME")) - } - assertThatThrownBy { s3Client.deleteObjects(multiObjectDeleteRequest) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun deleteBucket_noSuchBucket() { - assertThatThrownBy { s3Client.deleteBucket(randomName) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun listObjects_noSuchBucket() { - assertThatThrownBy { - s3Client.listObjects( - randomName, - UPLOAD_FILE_NAME - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun uploadParallel_noSuchBucket() { - val uploadFile = File(UPLOAD_FILE_NAME) - assertThatThrownBy { - val upload = transferManagerV1.upload( - PutObjectRequest(randomName, UPLOAD_FILE_NAME, uploadFile) - ) - upload.waitForUploadResult() - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun multipartUploads_noSuchBucket() { - assertThatThrownBy { - s3Client.initiateMultipartUpload( - InitiateMultipartUploadRequest(randomName, UPLOAD_FILE_NAME) - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun listMultipartUploads_noSuchBucket() { - assertThatThrownBy { - s3Client.listMultipartUploads( - ListMultipartUploadsRequest(randomName) - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun abortMultipartUpload_noSuchBucket() { - assertThatThrownBy { - s3Client.abortMultipartUpload( - AbortMultipartUploadRequest( - randomName, - UPLOAD_FILE_NAME, - "uploadId" - ) - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun uploadMultipart_invalidPartNumber(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload(InitiateMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME)) - val uploadId = initiateMultipartUploadResult.uploadId - assertThat( - s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)) - .multipartUploads - ).isNotEmpty - val invalidPartNumber = 0 - assertThatThrownBy { - s3Client.uploadPart( - UploadPartRequest() - .withBucketName(initiateMultipartUploadResult.bucketName) - .withKey(initiateMultipartUploadResult.key) - .withUploadId(uploadId) - .withFile(uploadFile) - .withFileOffset(0) - .withPartNumber(invalidPartNumber) - .withPartSize(uploadFile.length()) - .withLastPart(true) - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(INVALID_PART_NUMBER) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun completeMultipartUploadWithNonExistingPartNumber(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload(InitiateMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME)) - val uploadId = initiateMultipartUploadResult.uploadId - assertThat( - s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)) - .multipartUploads - ).isNotEmpty - val partETag = s3Client.uploadPart( - UploadPartRequest() - .withBucketName(initiateMultipartUploadResult.bucketName) - .withKey(initiateMultipartUploadResult.key) - .withUploadId(uploadId) - .withFile(uploadFile) - .withFileOffset(0) - .withPartNumber(1) - .withPartSize(uploadFile.length()) - .withLastPart(true) - ).partETag - - // Set to non-existing part number - partETag.partNumber = 2 - val partETags = listOf(partETag) - assertThatThrownBy { - s3Client.completeMultipartUpload( - CompleteMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME, uploadId, partETags) - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(INVALID_PART) - } - - @Test - @S3VerifiedSuccess(year = 2024) - @Throws(Exception::class) - fun rangeDownloadsFromNonExistingBucket() { - val transferManager = createTransferManagerV1() - val downloadFile = File.createTempFile(UUID.randomUUID().toString(), null) - assertThatThrownBy { - transferManager.download( - GetObjectRequest(randomName, UPLOAD_FILE_NAME).withRange(1, 2), - downloadFile - ).waitForCompletion() - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(STATUS_CODE_404) - } - - @Test - @S3VerifiedSuccess(year = 2024) - @Throws(Exception::class) - fun rangeDownloadsFromNonExistingObject(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val upload = transferManagerV1.upload(PutObjectRequest(bucketName, UPLOAD_FILE_NAME, uploadFile)) - upload.waitForUploadResult() - val downloadFile = File.createTempFile(UUID.randomUUID().toString(), null) - assertThatThrownBy { - transferManagerV1.download( - GetObjectRequest(bucketName, randomName).withRange(1, 2), - downloadFile - ).waitForCompletion() - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(STATUS_CODE_404) - } - - @Test - @S3VerifiedSuccess(year = 2024) - @Throws(InterruptedException::class) - fun multipartCopyToNonExistingBucket(testInfo: TestInfo) { - val sourceBucket = givenBucketV1(testInfo) - val destinationBucket = randomName - //content larger than default part threshold of 5MiB - val contentLen = 7 * _1MB - val objectMetadata = ObjectMetadata().apply { - this.contentLength = contentLen.toLong() - } - val assumedSourceKey = randomName - val sourceInputStream = randomInputStream(contentLen) - val upload = transferManagerV1 - .upload( - sourceBucket, assumedSourceKey, - sourceInputStream, objectMetadata - ) - val uploadResult = upload.waitForUploadResult() - assertThat(uploadResult.key).isEqualTo(assumedSourceKey) - val assumedDestinationKey = randomName - assertThatThrownBy { - transferManagerV1.copy( - sourceBucket, - assumedSourceKey, - destinationBucket, - assumedDestinationKey - ).waitForCopyResult() - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - @Throws(InterruptedException::class) - fun multipartCopyNonExistingObject(testInfo: TestInfo) { - val sourceBucket = givenBucketV1(testInfo) - val targetBucket = givenRandomBucketV1() - //content larger than default part threshold of 5MiB - val contentLen = 7 * _1MB - val objectMetadata = ObjectMetadata().apply { - this.contentLength = contentLen.toLong() - } - val assumedSourceKey = randomName - val sourceInputStream = randomInputStream(contentLen) - val upload = transferManagerV1 - .upload( - sourceBucket, assumedSourceKey, - sourceInputStream, objectMetadata - ) - val uploadResult = upload.waitForUploadResult() - assertThat(uploadResult.key).isEqualTo(assumedSourceKey) - val assumedDestinationKey = randomName - assertThatThrownBy { - transferManagerV1.copy( - sourceBucket, randomName, - targetBucket, assumedDestinationKey - ).waitForCopyResult() - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining(STATUS_CODE_404) - } - - companion object { - private const val NON_EXISTING_KEY = "NoSuchKey.json" - private const val NO_SUCH_BUCKET = "Status Code: 404; Error Code: NoSuchBucket" - private const val NO_SUCH_KEY = "Status Code: 404; Error Code: NoSuchKey" - private const val STATUS_CODE_404 = "Status Code: 404" - private const val INVALID_PART_NUMBER = "Status Code: 400; Error Code: InvalidArgument" - private const val INVALID_PART = "Status Code: 400; Error Code: InvalidPart" - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ErrorResponsesV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ErrorResponsesV2IT.kt deleted file mode 100644 index 39c44f99d..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ErrorResponsesV2IT.kt +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2017-2024 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest -import software.amazon.awssdk.services.s3.model.Delete -import software.amazon.awssdk.services.s3.model.DeleteBucketRequest -import software.amazon.awssdk.services.s3.model.DeleteObjectRequest -import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest -import software.amazon.awssdk.services.s3.model.GetObjectRequest -import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest -import software.amazon.awssdk.services.s3.model.NoSuchBucketException -import software.amazon.awssdk.services.s3.model.NoSuchKeyException -import software.amazon.awssdk.services.s3.model.ObjectIdentifier -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.S3Exception -import software.amazon.awssdk.services.s3.model.UploadPartRequest -import java.io.File - -internal class ErrorResponsesV2IT : S3TestBase() { - - private val s3ClientV2: S3Client = createS3ClientV2() - - @Test - @S3VerifiedSuccess(year = 2024) - fun getObject_noSuchKey(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val req = GetObjectRequest.builder().bucket(bucketName).key(NON_EXISTING_KEY).build() - - assertThatThrownBy { s3ClientV2.getObject(req) }.isInstanceOf( - NoSuchKeyException::class.java - ).hasMessageContaining(NO_SUCH_KEY) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun getObject_noSuchKey_startingSlash(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val req = GetObjectRequest.builder().bucket(bucketName).key("/$NON_EXISTING_KEY").build() - - assertThatThrownBy { s3ClientV2.getObject(req) }.isInstanceOf( - NoSuchKeyException::class.java - ).hasMessageContaining(NO_SUCH_KEY) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun putObject_noSuchBucket() { - val uploadFile = File(UPLOAD_FILE_NAME) - - assertThatThrownBy { - s3ClientV2.putObject( - PutObjectRequest - .builder() - .bucket(randomName) - .key(UPLOAD_FILE_NAME) - .build(), - RequestBody.fromFile(uploadFile) - ) - } - .isInstanceOf(NoSuchBucketException::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun copyObjectToNonExistingDestination_noSuchBucket(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - val destinationBucketName = randomName - val destinationKey = "copyOf/$sourceKey" - - assertThatThrownBy { s3ClientV2.copyObject( - software.amazon.awssdk.services.s3.model.CopyObjectRequest - .builder() - .sourceBucket(bucketName) - .sourceKey(sourceKey) - .destinationBucket(destinationBucketName) - .destinationKey(destinationKey) - .build() - ) } - .isInstanceOf(NoSuchBucketException::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun deleteObject_noSuchBucket() { - assertThatThrownBy { - s3ClientV2.deleteObject( - DeleteObjectRequest - .builder() - .bucket(randomName) - .key(NON_EXISTING_KEY) - .build() - ) - } - .isInstanceOf(NoSuchBucketException::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun deleteObject_nonExistent_OK(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - - s3ClientV2.deleteObject( - DeleteObjectRequest - .builder() - .bucket(bucketName) - .key(NON_EXISTING_KEY) - .build() - ) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun deleteObjects_noSuchBucket() { - assertThatThrownBy { - s3ClientV2.deleteObjects( - DeleteObjectsRequest - .builder() - .bucket(randomName) - .delete( - Delete - .builder() - .objects(ObjectIdentifier - .builder() - .key(NON_EXISTING_KEY) - .build() - ).build() - ).build() - ) - } - .isInstanceOf(NoSuchBucketException::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun deleteBucket_noSuchBucket() { - assertThatThrownBy { - s3ClientV2.deleteBucket( - DeleteBucketRequest - .builder() - .bucket(randomName) - .build() - ) - } - .isInstanceOf(NoSuchBucketException::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun multipartUploads_noSuchBucket() { - assertThatThrownBy { - s3ClientV2.createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(randomName) - .key(UPLOAD_FILE_NAME) - .build() - ) - } - .isInstanceOf(NoSuchBucketException::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun listMultipartUploads_noSuchBucket() { - assertThatThrownBy { - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(randomName) - .build() - ) - } - .isInstanceOf(NoSuchBucketException::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun abortMultipartUpload_noSuchBucket() { - assertThatThrownBy { - s3ClientV2.abortMultipartUpload( - AbortMultipartUploadRequest - .builder() - .bucket(randomName) - .key(UPLOAD_FILE_NAME) - .uploadId("uploadId") - .build() - ) - } - .isInstanceOf(NoSuchBucketException::class.java) - .hasMessageContaining(NO_SUCH_BUCKET) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun uploadMultipart_invalidPartNumber(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - assertThat( - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(bucketName) - .build() - ).uploads() - ).isNotEmpty - - val invalidPartNumber = 0 - assertThatThrownBy { - s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .partNumber(invalidPartNumber) - .build(), - RequestBody.fromFile(uploadFile) - ) - } - .isInstanceOf(S3Exception::class.java) - .hasMessageContaining(INVALID_PART_NUMBER) - } - - companion object { - private const val NON_EXISTING_KEY = "NoSuchKey.json" - private const val NO_SUCH_KEY = "The specified key does not exist." - private const val NO_SUCH_BUCKET = "The specified bucket does not exist" - private const val STATUS_CODE_404 = "Status Code: 404" - private const val INVALID_PART_NUMBER = "Part number must be an integer between 1 and 10000, inclusive" - private const val INVALID_PART = "Status Code: 400; Error Code: InvalidPart" - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectIT.kt new file mode 100644 index 000000000..33222cd4d --- /dev/null +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectIT.kt @@ -0,0 +1,1254 @@ +/* + * Copyright 2017-2025 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adobe.testing.s3mock.its + +import com.adobe.testing.s3mock.util.DigestUtil +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.CsvSource +import org.junit.jupiter.params.provider.MethodSource +import org.springframework.http.ContentDisposition +import software.amazon.awssdk.core.async.AsyncRequestBody +import software.amazon.awssdk.core.checksums.Algorithm +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3AsyncClient +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm +import software.amazon.awssdk.services.s3.model.ChecksumMode +import software.amazon.awssdk.services.s3.model.HeadObjectRequest +import software.amazon.awssdk.services.s3.model.NoSuchBucketException +import software.amazon.awssdk.services.s3.model.NoSuchKeyException +import software.amazon.awssdk.services.s3.model.ObjectAttributes +import software.amazon.awssdk.services.s3.model.PutObjectRequest +import software.amazon.awssdk.services.s3.model.S3Exception +import software.amazon.awssdk.services.s3.model.ServerSideEncryption +import software.amazon.awssdk.services.s3.model.StorageClass +import java.io.File +import java.io.FileInputStream +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.concurrent.TimeUnit +import kotlin.math.min + +internal class GetPutDeleteObjectIT : S3TestBase() { + + private val s3Client: S3Client = createS3Client() + private val s3ClientHttp: S3Client = createS3Client(serviceEndpointHttp) + private val s3AsyncClient: S3AsyncClient = createS3AsyncClient() + private val s3AsyncClientHttp: S3AsyncClient = createS3AsyncClient(serviceEndpointHttp) + private val s3CrtAsyncClient: S3AsyncClient = createS3CrtAsyncClient() + private val s3CrtAsyncClientHttp: S3AsyncClient = createS3CrtAsyncClient(serviceEndpointHttp) + private val autoS3CrtAsyncClient: S3AsyncClient = createAutoS3CrtAsyncClient() + private val autoS3CrtAsyncClientHttp: S3AsyncClient = createAutoS3CrtAsyncClient(serviceEndpointHttp) + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutGetHeadDeleteObject(testInfo: TestInfo) { + val key = UPLOAD_FILE_NAME + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.headObject { + it.bucket(bucketName) + it.key(key) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + }.use { + assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) + } + + s3Client.deleteObject { + it.bucket(bucketName) + it.key(key) + } + + assertThatThrownBy { + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + } + } + .isInstanceOf(NoSuchKeyException::class.java) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutGetHeadDeleteObjects(testInfo: TestInfo) { + val key = UPLOAD_FILE_NAME + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + val keys = listOf("${key}-1", "${key}-2", "${key}-3") + keys.forEach { key -> + s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + } + + s3Client.deleteObjects { + it.bucket(bucketName) + it.delete { + it.objects( + { it.key("${key}-1") }, + { it.key("${key}-2") }, + { it.key("${key}-3") }, + ) + } + } + + keys.forEach { key -> + assertThatThrownBy { + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + } + } + .isInstanceOf(NoSuchKeyException::class.java) + } + } + + + @Test + @S3VerifiedSuccess(year = 2025) + fun getObject_noSuchKey(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + + assertThatThrownBy { + s3Client.getObject { + it.bucket(bucketName) + it.key(NON_EXISTING_KEY) + } + }.isInstanceOf( + NoSuchKeyException::class.java + ).hasMessageContaining(NO_SUCH_KEY) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun getObject_noSuchKey_startingSlash(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + + assertThatThrownBy { + s3Client.getObject { + it.bucket(bucketName) + it.key("/$NON_EXISTING_KEY") + } + }.isInstanceOf( + NoSuchKeyException::class.java + ).hasMessageContaining(NO_SUCH_KEY) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun putObject_noSuchBucket() { + val uploadFile = File(UPLOAD_FILE_NAME) + + assertThatThrownBy { + s3Client.putObject( + { + it.bucket(randomName) + it.key(UPLOAD_FILE_NAME) + }, + RequestBody.fromFile(uploadFile) + ) + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun putObjectEncrypted_noSuchBucket() { + val uploadFile = File(UPLOAD_FILE_NAME) + + assertThatThrownBy { + s3Client.putObject( + { + it.bucket(randomName) + it.key(UPLOAD_FILE_NAME) + it.serverSideEncryption(ServerSideEncryption.AWS_KMS) + it.ssekmsKeyId(TEST_ENC_KEY_ID) + }, + RequestBody.fromFile(uploadFile) + ) + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun headObject_noSuchBucket() { + assertThatThrownBy { + s3Client.headObject { + it.bucket(randomName) + it.key(UPLOAD_FILE_NAME) + } + } + //TODO: not sure why AWS SDK v2 does not return the correct exception here, S3Mock returns the correct error message. + .isInstanceOf(NoSuchKeyException::class.java) + //.isInstanceOf(NoSuchBucketException::class.java) + //.hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun headObject_noSuchKey(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + + assertThatThrownBy { + s3Client.headObject { + it.bucket(bucketName) + it.key(NON_EXISTING_KEY) + } + } + .isInstanceOf(NoSuchKeyException::class.java) + //TODO: not sure why AWS SDK v2 does not return the correct error message, S3Mock returns the correct message. + //.hasMessageContaining(NO_SUCH_KEY) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun copyObjectToNonExistingDestination_noSuchBucket(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, _) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + val destinationBucketName = randomName + val destinationKey = "copyOf/$sourceKey" + + assertThatThrownBy { + s3Client.copyObject { + it.sourceBucket(bucketName) + it.sourceKey(sourceKey) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + } + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun deleteObject_noSuchBucket() { + assertThatThrownBy { + s3Client.deleteObject { + it.bucket(randomName) + it.key(NON_EXISTING_KEY) + } + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun deleteObject_nonExistent_OK(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + + s3Client.deleteObject { + it.bucket(bucketName) + it.key(NON_EXISTING_KEY) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun deleteObjects_noSuchBucket() { + assertThatThrownBy { + s3Client.deleteObjects { + it.bucket(randomName) + it.delete { + it.objects({ + it.key(NON_EXISTING_KEY) + }) + } + } + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun deleteBucket_noSuchBucket() { + assertThatThrownBy { + s3Client.deleteBucket { + it.bucket(randomName) + } + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutGetHeadDeleteObjects_nonExistentKey(testInfo: TestInfo) { + val key = UPLOAD_FILE_NAME + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + givenObject(bucketName, key) + + s3Client.deleteObjects { + it.bucket(bucketName) + it.delete { + it.objects( + { it.key(key) }, + { it.key(randomName) }, + ) + } + } + } + + /** + * Test safe characters in object keys + * + * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + */ + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["charsSafe", "charsSpecial", "charsToAvoid"]) + fun testPutHeadGetObject_keyNames_safe(key: String, testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.headObject { + it.bucket(bucketName) + it.key(key) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + }.use { + assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) + } + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["storageClasses"]) + fun testPutObject_storageClass(storageClass: StorageClass, testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + val key = UPLOAD_FILE_NAME + + val eTag = s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + it.storageClass(storageClass) + }, + RequestBody.fromFile(uploadFile) + ).eTag() + + s3Client.headObject { + it.bucket(bucketName) + it.key(key) + }.also { + assertThat(it.eTag()).isEqualTo(eTag) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + }.use { + assertThat(it.response().eTag()).isEqualTo(eTag) + if (storageClass == StorageClass.STANDARD) { + //storageClass STANDARD is never returned from S3 APIs... + assertThat(it.response().storageClass()).isNull() + } else { + assertThat(it.response().storageClass()).isEqualTo(storageClass) + } + } + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["testFileNames"]) + fun testPutObject_etagCreation_sync(testFileName: String, testInfo: TestInfo) { + testEtagCreation(testFileName, s3Client, testInfo) + testEtagCreation(testFileName, s3ClientHttp, testInfo) + } + + private fun GetPutDeleteObjectIT.testEtagCreation( + testFileName: String, + s3Client: S3Client, + testInfo: TestInfo + ) { + val uploadFile = File(testFileName) + val expectedEtag = FileInputStream(uploadFile).let { + "\"${DigestUtil.hexDigest(it)}\"" + } + val bucketName = givenBucket(testInfo) + s3Client.putObject({ + it.bucket(bucketName) + it.key(testFileName) + }, + RequestBody.fromFile(uploadFile) + ).eTag().also { + assertThat(it).isNotBlank + assertThat(it).isEqualTo(expectedEtag) + } + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["testFileNames"]) + fun testPutObject_etagCreation_async(testFileName: String) { + testEtagCreation(testFileName, s3AsyncClient) + testEtagCreation(testFileName, s3AsyncClientHttp) + testEtagCreation(testFileName, s3CrtAsyncClient) + testEtagCreation(testFileName, s3CrtAsyncClientHttp) + testEtagCreation(testFileName, autoS3CrtAsyncClient) + testEtagCreation(testFileName, autoS3CrtAsyncClientHttp) + } + + private fun GetPutDeleteObjectIT.testEtagCreation( + testFileName: String, + s3Client: S3AsyncClient + ) { + val uploadFile = File(testFileName) + val expectedEtag = FileInputStream(uploadFile).let { + "\"${DigestUtil.hexDigest(it)}\"" + } + val bucketName = givenBucket(randomName) + s3Client.putObject({ + it.bucket(bucketName) + it.key(testFileName) + }, + AsyncRequestBody.fromFile(uploadFile) + ).join().eTag().also { + assertThat(it).isNotBlank + assertThat(it).isEqualTo(expectedEtag) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObject_getObjectAttributes(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), Algorithm.SHA1) + val bucketName = givenBucket(testInfo) + + val eTag = s3Client.putObject({ + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.checksumAlgorithm(ChecksumAlgorithm.SHA1) + }, + RequestBody.fromFile(uploadFile) + ).eTag() + + s3Client.getObjectAttributes { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.objectAttributes( + ObjectAttributes.OBJECT_SIZE, + ObjectAttributes.STORAGE_CLASS, + ObjectAttributes.E_TAG, + ObjectAttributes.CHECKSUM + ) + }.also { + // + assertThat(it.eTag()).isEqualTo(eTag.trim('"')) + //default storageClass is STANDARD, which is never returned from APIs + assertThat(it.storageClass()).isEqualTo(StorageClass.STANDARD) + assertThat(it.objectSize()).isEqualTo(File(UPLOAD_FILE_NAME).length()) + assertThat(it.checksum().checksumSHA1()).isEqualTo(expectedChecksum) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObject_objectMetadata(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + val eTag = s3Client.putObject({ + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.metadata(mapOf("key1" to "value1", "key2" to "value2")) + }, + RequestBody.fromFile(uploadFile) + ).eTag() + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }.also { + // + assertThat(it.response().metadata()).containsAllEntriesOf(mapOf("key1" to "value1", "key2" to "value2")) + } + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["checksumAlgorithms"]) + fun testPutObject_checksumAlgorithm_http(checksumAlgorithm: ChecksumAlgorithm) { + if(checksumAlgorithm != ChecksumAlgorithm.SHA256) { + //TODO: find out why the SHA256 checksum sent by the Java SDKv2 is wrong and this test is failing... + testChecksumAlgorithm(SAMPLE_FILE, checksumAlgorithm, s3ClientHttp) + testChecksumAlgorithm(SAMPLE_FILE_LARGE, checksumAlgorithm, s3ClientHttp) + testChecksumAlgorithm(TEST_IMAGE, checksumAlgorithm, s3ClientHttp) + testChecksumAlgorithm(TEST_IMAGE_LARGE, checksumAlgorithm, s3ClientHttp) + } + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["checksumAlgorithms"]) + fun testPutObject_checksumAlgorithm_https(checksumAlgorithm: ChecksumAlgorithm) { + testChecksumAlgorithm(SAMPLE_FILE, checksumAlgorithm, s3Client) + testChecksumAlgorithm(SAMPLE_FILE_LARGE, checksumAlgorithm, s3Client) + testChecksumAlgorithm(TEST_IMAGE, checksumAlgorithm, s3Client) + testChecksumAlgorithm(TEST_IMAGE_LARGE, checksumAlgorithm, s3Client) + } + + private fun GetPutDeleteObjectIT.testChecksumAlgorithm( + testFileName: String, + checksumAlgorithm: ChecksumAlgorithm, + s3Client: S3Client, + ) { + val uploadFile = File(testFileName) + val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) + val bucketName = givenBucket(randomName) + + s3Client.putObject({ + it.bucket(bucketName) + it.key(testFileName) + it.checksumAlgorithm(checksumAlgorithm) + }, + RequestBody.fromFile(uploadFile) + ).also { + val putChecksum = it.checksum(checksumAlgorithm) + assertThat(putChecksum).isNotBlank + assertThat(putChecksum).isEqualTo(expectedChecksum) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(testFileName) + it.checksumMode(ChecksumMode.ENABLED) + }.use { + val getChecksum = it.response().checksum(checksumAlgorithm) + assertThat(getChecksum).isNotBlank + assertThat(getChecksum).isEqualTo(expectedChecksum) + } + + s3Client.headObject { + it.bucket(bucketName) + it.key(testFileName) + it.checksumMode(ChecksumMode.ENABLED) + }.also { + val headChecksum = it.checksum(checksumAlgorithm) + assertThat(headChecksum).isNotBlank + assertThat(headChecksum).isEqualTo(expectedChecksum) + } + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["checksumAlgorithms"]) + fun testPutObject_checksumAlgorithm_async_http(checksumAlgorithm: ChecksumAlgorithm) { + testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, s3AsyncClientHttp) + testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, s3AsyncClientHttp) + testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, s3AsyncClientHttp) + testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, s3AsyncClientHttp) + + testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, s3CrtAsyncClientHttp) + testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, s3CrtAsyncClientHttp) + testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, s3CrtAsyncClientHttp) + testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, s3CrtAsyncClientHttp) + + testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, autoS3CrtAsyncClientHttp) + testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, autoS3CrtAsyncClientHttp) + testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, autoS3CrtAsyncClientHttp) + testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, autoS3CrtAsyncClientHttp) + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["checksumAlgorithms"]) + fun testPutObject_checksumAlgorithm_async_https(checksumAlgorithm: ChecksumAlgorithm) { + testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, s3AsyncClient) + testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, s3AsyncClient) + testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, s3AsyncClient) + testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, s3AsyncClient) + + testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, s3CrtAsyncClient) + testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, s3CrtAsyncClient) + testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, s3CrtAsyncClient) + testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, s3CrtAsyncClient) + + testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, autoS3CrtAsyncClient) + testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, autoS3CrtAsyncClient) + testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, autoS3CrtAsyncClient) + testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, autoS3CrtAsyncClient) + } + + private fun GetPutDeleteObjectIT.testChecksumAlgorithm_async( + testFileName: String, + checksumAlgorithm: ChecksumAlgorithm, + s3Client: S3AsyncClient, + ) { + val uploadFile = File(testFileName) + val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) + val bucketName = givenBucket(randomName) + + s3Client.putObject({ + it.bucket(bucketName) + it.key(testFileName) + it.checksumAlgorithm(checksumAlgorithm) + }, + AsyncRequestBody.fromFile(uploadFile) + ).join().also { + val putChecksum = it.checksum(checksumAlgorithm) + assertThat(putChecksum).isNotBlank + assertThat(putChecksum).isEqualTo(expectedChecksum) + } + + this.s3Client.getObject { + it.bucket(bucketName) + it.key(testFileName) + it.checksumMode(ChecksumMode.ENABLED) + }.use { + val getChecksum = it.response().checksum(checksumAlgorithm) + assertThat(getChecksum).isNotBlank + assertThat(getChecksum).isEqualTo(expectedChecksum) + } + + this.s3Client.headObject { + it.bucket(bucketName) + it.key(testFileName) + it.checksumMode(ChecksumMode.ENABLED) + }.also { + val headChecksum = it.checksum(checksumAlgorithm) + assertThat(headChecksum).isNotBlank + assertThat(headChecksum).isEqualTo(expectedChecksum) + } + } + + private fun PutObjectRequest.Builder + .checksum(checksum: String, checksumAlgorithm: ChecksumAlgorithm): PutObjectRequest.Builder = + when (checksumAlgorithm) { + ChecksumAlgorithm.SHA1 -> this.checksumSHA1(checksum) + ChecksumAlgorithm.SHA256 -> this.checksumSHA256(checksum) + ChecksumAlgorithm.CRC32 -> this.checksumCRC32(checksum) + ChecksumAlgorithm.CRC32_C -> this.checksumCRC32C(checksum) + //ChecksumAlgorithm.CRC64_NVME -> this.checksumCRC64NVME(checksum) + else -> error("Unknown checksum algorithm") + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["checksumAlgorithms"]) + fun testPutObject_checksum(checksumAlgorithm: ChecksumAlgorithm, testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) + val bucketName = givenBucket(testInfo) + + s3Client.putObject({ + it.checksum(expectedChecksum, checksumAlgorithm) + it.bucket(bucketName).key(UPLOAD_FILE_NAME) + }, + RequestBody.fromFile(uploadFile) + ).also { + val putChecksum = it.checksum(checksumAlgorithm)!! + assertThat(putChecksum).isNotBlank + assertThat(putChecksum).isEqualTo(expectedChecksum) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.checksumMode(ChecksumMode.ENABLED) + }.use { + val getChecksum = it.response().checksum(checksumAlgorithm) + assertThat(getChecksum).isNotBlank + assertThat(getChecksum).isEqualTo(expectedChecksum) + } + + s3Client.headObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.checksumMode(ChecksumMode.ENABLED) + }.also { + val headChecksum = it.checksum(checksumAlgorithm) + assertThat(headChecksum).isNotBlank + assertThat(headChecksum).isEqualTo(expectedChecksum) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObject_wrongChecksum(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val expectedChecksum = "wrongChecksum" + val checksumAlgorithm = ChecksumAlgorithm.SHA1 + val bucketName = givenBucket(testInfo) + + assertThatThrownBy { + s3Client.putObject({ + it.checksum(expectedChecksum, checksumAlgorithm) + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }, + RequestBody.fromFile(uploadFile) + ) + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 400") + .hasMessageContaining("Value for x-amz-checksum-sha1 header is invalid.") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObject_wrongEncryptionKey(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + assertThatThrownBy { + s3Client.putObject({ + it.ssekmsKeyId(TEST_WRONG_KEY_ID) + it.serverSideEncryption(ServerSideEncryption.AWS_KMS) + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }, + RequestBody.fromFile(uploadFile) + ) + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 400") + .hasMessageContaining("Invalid keyId 'key-ID-WRONGWRONGWRONG'") + } + + /** + * Safe characters: + * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObject_safeCharacters(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + val key = "someKey${charsSafeKey()}" + + val eTag = s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ).eTag() + + s3Client.headObject { + it.bucket(bucketName) + it.key(key) + }.also { + assertThat(it.eTag()).isEqualTo(eTag) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + }.use { + assertThat(eTag).isEqualTo(it.response().eTag()) + } + } + + /** + * Characters needing special handling: + * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObject_specialHandlingCharacters(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + val key = "someKey${charsSpecialKey()}" + + val eTag = s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ).eTag() + + s3Client.headObject( + HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build() + ).also { + assertThat(it.eTag()).isEqualTo(eTag) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + }.use { + assertThat(eTag).isEqualTo(it.response().eTag()) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutGetDeleteObject_twoBuckets(testInfo: TestInfo) { + val bucket1 = givenBucket() + val bucket2 = givenBucket() + givenObject(bucket1, UPLOAD_FILE_NAME) + givenObject(bucket2, UPLOAD_FILE_NAME) + getObject(bucket1, UPLOAD_FILE_NAME) + + deleteObject(bucket1, UPLOAD_FILE_NAME) + assertThatThrownBy { + getObject(bucket1, UPLOAD_FILE_NAME) + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 404") + + getObject(bucket2, UPLOAD_FILE_NAME) + .use { + assertThat(getObject(bucket2, UPLOAD_FILE_NAME).response().eTag()).isEqualTo(it.response().eTag()) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutGetHeadObject_storeHeaders(testInfo: TestInfo) { + val bucket = givenBucket() + val uploadFile = File(UPLOAD_FILE_NAME) + val contentDisposition = ContentDisposition.formData() + .name("file") + .filename("sampleFile.txt") + .build() + .toString() + val expires = Instant.now() + val encoding = "SomeEncoding" + val contentLanguage = "SomeLanguage" + val cacheControl = "SomeCacheControl" + + s3Client.putObject({ + it.bucket(bucket) + it.key(UPLOAD_FILE_NAME) + it.contentDisposition(contentDisposition) + it.contentEncoding(encoding) + it.expires(expires) + it.contentLanguage(contentLanguage) + it.cacheControl(cacheControl) + }, + RequestBody.fromFile(uploadFile) + ) + + getObject(bucket, UPLOAD_FILE_NAME).also { + assertThat(it.response().contentDisposition()).isEqualTo(contentDisposition) + assertThat(it.response().contentEncoding()).isEqualTo(encoding) + // time in second precision, see + // https://www.rfc-editor.org/rfc/rfc7234#section-5.3 + // https://www.rfc-editor.org/rfc/rfc7231#section-7.1.1.1 + assertThat(it.response().expires()).isEqualTo(expires.truncatedTo(ChronoUnit.SECONDS)) + assertThat(it.response().contentLanguage()).isEqualTo(contentLanguage) + assertThat(it.response().cacheControl()).isEqualTo(cacheControl) + } + + + s3Client.headObject { + it.bucket(bucket) + it.key(UPLOAD_FILE_NAME) + }.also { + assertThat(it.contentDisposition()).isEqualTo(contentDisposition) + assertThat(it.contentEncoding()).isEqualTo(encoding) + assertThat(it.expires()).isEqualTo(expires.truncatedTo(ChronoUnit.SECONDS)) + assertThat(it.contentLanguage()).isEqualTo(contentLanguage) + assertThat(it.cacheControl()).isEqualTo(cacheControl) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_successWithMatchingEtag(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val matchingEtag = FileInputStream(uploadFile).let { + "\"${DigestUtil.hexDigest(it)}\"" + } + + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + val eTag = putObjectResponse.eTag().also { + assertThat(it).isEqualTo(matchingEtag) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifMatch(matchingEtag) + }.use { + assertThat(it.response().eTag()).isEqualTo(eTag) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_successWithSameLength(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val matchingEtag = FileInputStream(uploadFile).let { + "\"${DigestUtil.hexDigest(it)}\"" + } + + val (bucketName, _) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifMatch(matchingEtag) + }.use { + assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_successWithMatchingWildcardEtag(testInfo: TestInfo) { + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + val eTag = putObjectResponse.eTag() + val matchingEtag = "\"*\"" + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifMatch(matchingEtag) + }.use { + assertThat(it.response().eTag()).isEqualTo(eTag) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testHeadObject_successWithNonMatchEtag(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val expectedEtag = FileInputStream(uploadFile).let { + "\"${DigestUtil.hexDigest(it)}\"" + } + + val nonMatchingEtag = "\"$randomName\"" + + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + val eTag = putObjectResponse.eTag().also { + assertThat(it).isEqualTo(expectedEtag) + } + + s3Client.headObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifNoneMatch(nonMatchingEtag) + }.also { + assertThat(it.eTag()).isEqualTo(eTag) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testHeadObject_failureWithNonMatchWildcardEtag(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val expectedEtag = FileInputStream(uploadFile).let { + "\"${DigestUtil.hexDigest(it)}\"" + } + + val nonMatchingEtag = "\"*\"" + + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + putObjectResponse.eTag().also { + assertThat(it).isEqualTo(expectedEtag) + } + + assertThatThrownBy { + s3Client.headObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifNoneMatch(nonMatchingEtag) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 304") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testHeadObject_failureWithMatchEtag(testInfo: TestInfo) { + val expectedEtag = FileInputStream(File(UPLOAD_FILE_NAME)).let { + "\"${DigestUtil.hexDigest(it)}\"" + } + + val nonMatchingEtag = "\"$randomName\"" + + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + putObjectResponse.eTag().also { + assertThat(it).isEqualTo(expectedEtag) + } + + assertThatThrownBy { + s3Client.headObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifMatch(nonMatchingEtag) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 412") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_successWithMatchingIfModified(testInfo: TestInfo) { + val now = Instant.now().minusSeconds(60) + val (bucketName, _) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifModifiedSince(now) + }.use { + assertThat(it.response().eTag()).isNotNull() + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_failureWithNonMatchingIfModified(testInfo: TestInfo) { + val (bucketName, _) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + TimeUnit.SECONDS.sleep(10L) + val now = Instant.now() + + assertThatThrownBy { + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifModifiedSince(now) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 304") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_successWithMatchingIfUnmodified(testInfo: TestInfo) { + val (bucketName, _) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + val now = Instant.now().plusSeconds(60) + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifUnmodifiedSince(now) + }.use { + assertThat(it.response().eTag()).isNotNull() + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_failureWithNonMatchingIfUnmodified(testInfo: TestInfo) { + val now = Instant.now().minusSeconds(60) + val (bucketName, _) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + + assertThatThrownBy { + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifUnmodifiedSince(now) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 412") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_rangeDownloads(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) + val eTag = putObjectResponse.eTag() + val smallRequestStartBytes = 1L + val smallRequestEndBytes = 2L + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ifMatch(eTag) + it.range("bytes=$smallRequestStartBytes-$smallRequestEndBytes") + }.also { + assertThat(it.response().contentLength()).isEqualTo(smallRequestEndBytes) + assertThat(it.response().contentRange()) + .isEqualTo("bytes $smallRequestStartBytes-$smallRequestEndBytes/${uploadFile.length()}") + } + + val largeRequestStartBytes = 0L + val largeRequestEndBytes = 1000L + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.range("bytes=$largeRequestStartBytes-$largeRequestEndBytes") + }.use { + assertThat(it.response().contentLength()).isEqualTo(min(uploadFile.length(), largeRequestEndBytes + 1)) + assertThat(it.response().contentRange()) + .isEqualTo( + "bytes $largeRequestStartBytes-${min(uploadFile.length() - 1, largeRequestEndBytes)}/${uploadFile.length()}" + ) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_rangeDownloads_finalBytes_prefixOffset(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val key = givenObjectV2WithRandomBytes(bucketName) + val startBytes = 4500L + val totalBytes = _5MB.toInt() + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + it.range("bytes=$startBytes-") + }.use { + assertThat(it.response().contentLength()).isEqualTo(totalBytes - startBytes) + assertThat(it.response().contentRange()).isEqualTo("bytes $startBytes-${totalBytes-1}/$totalBytes") + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObject_rangeDownloads_finalBytes_suffixOffset(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val key = givenObjectV2WithRandomBytes(bucketName) + val endBytes = 500L + val totalBytes = _5MB.toInt() + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + it.range("bytes=-$endBytes") + }.use { + assertThat(it.response().contentLength()).isEqualTo(endBytes) + assertThat(it.response().contentRange()).isEqualTo("bytes ${totalBytes-endBytes}-${totalBytes-1}/$totalBytes") + } + } + + /** + * Tests if Object can be uploaded with KMS and Metadata can be retrieved. + */ + @Test + @S3VerifiedFailure(year = 2023, + reason = "No KMS configuration for AWS test account") + fun testPutObject_withEncryption(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + + val sseCustomerAlgorithm = "someCustomerAlgorithm" + val sseCustomerKey = "someCustomerKey" + val sseCustomerKeyMD5 = "someCustomerKeyMD5" + val ssekmsEncryptionContext = "someEncryptionContext" + s3Client.putObject({ + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.ssekmsKeyId(TEST_ENC_KEY_ID) + it.sseCustomerAlgorithm(sseCustomerAlgorithm) + it.sseCustomerKey(sseCustomerKey) + it.sseCustomerKeyMD5(sseCustomerKeyMD5) + it.ssekmsEncryptionContext(ssekmsEncryptionContext) + it.serverSideEncryption(ServerSideEncryption.AWS_KMS) + }, + RequestBody.fromFile(uploadFile) + ).also { + assertThat(it.ssekmsKeyId()).isEqualTo(TEST_ENC_KEY_ID) + assertThat(it.sseCustomerAlgorithm()).isEqualTo(sseCustomerAlgorithm) + assertThat(it.sseCustomerKeyMD5()).isEqualTo(sseCustomerKeyMD5) + assertThat(it.serverSideEncryption()).isEqualTo(ServerSideEncryption.AWS_KMS) + } + + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }.use { + assertThat(it.response().ssekmsKeyId()).isEqualTo(TEST_ENC_KEY_ID) + assertThat(it.response().sseCustomerAlgorithm()).isEqualTo(sseCustomerAlgorithm) + assertThat(it.response().sseCustomerKeyMD5()).isEqualTo(sseCustomerKeyMD5) + assertThat(it.response().serverSideEncryption()).isEqualTo(ServerSideEncryption.AWS_KMS) + } + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest(name = ParameterizedTest.INDEX_PLACEHOLDER + " uploadWithSigning={0}, uploadChunked={1}") + @CsvSource(value = ["true, true", "true, false", "false, true", "false, false"]) + fun testPutGetObject_signingAndChunkedEncoding(uploadWithSigning: Boolean, uploadChunked: Boolean, testInfo: TestInfo) { + val key = UPLOAD_FILE_NAME + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + val s3Client = this@GetPutDeleteObjectIT.createS3Client(chunkedEncodingEnabled = uploadChunked) + + s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.headObject { + it.bucket(bucketName) + it.key(key) + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + }.use { + assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) + } + } + + private fun givenObjectV2WithRandomBytes(bucketName: String): String { + val key = randomName + s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromBytes(random5MBytes()) + ) + return key + } + + companion object { + private const val NON_EXISTING_KEY = "NoSuchKey.json" + private const val NO_SUCH_KEY = "The specified key does not exist." + private const val NO_SUCH_BUCKET = "The specified bucket does not exist" + } +} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV1IT.kt deleted file mode 100644 index 624690425..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV1IT.kt +++ /dev/null @@ -1,477 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import com.adobe.testing.s3mock.util.DigestUtil.hexDigest -import com.amazonaws.HttpMethod -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.Headers -import com.amazonaws.services.s3.model.AmazonS3Exception -import com.amazonaws.services.s3.model.DeleteObjectsRequest -import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion -import com.amazonaws.services.s3.model.DeleteObjectsResult -import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest -import com.amazonaws.services.s3.model.GetObjectMetadataRequest -import com.amazonaws.services.s3.model.GetObjectRequest -import com.amazonaws.services.s3.model.ObjectMetadata -import com.amazonaws.services.s3.model.PutObjectRequest -import com.amazonaws.services.s3.model.ResponseHeaderOverrides -import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams -import com.amazonaws.services.s3.transfer.TransferManager -import org.apache.http.client.methods.HttpGet -import org.apache.http.impl.client.CloseableHttpClient -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.assertj.core.configuration.Configuration -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.CsvSource -import java.io.ByteArrayInputStream -import java.io.File -import java.io.FileInputStream -import java.io.InputStream -import java.util.UUID -import java.util.stream.Collectors -import kotlin.math.min - -/** - * Test the application using the AmazonS3 SDK V1. - */ -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class GetPutDeleteObjectV1IT : S3TestBase() { - - private val httpClient: CloseableHttpClient = createHttpClient() - private val s3Client: AmazonS3 = createS3ClientV1() - private val transferManagerV1: TransferManager = createTransferManagerV1() - - @Test - @S3VerifiedSuccess(year = 2024) - fun putObjectWhereKeyContainsPathFragments(testInfo: TestInfo) { - val (bucketName, _) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val objectExist = s3Client.doesObjectExist(bucketName, UPLOAD_FILE_NAME) - assertThat(objectExist).isTrue - } - - /** - * Stores a file in a previously created bucket. Downloads the file again and compares checksums - */ - @ParameterizedTest(name = ParameterizedTest.INDEX_PLACEHOLDER + " uploadWithSigning={0}, uploadChunked={1}") - @CsvSource(value = ["true, true", "true, false", "false, true", "false, false"]) - @S3VerifiedSuccess(year = 2024) - fun shouldUploadAndDownloadObject(uploadWithSigning: Boolean, uploadChunked: Boolean, - testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val uploadClient = defaultTestAmazonS3ClientBuilder() - .withPayloadSigningEnabled(uploadWithSigning) - .withChunkedEncodingDisabled(uploadChunked) - .build() - uploadClient.putObject(PutObjectRequest(bucketName, uploadFile.name, uploadFile)) - s3Client.getObject(bucketName, uploadFile.name).also { - assertThat(it.objectMetadata.contentLength).isEqualTo(uploadFile.length()) - verifyObjectContent(uploadFile, it) - } - } - - /** - * Uses weird, but valid characters in the key used to store an object. - * - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldTolerateWeirdCharactersInObjectKey(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val weirdStuff = "$&_ .,':\u0001" // use only characters that are safe or need special handling - val key = weirdStuff + uploadFile.name + weirdStuff - s3Client.putObject(PutObjectRequest(bucketName, key, uploadFile)) - - s3Client.getObject(bucketName, key).also { - verifyObjectContent(uploadFile, it) - } - } - - /** - * Stores a file in a previously created bucket. Downloads the file again and compares checksums - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldUploadAndDownloadStream(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val resourceId = UUID.randomUUID().toString() - val contentEncoding = "gzip" - val resource = byteArrayOf(1, 2, 3, 4, 5) - val inputStream = ByteArrayInputStream(resource) - val objectMetadata = ObjectMetadata().apply { - this.contentLength = resource.size.toLong() - this.contentEncoding = contentEncoding - } - val putObjectRequest = PutObjectRequest(bucketName, resourceId, inputStream, objectMetadata) - transferManagerV1.upload(putObjectRequest).also { - it.waitForUploadResult() - } - s3Client.getObject(bucketName, resourceId).use { - assertThat(it.objectMetadata.contentEncoding).isEqualTo(contentEncoding) - val uploadDigest = hexDigest(ByteArrayInputStream(resource)) - val downloadedDigest = hexDigest(it.objectContent) - assertThat(uploadDigest).isEqualTo(downloadedDigest) - } - } - - /** - * Tests if Object can be uploaded with KMS and Metadata can be retrieved. - */ - @Test - @S3VerifiedFailure(year = 2024, - reason = "No KMS configuration for AWS test account") - fun shouldUploadWithEncryption(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val objectKey = UPLOAD_FILE_NAME - val metadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - } - val putObjectRequest = PutObjectRequest(bucketName, objectKey, uploadFile) - .withMetadata(metadata) - .apply { - this.sseAwsKeyManagementParams = SSEAwsKeyManagementParams(TEST_ENC_KEY_ID) - } - s3Client.putObject(putObjectRequest) - val getObjectMetadataRequest = GetObjectMetadataRequest(bucketName, objectKey) - s3Client.getObjectMetadata(getObjectMetadataRequest).also { - assertThat(it.contentLength).isEqualTo(uploadFile.length()) - assertThat(it.userMetadata).isEqualTo(metadata.userMetadata) - assertThat(it.sseAwsKmsKeyId).isEqualTo(TEST_ENC_KEY_ID) - } - } - - /** - * Tests if Object can be uploaded with wrong KMS Key. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldNotUploadWithWrongEncryptionKey(testInfo: TestInfo) { - Configuration().apply { - this.setMaxStackTraceElementsDisplayed(10000) - this.apply() - } - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - assertThatThrownBy { s3Client - .putObject( - PutObjectRequest(bucketName, UPLOAD_FILE_NAME, uploadFile) - .apply { - this.sseAwsKeyManagementParams = SSEAwsKeyManagementParams(TEST_WRONG_KEY_ID) - } - ) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 400; Error Code: KMS.NotFoundException") - } - - /** - * Tests if Object can be uploaded with wrong KMS Key. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldNotUploadStreamingWithWrongEncryptionKey(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val bytes = UPLOAD_FILE_NAME.toByteArray() - val stream: InputStream = ByteArrayInputStream(bytes) - val objectKey = UUID.randomUUID().toString() - assertThatThrownBy { s3Client - .putObject( - PutObjectRequest(bucketName, objectKey, stream, - ObjectMetadata().apply { - this.contentLength = bytes.size.toLong() - } - ).apply { - this.sseAwsKeyManagementParams = SSEAwsKeyManagementParams(TEST_WRONG_KEY_ID) - } - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 400; Error Code: KMS.NotFoundException") - } - - /** - * Tests if the Metadata of an existing file can be retrieved. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldGetObjectMetadata(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val nonExistingFileName = randomName - val uploadFile = File(UPLOAD_FILE_NAME) - val objectMetadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - this.contentEncoding = "gzip" - } - val putObjectResult = s3Client.putObject( - PutObjectRequest(bucketName, UPLOAD_FILE_NAME, uploadFile).apply { - this.withMetadata(objectMetadata) - } - ) - s3Client.getObjectMetadata(bucketName, UPLOAD_FILE_NAME).also { - assertThat(it.contentEncoding).isEqualTo("gzip") - assertThat(it.eTag).isEqualTo(putObjectResult.eTag) - assertThat(it.userMetadata).isEqualTo(objectMetadata.userMetadata) - } - assertThatThrownBy { - s3Client.getObjectMetadata( - bucketName, - nonExistingFileName - ) - } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 404") - } - - /** - * Tests if an object can be deleted. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldDeleteObject(testInfo: TestInfo) { - val (bucketName, _) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - s3Client.deleteObject(bucketName, UPLOAD_FILE_NAME) - assertThatThrownBy { s3Client.getObjectMetadata(bucketName, UPLOAD_FILE_NAME) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 404") - } - - /** - * Tests if multiple objects can be deleted. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldBatchDeleteObjects(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile1 = File(UPLOAD_FILE_NAME) - val uploadFile2 = File(UPLOAD_FILE_NAME) - val uploadFile3 = File(UPLOAD_FILE_NAME) - val file1 = "1_$UPLOAD_FILE_NAME" - val file2 = "2_$UPLOAD_FILE_NAME" - val file3 = "3_$UPLOAD_FILE_NAME" - s3Client.putObject(PutObjectRequest(bucketName, file1, uploadFile1)) - s3Client.putObject(PutObjectRequest(bucketName, file2, uploadFile2)) - s3Client.putObject(PutObjectRequest(bucketName, file3, uploadFile3)) - val delObjRes = s3Client.deleteObjects( - DeleteObjectsRequest(bucketName).apply { - this.keys = ArrayList().apply { - this.add(KeyVersion(file1)) - this.add(KeyVersion(file2)) - this.add(KeyVersion(file3)) - } - } - ) - assertThat(delObjRes.deletedObjects.size).isEqualTo(3) - assertThat( - delObjRes.deletedObjects.stream() - .map { obj: DeleteObjectsResult.DeletedObject -> obj.key } - .collect(Collectors.toList())) - .contains(file1, file2, file3) - assertThatThrownBy { s3Client.getObjectMetadata(bucketName, UPLOAD_FILE_NAME) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 404") - } - - /** - * Tests if Error is thrown when DeleteObjectsRequest contains nonExisting key. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldThrowOnBatchDeleteObjectsWrongKey(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile1 = File(UPLOAD_FILE_NAME) - val file1 = "1_$UPLOAD_FILE_NAME" - val nonExistingFile = "4_" + UUID.randomUUID() - s3Client.putObject(PutObjectRequest(bucketName, file1, uploadFile1)) - val multiObjectDeleteRequest = DeleteObjectsRequest(bucketName).apply { - this.keys = ArrayList().apply { - this.add(KeyVersion(file1)) - this.add(KeyVersion(nonExistingFile)) - } - } - val delObjRes = s3Client.deleteObjects(multiObjectDeleteRequest) - assertThat(delObjRes.deletedObjects.size).isEqualTo(2) - assertThat( - delObjRes.deletedObjects.stream() - .map { obj: DeleteObjectsResult.DeletedObject -> obj.key } - .collect(Collectors.toList())) - .contains(file1, nonExistingFile) - } - - /** - * Tests if an object can be uploaded asynchronously. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldUploadInParallel(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - transferManagerV1.upload(PutObjectRequest(bucketName, UPLOAD_FILE_NAME, uploadFile)).also { upload -> - upload.waitForUploadResult().also { - assertThat(it.key).isEqualTo(UPLOAD_FILE_NAME) - } - } - s3Client.getObject(bucketName, UPLOAD_FILE_NAME).also { - assertThat(it.key).isEqualTo(UPLOAD_FILE_NAME) - } - } - - /** - * Verify that range-downloads work. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun checkRangeDownloads(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val upload = transferManagerV1.upload(PutObjectRequest(bucketName, UPLOAD_FILE_NAME, uploadFile)) - upload.waitForUploadResult() - - val smallRequestStartBytes = 1L - val smallRequestEndBytes = 2L - val downloadFile1 = File.createTempFile(UUID.randomUUID().toString(), null) - transferManagerV1.download( - GetObjectRequest(bucketName, UPLOAD_FILE_NAME) - .withRange(smallRequestStartBytes, smallRequestEndBytes), downloadFile1 - ).also { download -> - download.waitForCompletion() - assertThat(downloadFile1.length()).isEqualTo(smallRequestEndBytes) - assertThat(download.objectMetadata.instanceLength).isEqualTo(uploadFile.length()) - assertThat(download.objectMetadata.contentLength).isEqualTo(smallRequestEndBytes) - } - - val largeRequestStartBytes = 0L - val largeRequestEndBytes = 1000L - val downloadFile2 = File.createTempFile(UUID.randomUUID().toString(), null) - transferManagerV1 - .download( - GetObjectRequest(bucketName, UPLOAD_FILE_NAME).withRange(largeRequestStartBytes, largeRequestEndBytes), - downloadFile2 - ).also { download -> - download.waitForCompletion() - assertThat(downloadFile2.length()).isEqualTo(min(uploadFile.length(), largeRequestEndBytes + 1)) - assertThat(download.objectMetadata.instanceLength).isEqualTo(uploadFile.length()) - assertThat(download.objectMetadata.contentLength).isEqualTo(min(uploadFile.length(), largeRequestEndBytes + 1)) - assertThat(download.objectMetadata.contentRange) - .containsExactlyElementsOf(listOf(largeRequestStartBytes, min(uploadFile.length() - 1, largeRequestEndBytes))) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_successWithMatchingEtag(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val uploadFileIs: InputStream = FileInputStream(uploadFile) - val expectedEtag = hexDigest(uploadFileIs) - assertThat(putObjectResult.eTag).isEqualTo(expectedEtag) - - s3Client.getObject(GetObjectRequest(bucketName, UPLOAD_FILE_NAME) - .withMatchingETagConstraint("\"${putObjectResult.eTag}\"")).also { - //v1 SDK does not return ETag on GetObject. Can only check if response is returned here. - assertThat(it.objectContent).isNotNull - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_failureWithMatchingEtag(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val uploadFileIs: InputStream = FileInputStream(uploadFile) - val expectedEtag = hexDigest(uploadFileIs) - assertThat(putObjectResult.eTag).isEqualTo(expectedEtag) - - val nonMatchingEtag = "\"$randomName\"" - s3Client.getObject(GetObjectRequest(bucketName, UPLOAD_FILE_NAME) - .withMatchingETagConstraint(nonMatchingEtag)).also { - //v1 SDK does not return a 412 error on a non-matching GetObject. Check if response is null. - assertThat(it).isNull() - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_successWithNonMatchingEtag(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val uploadFileIs: InputStream = FileInputStream(uploadFile) - val expectedEtag = hexDigest(uploadFileIs) - assertThat(putObjectResult.eTag).isEqualTo(expectedEtag) - - val nonMatchingEtag = "\"$randomName\"" - s3Client.getObject(GetObjectRequest(bucketName, UPLOAD_FILE_NAME) - .withNonmatchingETagConstraint(nonMatchingEtag)).also { - //v1 SDK does not return ETag on GetObject. Can only check if response is returned here. - assertThat(it.objectContent).isNotNull - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_failureWithNonMatchingEtag(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val uploadFileIs: InputStream = FileInputStream(uploadFile) - val expectedEtag = hexDigest(uploadFileIs) - assertThat(putObjectResult.eTag).isEqualTo(expectedEtag) - - s3Client.getObject( - GetObjectRequest(bucketName, UPLOAD_FILE_NAME) - .withNonmatchingETagConstraint("\"${putObjectResult.eTag}\"") - ).also { - //v1 SDK does not return a 412 error on a non-matching GetObject. Check if response is null. - assertThat(it).isNull() - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun generatePresignedUrlWithResponseHeaderOverrides(testInfo: TestInfo) { - val (bucketName, _) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val presignedUrlRequest = GeneratePresignedUrlRequest(bucketName, UPLOAD_FILE_NAME).apply { - this.withResponseHeaders( - ResponseHeaderOverrides().apply { - this.cacheControl = "cacheControl" - this.contentDisposition = "contentDisposition" - this.contentEncoding = "contentEncoding" - this.contentLanguage = "contentLanguage" - this.contentType = "my/contentType" - this.expires = "expires" - } - ) - this.method = HttpMethod.GET - } - val resourceUrl = s3Client.generatePresignedUrl(presignedUrlRequest) - httpClient.use { - val getObject = HttpGet(resourceUrl.toString()) - it.execute(getObject).also { response -> - assertThat(response.getFirstHeader(Headers.CACHE_CONTROL).value).isEqualTo("cacheControl") - assertThat(response.getFirstHeader(Headers.CONTENT_DISPOSITION).value).isEqualTo("contentDisposition") - assertThat(response.getFirstHeader(Headers.CONTENT_ENCODING).value).isEqualTo("contentEncoding") - assertThat(response.getFirstHeader(Headers.CONTENT_LANGUAGE).value).isEqualTo("contentLanguage") - assertThat(response.getFirstHeader(Headers.CONTENT_TYPE).value).isEqualTo("my/contentType") - assertThat(response.getFirstHeader(Headers.EXPIRES).value).isEqualTo("expires") - } - } - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV2IT.kt deleted file mode 100644 index 2d12a8d29..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV2IT.kt +++ /dev/null @@ -1,974 +0,0 @@ -/* - * Copyright 2017-2024 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.adobe.testing.s3mock.its - -import com.adobe.testing.s3mock.util.DigestUtil -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource -import org.springframework.http.ContentDisposition -import software.amazon.awssdk.core.async.AsyncRequestBody -import software.amazon.awssdk.core.checksums.Algorithm -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.S3AsyncClient -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm -import software.amazon.awssdk.services.s3.model.ChecksumMode -import software.amazon.awssdk.services.s3.model.GetObjectAttributesRequest -import software.amazon.awssdk.services.s3.model.GetObjectRequest -import software.amazon.awssdk.services.s3.model.HeadObjectRequest -import software.amazon.awssdk.services.s3.model.ObjectAttributes -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.S3Exception -import software.amazon.awssdk.services.s3.model.ServerSideEncryption -import software.amazon.awssdk.services.s3.model.StorageClass -import software.amazon.awssdk.transfer.s3.S3TransferManager -import java.io.File -import java.io.FileInputStream -import java.time.Instant -import java.time.temporal.ChronoUnit -import kotlin.math.min - -internal class GetPutDeleteObjectV2IT : S3TestBase() { - - private val s3ClientV2: S3Client = createS3ClientV2() - private val s3ClientV2Http: S3Client = createS3ClientV2(serviceEndpointHttp) - private val s3AsyncClientV2: S3AsyncClient = createS3AsyncClientV2() - private val s3AsyncClientV2Http: S3AsyncClient = createS3AsyncClientV2(serviceEndpointHttp) - private val s3CrtAsyncClientV2: S3AsyncClient = createS3CrtAsyncClientV2() - private val s3CrtAsyncClientV2Http: S3AsyncClient = createS3CrtAsyncClientV2(serviceEndpointHttp) - private val autoS3CrtAsyncClientV2: S3AsyncClient = createAutoS3CrtAsyncClientV2() - private val autoS3CrtAsyncClientV2Http: S3AsyncClient = createAutoS3CrtAsyncClientV2(serviceEndpointHttp) - private val transferManagerV2: S3TransferManager = createTransferManagerV2() - - /** - * Test safe characters in object keys - * - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - */ - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["charsSafe", "charsSpecial", "charsToAvoid"]) - fun testPutHeadGetObject_keyNames_safe(key: String, testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build(), - RequestBody.fromFile(uploadFile) - ) - - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build() - ) - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build() - ).use { - assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) - } - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["storageClasses"]) - fun testPutObject_storageClass(storageClass: StorageClass, testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - val key = UPLOAD_FILE_NAME - - val eTag = s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(key) - .storageClass(storageClass) - .build(), - RequestBody.fromFile(uploadFile) - ).eTag() - - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build() - ).also { - assertThat(it.eTag()).isEqualTo(eTag) - } - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build() - ).use { - assertThat(it.response().eTag()).isEqualTo(eTag) - if (storageClass == StorageClass.STANDARD) { - //storageClass STANDARD is never returned from S3 APIs... - assertThat(it.response().storageClass()).isNull() - } else { - assertThat(it.response().storageClass()).isEqualTo(storageClass) - } - } - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["testFileNames"]) - fun testPutObject_etagCreation_sync(testFileName: String, testInfo: TestInfo) { - testEtagCreation(testFileName, s3ClientV2, testInfo) - testEtagCreation(testFileName, s3ClientV2Http, testInfo) - } - - private fun GetPutDeleteObjectV2IT.testEtagCreation( - testFileName: String, - s3Client: S3Client, - testInfo: TestInfo - ) { - val uploadFile = File(testFileName) - val expectedEtag = FileInputStream(uploadFile).let { - "\"${DigestUtil.hexDigest(it)}\"" - } - val bucketName = givenBucketV2(testInfo) - s3Client.putObject( - PutObjectRequest.builder() - .bucket(bucketName).key(testFileName) - .build(), - RequestBody.fromFile(uploadFile) - ).eTag().also { - assertThat(it).isNotBlank - assertThat(it).isEqualTo(expectedEtag) - } - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["testFileNames"]) - fun testPutObject_etagCreation_async(testFileName: String) { - testEtagCreation(testFileName, s3AsyncClientV2) - testEtagCreation(testFileName, s3AsyncClientV2Http) - testEtagCreation(testFileName, s3CrtAsyncClientV2) - testEtagCreation(testFileName, s3CrtAsyncClientV2Http) - testEtagCreation(testFileName, autoS3CrtAsyncClientV2) - testEtagCreation(testFileName, autoS3CrtAsyncClientV2Http) - } - - private fun GetPutDeleteObjectV2IT.testEtagCreation( - testFileName: String, - s3Client: S3AsyncClient - ) { - val uploadFile = File(testFileName) - val expectedEtag = FileInputStream(uploadFile).let { - "\"${DigestUtil.hexDigest(it)}\"" - } - val bucketName = givenBucketV2(randomName) - s3Client.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(testFileName) - .build(), - AsyncRequestBody.fromFile(uploadFile) - ).join().eTag().also { - assertThat(it).isNotBlank - assertThat(it).isEqualTo(expectedEtag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObject_getObjectAttributes(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), Algorithm.SHA1) - val bucketName = givenBucketV2(testInfo) - - val eTag = s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName).key(UPLOAD_FILE_NAME) - .checksumAlgorithm(ChecksumAlgorithm.SHA1) - .build(), - RequestBody.fromFile(uploadFile) - ).eTag() - - s3ClientV2.getObjectAttributes( - GetObjectAttributesRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .objectAttributes( - ObjectAttributes.OBJECT_SIZE, - ObjectAttributes.STORAGE_CLASS, - ObjectAttributes.E_TAG, - ObjectAttributes.CHECKSUM) - .build() - ).also { - // - assertThat(it.eTag()).isEqualTo(eTag.trim('"')) - //default storageClass is STANDARD, which is never returned from APIs - assertThat(it.storageClass()).isEqualTo(StorageClass.STANDARD) - assertThat(it.objectSize()).isEqualTo(File(UPLOAD_FILE_NAME).length()) - assertThat(it.checksum().checksumSHA1()).isEqualTo(expectedChecksum) - } - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["checksumAlgorithms"]) - fun testPutObject_checksumAlgorithm_http(checksumAlgorithm: ChecksumAlgorithm) { - if(checksumAlgorithm != ChecksumAlgorithm.SHA256) { - //TODO: find out why the SHA256 checksum sent by the Java SDKv2 is wrong and this test is failing... - testChecksumAlgorithm(SAMPLE_FILE, checksumAlgorithm, s3ClientV2Http) - testChecksumAlgorithm(SAMPLE_FILE_LARGE, checksumAlgorithm, s3ClientV2Http) - testChecksumAlgorithm(TEST_IMAGE, checksumAlgorithm, s3ClientV2Http) - testChecksumAlgorithm(TEST_IMAGE_LARGE, checksumAlgorithm, s3ClientV2Http) - } - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["checksumAlgorithms"]) - fun testPutObject_checksumAlgorithm_https(checksumAlgorithm: ChecksumAlgorithm) { - testChecksumAlgorithm(SAMPLE_FILE, checksumAlgorithm, s3ClientV2) - testChecksumAlgorithm(SAMPLE_FILE_LARGE, checksumAlgorithm, s3ClientV2) - testChecksumAlgorithm(TEST_IMAGE, checksumAlgorithm, s3ClientV2) - testChecksumAlgorithm(TEST_IMAGE_LARGE, checksumAlgorithm, s3ClientV2) - } - - private fun GetPutDeleteObjectV2IT.testChecksumAlgorithm( - testFileName: String, - checksumAlgorithm: ChecksumAlgorithm, - s3Client: S3Client, - ) { - val uploadFile = File(testFileName) - val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) - val bucketName = givenBucketV2(randomName) - - s3Client.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(testFileName) - .checksumAlgorithm(checksumAlgorithm) - .build(), - RequestBody.fromFile(uploadFile) - ).also { - val putChecksum = it.checksum(checksumAlgorithm) - assertThat(putChecksum).isNotBlank - assertThat(putChecksum).isEqualTo(expectedChecksum) - } - - s3Client.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(testFileName) - .checksumMode(ChecksumMode.ENABLED) - .build() - ).use { - val getChecksum = it.response().checksum(checksumAlgorithm) - assertThat(getChecksum).isNotBlank - assertThat(getChecksum).isEqualTo(expectedChecksum) - } - - s3Client.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(testFileName) - .checksumMode(ChecksumMode.ENABLED) - .build() - ).also { - val headChecksum = it.checksum(checksumAlgorithm) - assertThat(headChecksum).isNotBlank - assertThat(headChecksum).isEqualTo(expectedChecksum) - } - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["checksumAlgorithms"]) - fun testPutObject_checksumAlgorithm_async_http(checksumAlgorithm: ChecksumAlgorithm) { - testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, s3AsyncClientV2Http) - testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, s3AsyncClientV2Http) - testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, s3AsyncClientV2Http) - testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, s3AsyncClientV2Http) - - testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, s3CrtAsyncClientV2Http) - testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, s3CrtAsyncClientV2Http) - testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, s3CrtAsyncClientV2Http) - testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, s3CrtAsyncClientV2Http) - - testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, autoS3CrtAsyncClientV2Http) - testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, autoS3CrtAsyncClientV2Http) - testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, autoS3CrtAsyncClientV2Http) - testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, autoS3CrtAsyncClientV2Http) - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["checksumAlgorithms"]) - fun testPutObject_checksumAlgorithm_async_https(checksumAlgorithm: ChecksumAlgorithm) { - testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, s3AsyncClientV2) - testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, s3AsyncClientV2) - testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, s3AsyncClientV2) - testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, s3AsyncClientV2) - - testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, s3CrtAsyncClientV2) - testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, s3CrtAsyncClientV2) - testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, s3CrtAsyncClientV2) - testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, s3CrtAsyncClientV2) - - testChecksumAlgorithm_async(SAMPLE_FILE, checksumAlgorithm, autoS3CrtAsyncClientV2) - testChecksumAlgorithm_async(SAMPLE_FILE_LARGE, checksumAlgorithm, autoS3CrtAsyncClientV2) - testChecksumAlgorithm_async(TEST_IMAGE, checksumAlgorithm, autoS3CrtAsyncClientV2) - testChecksumAlgorithm_async(TEST_IMAGE_LARGE, checksumAlgorithm, autoS3CrtAsyncClientV2) - } - - private fun GetPutDeleteObjectV2IT.testChecksumAlgorithm_async( - testFileName: String, - checksumAlgorithm: ChecksumAlgorithm, - s3Client: S3AsyncClient, - ) { - val uploadFile = File(testFileName) - val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) - val bucketName = givenBucketV2(randomName) - - s3Client.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(testFileName) - .checksumAlgorithm(checksumAlgorithm) - .build(), - AsyncRequestBody.fromFile(uploadFile) - ).join().also { - val putChecksum = it.checksum(checksumAlgorithm) - assertThat(putChecksum).isNotBlank - assertThat(putChecksum).isEqualTo(expectedChecksum) - } - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(testFileName) - .checksumMode(ChecksumMode.ENABLED) - .build() - ).use { - val getChecksum = it.response().checksum(checksumAlgorithm) - assertThat(getChecksum).isNotBlank - assertThat(getChecksum).isEqualTo(expectedChecksum) - } - - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(testFileName) - .checksumMode(ChecksumMode.ENABLED) - .build() - ).also { - val headChecksum = it.checksum(checksumAlgorithm) - assertThat(headChecksum).isNotBlank - assertThat(headChecksum).isEqualTo(expectedChecksum) - } - } - - private fun PutObjectRequest.Builder - .checksum(checksum: String, checksumAlgorithm: ChecksumAlgorithm): PutObjectRequest.Builder = - when (checksumAlgorithm) { - ChecksumAlgorithm.SHA1 -> this.checksumSHA1(checksum) - ChecksumAlgorithm.SHA256 -> this.checksumSHA256(checksum) - ChecksumAlgorithm.CRC32 -> this.checksumCRC32(checksum) - ChecksumAlgorithm.CRC32_C -> this.checksumCRC32C(checksum) - else -> error("Unknown checksum algorithm") - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["checksumAlgorithms"]) - fun testPutObject_checksum(checksumAlgorithm: ChecksumAlgorithm, testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) - val bucketName = givenBucketV2(testInfo) - - s3ClientV2.putObject( - PutObjectRequest - .builder() - .checksum(expectedChecksum, checksumAlgorithm) - .bucket(bucketName).key(UPLOAD_FILE_NAME) - .build(), - RequestBody.fromFile(uploadFile) - ).also { - val putChecksum = it.checksum(checksumAlgorithm)!! - assertThat(putChecksum).isNotBlank - assertThat(putChecksum).isEqualTo(expectedChecksum) - } - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .checksumMode(ChecksumMode.ENABLED) - .build() - ).use { - val getChecksum = it.response().checksum(checksumAlgorithm) - assertThat(getChecksum).isNotBlank - assertThat(getChecksum).isEqualTo(expectedChecksum) - } - - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .checksumMode(ChecksumMode.ENABLED) - .build() - ).also { - val headChecksum = it.checksum(checksumAlgorithm) - assertThat(headChecksum).isNotBlank - assertThat(headChecksum).isEqualTo(expectedChecksum) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObject_wrongChecksum(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val expectedChecksum = "wrongChecksum" - val checksumAlgorithm = ChecksumAlgorithm.SHA1 - val bucketName = givenBucketV2(testInfo) - - assertThatThrownBy { - s3ClientV2.putObject( - PutObjectRequest - .builder() - .checksum(expectedChecksum, checksumAlgorithm) - .bucket(bucketName).key(UPLOAD_FILE_NAME) - .build(), - RequestBody.fromFile(uploadFile) - ) - } - .isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 400") - .hasMessageContaining("Value for x-amz-checksum-sha1 header is invalid.") - } - - /** - * Safe characters: - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObject_safeCharacters(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - val key = "someKey${charsSafeKey()}" - - val eTag = s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build(), - RequestBody.fromFile(uploadFile) - ).eTag() - - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build() - ).also { - assertThat(it.eTag()).isEqualTo(eTag) - } - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build() - ).use { - assertThat(eTag).isEqualTo(it.response().eTag()) - } - } - - /** - * Characters needing special handling: - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObject_specialHandlingCharacters(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - val key = "someKey${charsSpecialKey()}" - - val eTag = s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build(), - RequestBody.fromFile(uploadFile) - ).eTag() - - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build() - ).also { - assertThat(it.eTag()).isEqualTo(eTag) - } - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build() - ).use { - assertThat(eTag).isEqualTo(it.response().eTag()) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutGetDeleteObject_twoBuckets(testInfo: TestInfo) { - val bucket1 = givenRandomBucketV2() - val bucket2 = givenRandomBucketV2() - givenObjectV2(bucket1, UPLOAD_FILE_NAME) - givenObjectV2(bucket2, UPLOAD_FILE_NAME) - getObjectV2(bucket1, UPLOAD_FILE_NAME) - - deleteObjectV2(bucket1, UPLOAD_FILE_NAME) - assertThatThrownBy { - getObjectV2(bucket1, UPLOAD_FILE_NAME) - }.isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 404") - - getObjectV2(bucket2, UPLOAD_FILE_NAME) - .use { - assertThat(getObjectV2(bucket2, UPLOAD_FILE_NAME).response().eTag()).isEqualTo(it.response().eTag()) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutGetHeadObject_storeHeaders(testInfo: TestInfo) { - val bucket = givenRandomBucketV2() - val uploadFile = File(UPLOAD_FILE_NAME) - val contentDisposition = ContentDisposition.formData() - .name("file") - .filename("sampleFile.txt") - .build() - .toString() - val expires = Instant.now() - val encoding = "SomeEncoding" - val contentLanguage = "SomeLanguage" - val cacheControl = "SomeCacheControl" - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucket) - .key(UPLOAD_FILE_NAME) - .contentDisposition(contentDisposition) - .contentEncoding(encoding) - .expires(expires) - .contentLanguage(contentLanguage) - .cacheControl(cacheControl) - .build(), - RequestBody.fromFile(uploadFile)) - - getObjectV2(bucket, UPLOAD_FILE_NAME).also { - assertThat(it.response().contentDisposition()).isEqualTo(contentDisposition) - assertThat(it.response().contentEncoding()).isEqualTo(encoding) - // time in second precision, see - // https://www.rfc-editor.org/rfc/rfc7234#section-5.3 - // https://www.rfc-editor.org/rfc/rfc7231#section-7.1.1.1 - assertThat(it.response().expires()).isEqualTo(expires.truncatedTo(ChronoUnit.SECONDS)) - assertThat(it.response().contentLanguage()).isEqualTo(contentLanguage) - assertThat(it.response().cacheControl()).isEqualTo(cacheControl) - } - - - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucket) - .key(UPLOAD_FILE_NAME) - .build() - ).also { - assertThat(it.contentDisposition()).isEqualTo(contentDisposition) - assertThat(it.contentEncoding()).isEqualTo(encoding) - assertThat(it.expires()).isEqualTo(expires.truncatedTo(ChronoUnit.SECONDS)) - assertThat(it.contentLanguage()).isEqualTo(contentLanguage) - assertThat(it.cacheControl()).isEqualTo(cacheControl) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_successWithMatchingEtag(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val matchingEtag = FileInputStream(uploadFile).let { - "\"${DigestUtil.hexDigest(it)}\"" - } - - val (bucketName, putObjectResponse) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - val eTag = putObjectResponse.eTag().also { - assertThat(it).isEqualTo(matchingEtag) - } - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifMatch(matchingEtag) - .build() - ).use { - assertThat(it.response().eTag()).isEqualTo(eTag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_successWithSameLength(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val matchingEtag = FileInputStream(uploadFile).let { - "\"${DigestUtil.hexDigest(it)}\"" - } - - val (bucketName, _) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifMatch(matchingEtag) - .build() - ).use { - assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_successWithMatchingWildcardEtag(testInfo: TestInfo) { - val (bucketName, putObjectResponse) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - val eTag = putObjectResponse.eTag() - val matchingEtag = "\"*\"" - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifMatch(matchingEtag) - .build() - ).use { - assertThat(it.response().eTag()).isEqualTo(eTag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testHeadObject_successWithNonMatchEtag(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val expectedEtag = FileInputStream(uploadFile).let { - "\"${DigestUtil.hexDigest(it)}\"" - } - - val nonMatchingEtag = "\"$randomName\"" - - val (bucketName, putObjectResponse) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - val eTag = putObjectResponse.eTag().also { - assertThat(it).isEqualTo(expectedEtag) - } - - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifNoneMatch(nonMatchingEtag) - .build() - ).also { - assertThat(it.eTag()).isEqualTo(eTag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testHeadObject_failureWithNonMatchWildcardEtag(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val expectedEtag = FileInputStream(uploadFile).let { - "\"${DigestUtil.hexDigest(it)}\"" - } - - val nonMatchingEtag = "\"*\"" - - val (bucketName, putObjectResponse) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - putObjectResponse.eTag().also { - assertThat(it).isEqualTo(expectedEtag) - } - - assertThatThrownBy { - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifNoneMatch(nonMatchingEtag) - .build() - ) - }.isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 304") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testHeadObject_failureWithMatchEtag(testInfo: TestInfo) { - val expectedEtag = FileInputStream(File(UPLOAD_FILE_NAME)).let { - "\"${DigestUtil.hexDigest(it)}\"" - } - - val nonMatchingEtag = "\"$randomName\"" - - val (bucketName, putObjectResponse) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - putObjectResponse.eTag().also { - assertThat(it).isEqualTo(expectedEtag) - } - - assertThatThrownBy { - s3ClientV2.headObject( - HeadObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifMatch(nonMatchingEtag) - .build() - ) - }.isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 412") - } - - @Test - @S3VerifiedTodo - fun testGetObject_successWithMatchingIfModified(testInfo: TestInfo) { - val now = Instant.now().minusSeconds(60) - val (bucketName, _) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifModifiedSince(now) - .build() - ).use { - assertThat(it.response().eTag()).isNotNull() - } - } - - @Test - @S3VerifiedTodo - fun testGetObject_failureWithNonMatchingIfModified(testInfo: TestInfo) { - val (bucketName, _) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - val now = Instant.now().plusSeconds(60) - - assertThatThrownBy { - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifModifiedSince(now) - .build() - ) - }.isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 412") - } - - @Test - @S3VerifiedTodo - fun testGetObject_successWithMatchingIfUnmodified(testInfo: TestInfo) { - val (bucketName, _) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - val now = Instant.now().plusSeconds(60) - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifUnmodifiedSince(now) - .build() - ).use { - assertThat(it.response().eTag()).isNotNull() - } - } - - - @Test - @S3VerifiedTodo - fun testGetObject_failureWithNonMatchingIfUnmodified(testInfo: TestInfo) { - val now = Instant.now().minusSeconds(60) - val (bucketName, _) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - - assertThatThrownBy { - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifUnmodifiedSince(now) - .build() - ) - }.isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 412") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_rangeDownloads(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val (bucketName, putObjectResponse) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) - val eTag = putObjectResponse.eTag() - val smallRequestStartBytes = 1L - val smallRequestEndBytes = 2L - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ifMatch(eTag) - .range("bytes=$smallRequestStartBytes-$smallRequestEndBytes") - .build() - ).also { - assertThat(it.response().contentLength()).isEqualTo(smallRequestEndBytes) - assertThat(it.response().contentRange()) - .isEqualTo("bytes $smallRequestStartBytes-$smallRequestEndBytes/${uploadFile.length()}") - } - - val largeRequestStartBytes = 0L - val largeRequestEndBytes = 1000L - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .range("bytes=$largeRequestStartBytes-$largeRequestEndBytes") - .build() - ).use { - assertThat(it.response().contentLength()).isEqualTo(min(uploadFile.length(), largeRequestEndBytes + 1)) - assertThat(it.response().contentRange()) - .isEqualTo( - "bytes $largeRequestStartBytes-${min(uploadFile.length() - 1, largeRequestEndBytes)}/${uploadFile.length()}" - ) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_rangeDownloads_finalBytes_prefixOffset(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val key = givenObjectV2WithRandomBytes(bucketName) - val startBytes = 4500L - val totalBytes = _5MB.toInt() - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .range("bytes=$startBytes-") - .build() - ).use { - assertThat(it.response().contentLength()).isEqualTo(totalBytes - startBytes) - assertThat(it.response().contentRange()).isEqualTo("bytes $startBytes-${totalBytes-1}/$totalBytes") - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObject_rangeDownloads_finalBytes_suffixOffset(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val key = givenObjectV2WithRandomBytes(bucketName) - val endBytes = 500L - val totalBytes = _5MB.toInt() - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(key) - .range("bytes=-$endBytes") - .build() - ).use { - assertThat(it.response().contentLength()).isEqualTo(endBytes) - assertThat(it.response().contentRange()).isEqualTo("bytes ${totalBytes-endBytes}-${totalBytes-1}/$totalBytes") - } - } - - /** - * Tests if Object can be uploaded with KMS and Metadata can be retrieved. - */ - @Test - @S3VerifiedFailure(year = 2023, - reason = "No KMS configuration for AWS test account") - fun testPutObject_withEncryption(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - - val sseCustomerAlgorithm = "someCustomerAlgorithm" - val sseCustomerKey = "someCustomerKey" - val sseCustomerKeyMD5 = "someCustomerKeyMD5" - val ssekmsEncryptionContext = "someEncryptionContext" - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .ssekmsKeyId(TEST_ENC_KEY_ID) - .sseCustomerAlgorithm(sseCustomerAlgorithm) - .sseCustomerKey(sseCustomerKey) - .sseCustomerKeyMD5(sseCustomerKeyMD5) - .ssekmsEncryptionContext(ssekmsEncryptionContext) - .serverSideEncryption(ServerSideEncryption.AWS_KMS) - .build(), - RequestBody.fromFile(uploadFile) - ).also { - assertThat(it.ssekmsKeyId()).isEqualTo(TEST_ENC_KEY_ID) - assertThat(it.sseCustomerAlgorithm()).isEqualTo(sseCustomerAlgorithm) - assertThat(it.sseCustomerKeyMD5()).isEqualTo(sseCustomerKeyMD5) - assertThat(it.serverSideEncryption()).isEqualTo(ServerSideEncryption.AWS_KMS) - } - - - s3ClientV2.getObject( - GetObjectRequest.builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ).use { - assertThat(it.response().ssekmsKeyId()).isEqualTo(TEST_ENC_KEY_ID) - assertThat(it.response().sseCustomerAlgorithm()).isEqualTo(sseCustomerAlgorithm) - assertThat(it.response().sseCustomerKeyMD5()).isEqualTo(sseCustomerKeyMD5) - assertThat(it.response().serverSideEncryption()).isEqualTo(ServerSideEncryption.AWS_KMS) - } - } - - private fun givenObjectV2WithRandomBytes(bucketName: String): String { - val key = randomName - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName) - .key(key) - .build(), - RequestBody.fromBytes(random5MBytes()) - ) - return key - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/KotlinSDKIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/KotlinSDKIT.kt index 398c4a7be..6ac80e490 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/KotlinSDKIT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/KotlinSDKIT.kt @@ -31,7 +31,8 @@ internal class KotlinSDKIT: S3TestBase() { private val s3Client = createS3ClientKotlin() @Test - @S3VerifiedTodo + @S3VerifiedFailure(year = 2025, + reason = "The unspecified location constraint is incompatible for the region specific endpoint this request was sent to.") fun createAndDeleteBucket(testInfo: TestInfo) : Unit = runBlocking { val bucketName = bucketName(testInfo) s3Client.createBucket(CreateBucketRequest { bucket = bucketName }) diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/LegalHoldIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/LegalHoldIT.kt new file mode 100644 index 000000000..dc87abcd7 --- /dev/null +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/LegalHoldIT.kt @@ -0,0 +1,125 @@ +/* + * Copyright 2017-2025 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adobe.testing.s3mock.its + +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus +import software.amazon.awssdk.services.s3.model.S3Exception +import java.io.File + +internal class LegalHoldIT : S3TestBase() { + + private val s3Client: S3Client = createS3Client() + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetLegalHoldNoBucketLockConfiguration(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) + + assertThatThrownBy { + s3Client.getObjectLegalHold { + it.bucket(bucketName) + it.key(sourceKey) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Bucket is missing Object Lock Configuration") + .hasMessageContaining("Service: S3, Status Code: 400") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetLegalHoldNoObjectLockConfiguration(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val sourceKey = UPLOAD_FILE_NAME + val bucketName = bucketName(testInfo) + s3Client.createBucket { + it.bucket(bucketName) + it.objectLockEnabledForBucket(true) + } + s3Client.putObject( + { + it.bucket(bucketName) + it.key(sourceKey) + }, + RequestBody.fromFile(uploadFile) + ) + + assertThatThrownBy { + s3Client.getObjectLegalHold { + it.bucket(bucketName) + it.key(sourceKey) + } + }.isInstanceOf(S3Exception::class.java) + .hasMessageContaining("The specified object does not have a ObjectLock configuration") + .hasMessageContaining("Service: S3, Status Code: 404") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutAndGetLegalHold(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val sourceKey = UPLOAD_FILE_NAME + val bucketName = bucketName(testInfo) + s3Client.createBucket { + it.bucket(bucketName) + it.objectLockEnabledForBucket(true) + } + s3Client.putObject( + { + it.bucket(bucketName) + it.key(sourceKey) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.putObjectLegalHold { + it.bucket(bucketName) + it.key(sourceKey) + it.legalHold { + it.status(ObjectLockLegalHoldStatus.ON) + } + } + + s3Client.getObjectLegalHold { + it.bucket(bucketName) + it.key(sourceKey) + }.also { + assertThat(it.legalHold().status()).isEqualTo(ObjectLockLegalHoldStatus.ON) + } + + s3Client.putObjectLegalHold { + it.bucket(bucketName) + it.key(sourceKey) + it.legalHold { + it.status(ObjectLockLegalHoldStatus.OFF) + } + } + + s3Client.getObjectLegalHold { + it.bucket(bucketName) + it.key(sourceKey) + }.also { + assertThat(it.legalHold().status()).isEqualTo(ObjectLockLegalHoldStatus.OFF) + } + } +} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/LegalHoldV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/LegalHoldV2IT.kt deleted file mode 100644 index b79bc91e4..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/LegalHoldV2IT.kt +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2017-2024 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.adobe.testing.s3mock.its - -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.CreateBucketRequest -import software.amazon.awssdk.services.s3.model.GetObjectLegalHoldRequest -import software.amazon.awssdk.services.s3.model.ObjectLockLegalHold -import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus -import software.amazon.awssdk.services.s3.model.PutObjectLegalHoldRequest -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.S3Exception -import java.io.File - -internal class LegalHoldV2IT : S3TestBase() { - - private val s3ClientV2: S3Client = createS3ClientV2() - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetLegalHoldNoBucketLockConfiguration(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV1(testInfo, sourceKey) - - assertThatThrownBy { - s3ClientV2.getObjectLegalHold( - GetObjectLegalHoldRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build() - ) - }.isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Bucket is missing Object Lock Configuration") - .hasMessageContaining("Service: S3, Status Code: 400") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetLegalHoldNoObjectLockConfiguration(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - val bucketName = bucketName(testInfo) - s3ClientV2.createBucket(CreateBucketRequest - .builder() - .bucket(bucketName) - .objectLockEnabledForBucket(true) - .build() - ) - s3ClientV2.putObject( - PutObjectRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build(), - RequestBody.fromFile(uploadFile) - ) - - assertThatThrownBy { - s3ClientV2.getObjectLegalHold( - GetObjectLegalHoldRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build() - ) - }.isInstanceOf(S3Exception::class.java) - .hasMessageContaining("The specified object does not have a ObjectLock configuration") - .hasMessageContaining("Service: S3, Status Code: 404") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutAndGetLegalHold(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val sourceKey = UPLOAD_FILE_NAME - val bucketName = bucketName(testInfo) - s3ClientV2.createBucket(CreateBucketRequest - .builder() - .bucket(bucketName) - .objectLockEnabledForBucket(true) - .build() - ) - s3ClientV2.putObject( - PutObjectRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build(), - RequestBody.fromFile(uploadFile) - ) - - s3ClientV2.putObjectLegalHold(PutObjectLegalHoldRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .legalHold(ObjectLockLegalHold - .builder() - .status(ObjectLockLegalHoldStatus.ON) - .build() - ) - .build() - ) - - s3ClientV2.getObjectLegalHold( - GetObjectLegalHoldRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build() - ).also { - assertThat(it.legalHold().status()).isEqualTo(ObjectLockLegalHoldStatus.ON) - } - - s3ClientV2.putObjectLegalHold(PutObjectLegalHoldRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .legalHold(ObjectLockLegalHold - .builder() - .status(ObjectLockLegalHoldStatus.OFF) - .build() - ) - .build() - ) - - s3ClientV2.getObjectLegalHold( - GetObjectLegalHoldRequest - .builder() - .bucket(bucketName) - .key(sourceKey) - .build() - ).also { - assertThat(it.legalHold().status()).isEqualTo(ObjectLockLegalHoldStatus.OFF) - } - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1IT.kt deleted file mode 100644 index 17ebe12db..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1IT.kt +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ListObjectsRequest -import com.amazonaws.services.s3.model.ListObjectsV2Request -import com.amazonaws.services.s3.model.PutObjectRequest -import com.amazonaws.services.s3.model.S3ObjectSummary -import com.amazonaws.services.s3.transfer.TransferManager -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource -import software.amazon.awssdk.utils.http.SdkHttpUtils -import java.io.File -import java.util.stream.Collectors - -/** - * Test the application using the AmazonS3 SDK V1. - */ -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class ListObjectV1IT : S3TestBase() { - - val s3Client: AmazonS3 = createS3ClientV1() - val transferManagerV1: TransferManager = createTransferManagerV1() - - class Param( - val prefix: String?, - val delimiter: String?, - val startAfter: String? - ) { - var expectedKeys: Array = arrayOfNulls(0) - var expectedPrefixes: Array = arrayOfNulls(0) - var expectedEncoding: String? = null - - fun keys(vararg expectedKeys: String?): Param { - this.expectedKeys = arrayOf(*expectedKeys) - return this - } - - fun encodedKeys(vararg expectedKeys: String): Param { - this.expectedKeys = arrayOf(*expectedKeys) - .map { toEncode: String? -> SdkHttpUtils.urlEncodeIgnoreSlashes(toEncode) } - .toTypedArray() - expectedEncoding = "url" - return this - } - - fun decodedKeys(): Array { - return arrayOf(*expectedKeys) - .map { toDecode: String? -> SdkHttpUtils.urlDecode(toDecode) } - .toTypedArray() - } - - fun prefixes(vararg expectedPrefixes: String?): Param { - this.expectedPrefixes = arrayOf(*expectedPrefixes) - return this - } - - override fun toString(): String { - return "prefix=$prefix, delimiter=$delimiter" - } - } - - /** - * Test the list V1 endpoint. - */ - @ParameterizedTest - @MethodSource("data") - @S3VerifiedSuccess(year = 2024) - fun listV1(parameters: Param, testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - // create all expected objects - for (key in ALL_OBJECTS) { - s3Client.putObject(bucketName, key, "Test") - } - val request = ListObjectsRequest( - bucketName, parameters.prefix, - parameters.startAfter, parameters.delimiter, null - ).apply { - this.encodingType = parameters.expectedEncoding - } - - val l = s3Client.listObjects(request) - LOG.info( - "list V1, prefix='{}', delimiter='{}': \n Objects: \n {}\n Prefixes: \n {}\n", // - parameters.prefix, // - parameters.delimiter, // - l.objectSummaries.stream().map { obj: S3ObjectSummary -> obj.key } - .collect(Collectors.joining("\n ")), // - java.lang.String.join("\n ", l.commonPrefixes) // - ) - var expectedPrefixes = parameters.expectedPrefixes - // AmazonS3#listObjects does not decode the prefixes, need to encode expected values - if (parameters.expectedEncoding != null) { - expectedPrefixes = arrayOf(*parameters.expectedPrefixes) - .map { toEncode: String? -> SdkHttpUtils.urlEncodeIgnoreSlashes(toEncode) } - .toTypedArray() - } - - assertThat(l.objectSummaries.stream() - .map { obj: S3ObjectSummary -> obj.key } - .collect(Collectors.toList()) - ).containsExactlyInAnyOrderElementsOf(listOf(*parameters.expectedKeys)) - assertThat(ArrayList(l.commonPrefixes)).containsExactlyInAnyOrderElementsOf(listOf(*expectedPrefixes)) - assertThat(l.encodingType).isEqualTo(parameters.expectedEncoding) - } - - /** - * Test the list V2 endpoint. - */ - @ParameterizedTest - @MethodSource("data") - @S3VerifiedSuccess(year = 2024) - fun listV2(parameters: Param, testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - // create all expected objects - for (key in ALL_OBJECTS) { - s3Client.putObject(bucketName, key, "Test") - } - val l = s3Client.listObjectsV2( - ListObjectsV2Request() - .withBucketName(bucketName) - .withDelimiter(parameters.delimiter) - .withPrefix(parameters.prefix) - .withStartAfter(parameters.startAfter) - .withEncodingType(parameters.expectedEncoding) - ) - LOG.info("list V2, prefix='{}', delimiter='{}', startAfter='{}': Objects: {} Prefixes: {}", - parameters.prefix, - parameters.delimiter, - parameters.startAfter, - l.objectSummaries.stream().map { s: S3ObjectSummary -> SdkHttpUtils.urlDecode(s.key) } - .collect(Collectors.joining("\n ")), - java.lang.String.join("\n ", l.commonPrefixes) - ) - - // listV2 automatically decodes the keys so the expected keys have to be decoded - val expectedDecodedKeys = parameters.decodedKeys() - assertThat(l.objectSummaries.stream() - .map { obj: S3ObjectSummary -> obj.key } - .collect(Collectors.toList()) - ).containsExactlyInAnyOrderElementsOf(listOf(*expectedDecodedKeys)) - // AmazonS3#listObjectsV2 returns decoded prefixes - assertThat(ArrayList(l.commonPrefixes)).containsExactlyInAnyOrderElementsOf(listOf(*parameters.expectedPrefixes)) - assertThat(l.encodingType).isEqualTo(parameters.expectedEncoding) - } - - /** - * Uses weird, but valid characters in the key used to store an object. Verifies - * that ListObject returns the correct object names. - * - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldListWithCorrectObjectNames(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val weirdStuff = charsSafe() - val prefix = "shouldListWithCorrectObjectNames/" - val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" - s3Client.putObject(PutObjectRequest(bucketName, key, uploadFile)) - - s3Client.listObjects(bucketName, prefix).also { listing -> - listing.objectSummaries.also { - assertThat(it).hasSize(1) - assertThat(it[0].key).isEqualTo(key) - } - } - } - - /** - * Same as [shouldListWithCorrectObjectNames] but for V2 API. - * - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldListV2WithCorrectObjectNames(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val weirdStuff = charsSafe() - val prefix = "shouldListWithCorrectObjectNames/" - val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" - s3Client.putObject(PutObjectRequest(bucketName, key, uploadFile)) - - // AWS client ListObjects V2 defaults to no encoding whereas V1 defaults to URL - val request = ListObjectsV2Request().apply { - this.bucketName = bucketName - this.prefix = prefix - this.encodingType = "url" // do use encoding! - } - val listing = s3Client.listObjectsV2(request) - - listing.objectSummaries.also { - assertThat(it).hasSize(1) - assertThat(it[0].key).isEqualTo(key) - } - } - - - /** - * Uses a key that cannot be represented in XML without encoding. Then lists - * the objects without encoding, expecting a parse exception and thus verifying - * that the encoding parameter is honored. - * - * - * This isn't the greatest way to test this functionality, however, there - * is currently no low-level testing infrastructure in place. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldHonorEncodingType(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val prefix = "shouldHonorEncodingType/" - val key = prefix + "\u0001" // key invalid in XML - s3Client.putObject(PutObjectRequest(bucketName, key, uploadFile)) - val lor = ListObjectsRequest(bucketName, prefix, null, null, null).apply { - this.encodingType = "url" - } - - s3Client.listObjects(lor).also { listing -> - listing.objectSummaries.also { - assertThat(it).hasSize(1) - assertThat(it[0].key).isEqualTo("shouldHonorEncodingType/%01") - } - } - } - - /** - * The same as [shouldHonorEncodingType] but for V2 API. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldHonorEncodingTypeV2(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val prefix = "shouldHonorEncodingType/" - val key = prefix + "\u0001" // key invalid in XML - s3Client.putObject(PutObjectRequest(bucketName, key, uploadFile)) - val request = ListObjectsV2Request().apply { - this.bucketName = bucketName - this.prefix = prefix - this.encodingType = "url" - } - - s3Client.listObjectsV2(request).also { listing -> - listing.objectSummaries.also { - assertThat(it).hasSize(1) - assertThat(it[0].key).isEqualTo("shouldHonorEncodingType/\u0001") - } - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldGetObjectListing(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - s3Client.putObject(PutObjectRequest(bucketName, UPLOAD_FILE_NAME, uploadFile)) - - s3Client.listObjects(bucketName, UPLOAD_FILE_NAME).also { - assertThat(it.objectSummaries).hasSizeGreaterThan(0) - assertThat(it.objectSummaries[0].key).isEqualTo(UPLOAD_FILE_NAME) - } - } - - /** - * Stores files in a previously created bucket. List files using ListObjectsV2Request - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldUploadAndListV2Objects(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - s3Client.putObject( - PutObjectRequest( - bucketName, - uploadFile.name, uploadFile - ) - ) - s3Client.putObject( - PutObjectRequest( - bucketName, - uploadFile.name + "copy1", uploadFile - ) - ) - s3Client.putObject( - PutObjectRequest( - bucketName, - uploadFile.name + "copy2", uploadFile - ) - ) - - val request = ListObjectsV2Request().withBucketName(bucketName).withMaxKeys(3) - val listResult = s3Client.listObjectsV2(request) - assertThat(listResult.keyCount).isEqualTo(3) - for (objectSummary in listResult.objectSummaries) { - assertThat(objectSummary.key).contains(uploadFile.name) - val s3Object = s3Client.getObject(bucketName, objectSummary.key) - verifyObjectContent(uploadFile, s3Object) - } - } - - companion object { - private val ALL_OBJECTS = arrayOf( - "3330/0", "33309/0", "a", - "b", "b/1", "b/1/1", "b/1/2", "b/2", - "c/1", "c/1/1", - "d:1", "d:1:1", - "eor.txt", "foo/eor.txt" - ) - - private fun param(prefix: String?, delimiter: String?, startAfter: String?): Param { - return Param(prefix, delimiter, startAfter) - } - - /** - * Parameter factory. - */ - @JvmStatic - fun data(): Iterable { - return listOf( // - param(null, null, null).keys(*ALL_OBJECTS), // - param("", null, null).keys(*ALL_OBJECTS), // - param(null, "", null).keys(*ALL_OBJECTS), // - param(null, "/", null).keys("a", "b", "d:1", "d:1:1", "eor.txt") - .prefixes("3330/", "foo/", "c/", "b/", "33309/"), - param("", "", null).keys(*ALL_OBJECTS), // - param("/", null, null), // - param("b", null, null).keys("b", "b/1", "b/1/1", "b/1/2", "b/2"), // - param("b/", null, null).keys("b/1", "b/1/1", "b/1/2", "b/2"), // - param("b", "", null).keys("b", "b/1", "b/1/1", "b/1/2", "b/2"), // - param("b", "/", null).keys("b").prefixes("b/"), // - param("b/", "/", null).keys("b/1", "b/2").prefixes("b/1/"), // - param("b/1", "/", null).keys("b/1").prefixes("b/1/"), // - param("b/1/", "/", null).keys("b/1/1", "b/1/2"), // - param("c", "/", null).prefixes("c/"), // - param("c/", "/", null).keys("c/1").prefixes("c/1/"), // - param("eor", "/", null).keys("eor.txt"), // - // start after existing key - param("b", null, "b/1/1").keys("b/1/2", "b/2"), // - // start after non-existing key - param("b", null, "b/0").keys("b/1", "b/1/1", "b/1/2", "b/2"), - param("3330/", null, null).keys("3330/0"), - param(null, null, null).encodedKeys(*ALL_OBJECTS), - param("b/1", "/", null).encodedKeys("b/1").prefixes("b/1/") - ) - } - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1MaxKeysIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1MaxKeysIT.kt deleted file mode 100644 index eb02f0f67..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1MaxKeysIT.kt +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ListObjectsRequest -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo - -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class ListObjectV1MaxKeysIT : S3TestBase() { - val s3Client: AmazonS3 = createS3ClientV1() - - @Test - @S3VerifiedSuccess(year = 2024) - fun returnsLimitedAmountOfObjectsBasedOnMaxKeys(testInfo: TestInfo) { - val bucketName = givenBucketWithTwoObjects(testInfo) - val request = ListObjectsRequest().withBucketName(bucketName).withMaxKeys(1) - - s3Client.listObjects(request).also { - assertThat(it.objectSummaries).hasSize(1) - assertThat(it.maxKeys).isEqualTo(1) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun returnsAllObjectsIfMaxKeysIsDefault(testInfo: TestInfo) { - val bucketName = givenBucketWithTwoObjects(testInfo) - val request = ListObjectsRequest().withBucketName(bucketName) - - s3Client.listObjects(request).also { - assertThat(it.objectSummaries).hasSize(2) - assertThat(it.maxKeys).isEqualTo(1000) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun returnsAllObjectsIfMaxKeysEqualToAmountOfObjects(testInfo: TestInfo) { - val bucketName = givenBucketWithTwoObjects(testInfo) - val request = ListObjectsRequest().withBucketName(bucketName).withMaxKeys(2) - - s3Client.listObjects(request).also { - assertThat(it.objectSummaries).hasSize(2) - assertThat(it.maxKeys).isEqualTo(2) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun returnsAllObjectsIfMaxKeysMoreThanAmountOfObjects(testInfo: TestInfo) { - val bucketName = givenBucketWithTwoObjects(testInfo) - val request = ListObjectsRequest().withBucketName(bucketName).withMaxKeys(3) - - s3Client.listObjects(request).also { - assertThat(it.objectSummaries).hasSize(2) - assertThat(it.maxKeys).isEqualTo(3) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun returnsEmptyListIfMaxKeysIsZero(testInfo: TestInfo) { - val bucketName = givenBucketWithTwoObjects(testInfo) - val request = ListObjectsRequest().withBucketName(bucketName).withMaxKeys(0) - - s3Client.listObjects(request).also { - assertThat(it.objectSummaries).hasSize(0) - assertThat(it.maxKeys).isEqualTo(0) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun returnsAllObjectsIfMaxKeysIsNegative(testInfo: TestInfo) { - val bucketName = givenBucketWithTwoObjects(testInfo) - val request = ListObjectsRequest().withBucketName(bucketName).withMaxKeys(-1) - s3Client.listObjects(request).also { - // Apparently, the Amazon SDK rejects negative max keys, and by default it's 1000 - assertThat(it.objectSummaries).hasSize(2) - assertThat(it.maxKeys).isEqualTo(1000) - } - } - - private fun givenBucketWithTwoObjects(testInfo: TestInfo): String { - val bucketName = givenBucketV1(testInfo) - s3Client.putObject(bucketName, "a", "") - s3Client.putObject(bucketName, "b", "") - return bucketName - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1PaginationIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1PaginationIT.kt deleted file mode 100644 index 58e9e35d5..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV1PaginationIT.kt +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ListObjectsRequest -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo - -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class ListObjectV1PaginationIT : S3TestBase() { - val s3Client: AmazonS3 = createS3ClientV1() - - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldTruncateAndReturnNextMarker(testInfo: TestInfo) { - val bucketName = givenBucketWithTwoObjects(testInfo) - val request = ListObjectsRequest().withBucketName(bucketName).withMaxKeys(1) - - val objectListing = s3Client.listObjects(request).also { - assertThat(it.objectSummaries).hasSize(1) - assertThat(it.maxKeys).isEqualTo(1) - assertThat(it.nextMarker).isEqualTo("a") - assertThat(it.isTruncated).isTrue - } - - val continueRequest = ListObjectsRequest().withBucketName(bucketName).withMarker(objectListing.nextMarker) - s3Client.listObjects(continueRequest).also { - assertThat(it.objectSummaries.size).isEqualTo(1) - assertThat(it.objectSummaries[0].key).isEqualTo("b") - } - } - - private fun givenBucketWithTwoObjects(testInfo: TestInfo): String { - val bucketName = givenBucketV1(testInfo) - s3Client.putObject(bucketName, "a", "") - s3Client.putObject(bucketName, "b", "") - return bucketName - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV2IT.kt deleted file mode 100644 index 68e825b41..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectV2IT.kt +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2017-2024 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.adobe.testing.s3mock.its - -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.groups.Tuple -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm -import software.amazon.awssdk.services.s3.model.EncodingType -import software.amazon.awssdk.services.s3.model.ListObjectsRequest -import software.amazon.awssdk.services.s3.model.ListObjectsV2Request -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.S3Object -import java.io.File - -internal class ListObjectV2IT : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObjectsListObjectsV2_checksumAlgorithm_sha256(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName).key("$UPLOAD_FILE_NAME-1") - .checksumAlgorithm(ChecksumAlgorithm.SHA256) - .build(), - RequestBody.fromFile(uploadFile) - ) - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName).key("$UPLOAD_FILE_NAME-2") - .checksumAlgorithm(ChecksumAlgorithm.SHA256) - .build(), - RequestBody.fromFile(uploadFile) - ) - - s3ClientV2.listObjectsV2( - ListObjectsV2Request.builder() - .bucket(bucketName) - .build() - ).also { - assertThat(it.contents()) - .hasSize(2) - .extracting(S3Object::checksumAlgorithm) - .containsOnly( - Tuple(arrayListOf(ChecksumAlgorithm.SHA256)), - Tuple(arrayListOf(ChecksumAlgorithm.SHA256)) - ) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObjectsListObjectsV1_checksumAlgorithm_sha256(testInfo: TestInfo) { - val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName).key("$UPLOAD_FILE_NAME-1") - .checksumAlgorithm(ChecksumAlgorithm.SHA256) - .build(), - RequestBody.fromFile(uploadFile) - ) - - s3ClientV2.putObject( - PutObjectRequest.builder() - .bucket(bucketName).key("$UPLOAD_FILE_NAME-2") - .checksumAlgorithm(ChecksumAlgorithm.SHA256) - .build(), - RequestBody.fromFile(uploadFile) - ) - - s3ClientV2.listObjects( - ListObjectsRequest.builder() - .bucket(bucketName) - .build() - ).also { - assertThat(it.contents()) - .hasSize(2) - .extracting(S3Object::checksumAlgorithm) - .containsOnly( - Tuple(arrayListOf(ChecksumAlgorithm.SHA256)), - Tuple(arrayListOf(ChecksumAlgorithm.SHA256)) - ) - } - } - - /** - * Test list with safe characters in keys. - * - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldListV2WithCorrectObjectNames(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val weirdStuff = charsSafe() - val prefix = "shouldListWithCorrectObjectNames/" - val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" - s3ClientV2.putObject(PutObjectRequest - .builder() - .bucket(bucketName) - .key(key) - .build(), - RequestBody.fromFile(uploadFile) - ) - - s3ClientV2.listObjectsV2( - ListObjectsV2Request - .builder() - .bucket(bucketName) - .prefix(prefix) - .encodingType(EncodingType.URL) - .build() - ).also { listing -> - listing.contents().also { - assertThat(it).hasSize(1) - assertThat(it[0].key()).isEqualTo(key) - } - } - } - -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectVersionsV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectVersionsIT.kt similarity index 82% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectVersionsV2IT.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectVersionsIT.kt index a1c5b5a76..85761777e 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectVersionsV2IT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectVersionsIT.kt @@ -25,22 +25,22 @@ import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.BucketVersioningStatus import java.io.File -internal class ListObjectVersionsV2IT : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() +internal class ListObjectVersionsIT : S3TestBase() { + private val s3Client: S3Client = createS3Client() @Test @S3VerifiedSuccess(year = 2025) fun listObjectVersions(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - s3ClientV2.putBucketVersioning { + val bucketName = givenBucket(testInfo) + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - val version1 = s3ClientV2.putObject( + val version1 = s3Client.putObject( { it.bucket(bucketName) it.key("$UPLOAD_FILE_NAME-1") @@ -48,7 +48,7 @@ internal class ListObjectVersionsV2IT : S3TestBase() { RequestBody.fromFile(uploadFile) ).versionId() - val version2 = s3ClientV2.putObject( + val version2 = s3Client.putObject( { it.bucket(bucketName) it.key("$UPLOAD_FILE_NAME-2") @@ -56,7 +56,7 @@ internal class ListObjectVersionsV2IT : S3TestBase() { RequestBody.fromFile(uploadFile) ).versionId() - s3ClientV2.listObjectVersions { + s3Client.listObjectVersions { it.bucket(bucketName) }.also { assertThat(it.versions()) @@ -67,12 +67,12 @@ internal class ListObjectVersionsV2IT : S3TestBase() { } @Test - @S3VerifiedTodo + @S3VerifiedSuccess(year = 2025) fun listObjectVersions_noVersioning(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) - s3ClientV2.putObject( + s3Client.putObject( { it.bucket(bucketName) it.key("$UPLOAD_FILE_NAME-1") @@ -80,7 +80,7 @@ internal class ListObjectVersionsV2IT : S3TestBase() { RequestBody.fromFile(uploadFile) ) - s3ClientV2.putObject( + s3Client.putObject( { it.bucket(bucketName) it.key("$UPLOAD_FILE_NAME-2") @@ -88,7 +88,7 @@ internal class ListObjectVersionsV2IT : S3TestBase() { RequestBody.fromFile(uploadFile) ) - s3ClientV2.listObjectVersions { + s3Client.listObjectVersions { it.bucket(bucketName) }.also { assertThat(it.versions()) @@ -102,15 +102,15 @@ internal class ListObjectVersionsV2IT : S3TestBase() { @S3VerifiedSuccess(year = 2025) fun listObjectVersions_deleteMarker(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) - s3ClientV2.putBucketVersioning { + val bucketName = givenBucket(testInfo) + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - val version1 = s3ClientV2.putObject( + val version1 = s3Client.putObject( { it.bucket(bucketName) it.key("$UPLOAD_FILE_NAME-1") @@ -118,7 +118,7 @@ internal class ListObjectVersionsV2IT : S3TestBase() { RequestBody.fromFile(uploadFile) ).versionId() - val version2 = s3ClientV2.putObject( + val version2 = s3Client.putObject( { it.bucket(bucketName) it.key("$UPLOAD_FILE_NAME-2") @@ -126,7 +126,7 @@ internal class ListObjectVersionsV2IT : S3TestBase() { RequestBody.fromFile(uploadFile) ).versionId() - val version3 = s3ClientV2.putObject( + val version3 = s3Client.putObject( { it.bucket(bucketName) it.key("$UPLOAD_FILE_NAME-3") @@ -134,12 +134,12 @@ internal class ListObjectVersionsV2IT : S3TestBase() { RequestBody.fromFile(uploadFile) ).versionId() - s3ClientV2.deleteObject { + s3Client.deleteObject { it.bucket(bucketName) it.key("$UPLOAD_FILE_NAME-3") } - s3ClientV2.listObjectVersions { + s3Client.listObjectVersions { it.bucket(bucketName) }.also { assertThat(it.versions()) diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectsIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectsIT.kt new file mode 100644 index 000000000..bc3b53fb2 --- /dev/null +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ListObjectsIT.kt @@ -0,0 +1,576 @@ +/* + * Copyright 2017-2025 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.adobe.testing.s3mock.its + +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.assertj.core.groups.Tuple +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm +import software.amazon.awssdk.services.s3.model.CommonPrefix +import software.amazon.awssdk.services.s3.model.EncodingType +import software.amazon.awssdk.services.s3.model.NoSuchBucketException +import software.amazon.awssdk.services.s3.model.S3Object +import software.amazon.awssdk.utils.http.SdkHttpUtils +import java.io.File +import java.util.stream.Collectors + +internal class ListObjectsIT : S3TestBase() { + private val s3Client: S3Client = createS3Client() + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObjectsListObjectsV2_checksumAlgorithm_sha256(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + s3Client.putObject( + { + it.bucket(bucketName) + it.key("$UPLOAD_FILE_NAME-1") + it.checksumAlgorithm(ChecksumAlgorithm.SHA256) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.putObject( + { + it.bucket(bucketName).key("$UPLOAD_FILE_NAME-2") + it.checksumAlgorithm(ChecksumAlgorithm.SHA256) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.listObjectsV2 { + it.bucket(bucketName) + }.also { + assertThat(it.contents()) + .hasSize(2) + .extracting(S3Object::checksumAlgorithm) + .containsOnly( + Tuple(arrayListOf(ChecksumAlgorithm.SHA256)), + Tuple(arrayListOf(ChecksumAlgorithm.SHA256)) + ) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObjectsListObjectsV1_checksumAlgorithm_sha256(testInfo: TestInfo) { + val uploadFile = File(UPLOAD_FILE_NAME) + val bucketName = givenBucket(testInfo) + + s3Client.putObject( + { + it.bucket(bucketName).key("$UPLOAD_FILE_NAME-1") + it.checksumAlgorithm(ChecksumAlgorithm.SHA256) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.putObject( + { + it.bucket(bucketName).key("$UPLOAD_FILE_NAME-2") + it.checksumAlgorithm(ChecksumAlgorithm.SHA256) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.listObjects { + it.bucket(bucketName) + }.also { + assertThat(it.contents()) + .hasSize(2) + .extracting(S3Object::checksumAlgorithm) + .containsOnly( + Tuple(arrayListOf(ChecksumAlgorithm.SHA256)), + Tuple(arrayListOf(ChecksumAlgorithm.SHA256)) + ) + } + } + + /** + * Test list with safe characters in keys. + * + * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun shouldListV1WithCorrectObjectNames(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val weirdStuff = charsSafe() + val prefix = "shouldListWithCorrectObjectNames/" + val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" + s3Client.putObject( + { + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.listObjects { + it.bucket(bucketName) + it.prefix(prefix) + it.encodingType(EncodingType.URL) + }.also { listing -> + listing.contents().also { + assertThat(it).hasSize(1) + assertThat(it[0].key()).isEqualTo(key) + } + } + } + + /** + * Test list with safe characters in keys. + * + * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun shouldListV2WithCorrectObjectNames(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val weirdStuff = charsSafe() + val prefix = "shouldListWithCorrectObjectNames/" + val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" + s3Client.putObject( + { + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.listObjectsV2 { + it.bucket(bucketName) + it.prefix(prefix) + it.encodingType(EncodingType.URL) + }.also { listing -> + listing.contents().also { + assertThat(it).hasSize(1) + assertThat(it[0].key()).isEqualTo(key) + } + } + } + + /** + * Uses a key that cannot be represented in XML without encoding. Then lists + * the objects without encoding, expecting a parse exception and thus verifying + * that the encoding parameter is honored. + * + * + * This isn't the greatest way to test this functionality, however, there + * is currently no low-level testing infrastructure in place. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun shouldHonorEncodingTypeV1(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val weirdStuff = "\u0001" // key invalid in XML + val prefix = "shouldHonorEncodingTypeV1/" + val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" + s3Client.putObject( + { + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.listObjects { + it.bucket(bucketName) + it.prefix(prefix) + it.encodingType(EncodingType.URL) + }.also { listing -> + listing.contents().also { + assertThat(it).hasSize(1) + assertThat(it[0].key()).isEqualTo(key) + } + } + } + + /** + * Uses a key that cannot be represented in XML without encoding. Then lists + * the objects without encoding, expecting a parse exception and thus verifying + * that the encoding parameter is honored. + * + * + * This isn't the greatest way to test this functionality, however, there + * is currently no low-level testing infrastructure in place. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun shouldHonorEncodingTypeV2(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val weirdStuff = "\u0001" // key invalid in XML + val prefix = "shouldHonorEncodingTypeV2/" + val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" + s3Client.putObject( + { + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + + s3Client.listObjectsV2 { + it.bucket(bucketName) + it.prefix(prefix) + it.encodingType(EncodingType.URL) + }.also { listing -> + listing.contents().also { + assertThat(it).hasSize(1) + assertThat(it[0].key()).isEqualTo(key) + } + } + } + + @ParameterizedTest + @MethodSource("data") + @S3VerifiedSuccess(year = 2025) + fun listV1(parameters: Param, testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val weirdStuff = "\u0001" // key invalid in XML + val prefix = "shouldHonorEncodingTypeV2/" + val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" + + for(key in ALL_OBJECTS) { + s3Client.putObject( + { + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + } + + // listV2 automatically decodes the keys so the expected keys have to be decoded + val expectedDecodedKeys = parameters.decodedKeys() + + s3Client.listObjects { + it.bucket(bucketName) + it.prefix(parameters.prefix) + it.delimiter(parameters.delimiter) + it.marker(parameters.startAfter) + it.encodingType(parameters.expectedEncoding) + }.also { listing -> + LOG.info("list V1, prefix='{}', delimiter='{}', startAfter='{}': Objects: {} Prefixes: {}", + parameters.prefix, + parameters.delimiter, + parameters.startAfter, + listing.contents().stream().map { s: S3Object -> SdkHttpUtils.urlDecode(s.key()) } + .collect(Collectors.joining("\n ")), + java.lang.String.join("\n ", listing.commonPrefixes().map(CommonPrefix::prefix)) + ) + listing.commonPrefixes().also { + assertThat(it.stream().map { s: CommonPrefix -> SdkHttpUtils.urlDecode(s.prefix()) } + .collect(Collectors.toList())) + .containsExactlyInAnyOrder(*parameters.expectedPrefixes) + } + listing.contents().also { + assertThat(it.stream().map { s: S3Object -> SdkHttpUtils.urlDecode(s.key()) }.toList()).isEqualTo(listOf(*expectedDecodedKeys)) + } + if (parameters.expectedEncoding != null) { + assertThat(listing.encodingType().toString()).isEqualTo(parameters.expectedEncoding) + } else { + assertThat(listing.encodingType()).isNull() + } + } + } + + @ParameterizedTest + @MethodSource("data") + @S3VerifiedSuccess(year = 2025) + fun listV2(parameters: Param, testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val weirdStuff = "\u0001" // key invalid in XML + val prefix = "shouldHonorEncodingTypeV2/" + val key = "$prefix$weirdStuff${uploadFile.name}$weirdStuff" + + for(key in ALL_OBJECTS) { + s3Client.putObject( + { + it.bucket(bucketName) + it.key(key) + }, + RequestBody.fromFile(uploadFile) + ) + } + + // listV2 automatically decodes the keys so the expected keys have to be decoded + val expectedDecodedKeys = parameters.decodedKeys() + + s3Client.listObjectsV2 { + it.bucket(bucketName) + it.prefix(parameters.prefix) + it.delimiter(parameters.delimiter) + it.startAfter(parameters.startAfter) + it.encodingType(parameters.expectedEncoding) + }.also { listing -> + LOG.info("list V2, prefix='{}', delimiter='{}', startAfter='{}': Objects: {} Prefixes: {}", + parameters.prefix, + parameters.delimiter, + parameters.startAfter, + listing.contents().stream().map { s: S3Object -> SdkHttpUtils.urlDecode(s.key()) } + .collect(Collectors.joining("\n ")), + java.lang.String.join("\n ", listing.commonPrefixes().map(CommonPrefix::prefix)) + ) + listing.commonPrefixes().also { + assertThat(it.stream().map { s: CommonPrefix -> SdkHttpUtils.urlDecode(s.prefix()) } + .collect(Collectors.toList())) + .containsExactlyInAnyOrder(*parameters.expectedPrefixes) + } + listing.contents().also { + assertThat(it.stream().map { s: S3Object -> SdkHttpUtils.urlDecode(s.key()) }.toList()).isEqualTo(listOf(*expectedDecodedKeys)) + } + if (parameters.expectedEncoding != null) { + assertThat(listing.encodingType().toString()).isEqualTo(parameters.expectedEncoding) + } else { + assertThat(listing.encodingType()).isNull() + } + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun returnsLimitedAmountOfObjectsBasedOnMaxKeys(testInfo: TestInfo) { + val (bucketName, keys) = givenBucketAndObjects(testInfo, 30) + val maxKeys = 10 + val listedObjects = mutableListOf() + + val continuationToken1 = s3Client.listObjectsV2 { + it.bucket(bucketName) + it.maxKeys(maxKeys) + }.let { listing -> + assertThat(listing.contents().size).isEqualTo(maxKeys) + assertThat(listing.isTruncated).isTrue + assertThat(listing.maxKeys()).isEqualTo(maxKeys) + assertThat(listing.nextContinuationToken()).isNotNull + listedObjects.addAll(listing.contents().map(S3Object::key)) + listing.nextContinuationToken() + } + + val continuationToken2 = s3Client.listObjectsV2 { + it.bucket(bucketName) + it.maxKeys(maxKeys) + it.continuationToken(continuationToken1) + }.let { listing -> + assertThat(listing.contents().size).isEqualTo(maxKeys) + assertThat(listing.isTruncated).isTrue + assertThat(listing.maxKeys()).isEqualTo(maxKeys) + assertThat(listing.nextContinuationToken()).isNotNull + listedObjects.addAll(listing.contents().map(S3Object::key)) + listing.nextContinuationToken() + } + + s3Client.listObjectsV2 { + it.bucket(bucketName) + it.maxKeys(maxKeys) + it.continuationToken(continuationToken2) + }.also { listing -> + assertThat(listing.contents().size).isEqualTo(maxKeys) + assertThat(listing.isTruncated).isFalse + assertThat(listing.maxKeys()).isEqualTo(maxKeys) + assertThat(listing.nextContinuationToken()).isNull() + listedObjects.addAll(listing.contents().map(S3Object::key)) + } + + assertThat(listedObjects).hasSize(30) + assertThat(listedObjects).hasSameElementsAs(keys) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun returnsAllObjectsIfMaxKeysIsDefault(testInfo: TestInfo) { + val (bucketName, _) = givenBucketAndObjects(testInfo, 30) + s3Client.listObjectsV2 { + it.bucket(bucketName) + }.also { listing -> + assertThat(listing.contents().size).isEqualTo(30) + assertThat(listing.isTruncated).isFalse + assertThat(listing.maxKeys()).isEqualTo(1000) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun returnsAllObjectsIfMaxKeysEqualToAmountOfObjects(testInfo: TestInfo) { + val (bucketName, _) = givenBucketAndObjects(testInfo, 30) + s3Client.listObjectsV2 { + it.bucket(bucketName) + it.maxKeys(30) + }.also { listing -> + assertThat(listing.contents().size).isEqualTo(30) + assertThat(listing.isTruncated).isFalse + assertThat(listing.maxKeys()).isEqualTo(30) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun returnsAllObjectsIfMaxKeysMoreThanAmountOfObjects(testInfo: TestInfo) { + val (bucketName, _) = givenBucketAndObjects(testInfo, 30) + s3Client.listObjectsV2 { + it.bucket(bucketName) + it.maxKeys(400) + }.also { listing -> + assertThat(listing.contents().size).isEqualTo(30) + assertThat(listing.isTruncated).isFalse + assertThat(listing.maxKeys()).isEqualTo(400) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun returnsEmptyListIfMaxKeysIsZero(testInfo: TestInfo) { + val (bucketName, _) = givenBucketAndObjects(testInfo, 30) + s3Client.listObjects { + it.bucket(bucketName) + it.maxKeys(0) + }.also { listing -> + assertThat(listing.contents()).isEmpty() + assertThat(listing.isTruncated).isFalse + assertThat(listing.maxKeys()).isEqualTo(0) + assertThat(listing.nextMarker()).isNull() + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun returnsEmptyListIfMaxKeysIsZeroV2(testInfo: TestInfo) { + val (bucketName, _) = givenBucketAndObjects(testInfo, 30) + s3Client.listObjectsV2 { + it.bucket(bucketName) + it.maxKeys(0) + }.also { listing -> + assertThat(listing.contents()).isEmpty() + assertThat(listing.isTruncated).isFalse + assertThat(listing.maxKeys()).isEqualTo(0) + assertThat(listing.nextContinuationToken()).isNull() + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun listObjects_noSuchBucket() { + assertThatThrownBy { + s3Client.listObjects { + it.bucket(randomName) + it.prefix(UPLOAD_FILE_NAME) + } + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + companion object { + private const val NO_SUCH_BUCKET = "The specified bucket does not exist" + private val ALL_OBJECTS = arrayOf( + "3330/0", "33309/0", "a", + "b", "b/1", "b/1/1", "b/1/2", "b/2", + "c/1", "c/1/1", + "d:1", "d:1:1", + "eor.txt", "foo/eor.txt" + ) + + private fun param(prefix: String?, delimiter: String?, startAfter: String?): Param { + return Param(prefix, delimiter, startAfter) + } + + /** + * Parameter factory. + */ + @JvmStatic + fun data(): Iterable { + return listOf( // + param(null, null, null).keys(*ALL_OBJECTS), // + param("", null, null).keys(*ALL_OBJECTS), // + param(null, "", null).keys(*ALL_OBJECTS), // + param(null, "/", null).keys("a", "b", "d:1", "d:1:1", "eor.txt") + .prefixes("3330/", "foo/", "c/", "b/", "33309/"), + param("", "", null).keys(*ALL_OBJECTS), // + param("/", null, null), // + param("b", null, null).keys("b", "b/1", "b/1/1", "b/1/2", "b/2"), // + param("b/", null, null).keys("b/1", "b/1/1", "b/1/2", "b/2"), // + param("b", "", null).keys("b", "b/1", "b/1/1", "b/1/2", "b/2"), // + param("b", "/", null).keys("b").prefixes("b/"), // + param("b/", "/", null).keys("b/1", "b/2").prefixes("b/1/"), // + param("b/1", "/", null).keys("b/1").prefixes("b/1/"), // + param("b/1/", "/", null).keys("b/1/1", "b/1/2"), // + param("c", "/", null).prefixes("c/"), // + param("c/", "/", null).keys("c/1").prefixes("c/1/"), // + param("eor", "/", null).keys("eor.txt"), // + // start after existing key + param("b", null, "b/1/1").keys("b/1/2", "b/2"), // + // start after non-existing key + param("b", null, "b/0").keys("b/1", "b/1/1", "b/1/2", "b/2"), + param("3330/", null, null).keys("3330/0"), + param(null, null, null).encodedKeys(*ALL_OBJECTS), + param("b/1", "/", null).encodedKeys("b/1").prefixes("b/1/") + ) + } + + class Param( + val prefix: String?, + val delimiter: String?, + val startAfter: String? + ) { + var expectedKeys: Array = arrayOfNulls(0) + var expectedPrefixes: Array = arrayOfNulls(0) + var expectedEncoding: String? = null + + fun keys(vararg expectedKeys: String?): Param { + this.expectedKeys = arrayOf(*expectedKeys) + return this + } + + fun encodedKeys(vararg expectedKeys: String): Param { + this.expectedKeys = arrayOf(*expectedKeys) + .map { toEncode: String? -> SdkHttpUtils.urlEncodeIgnoreSlashes(toEncode) } + .toTypedArray() + expectedEncoding = "url" + return this + } + + fun decodedKeys(): Array { + return arrayOf(*expectedKeys) + .map { toDecode: String? -> SdkHttpUtils.urlDecode(toDecode) } + .toTypedArray() + } + + fun prefixes(vararg expectedPrefixes: String?): Param { + this.expectedPrefixes = arrayOf(*expectedPrefixes) + return this + } + + override fun toString(): String { + return "prefix=$prefix, delimiter=$delimiter" + } + } + } +} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartIT.kt new file mode 100644 index 000000000..a3bafcef2 --- /dev/null +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartIT.kt @@ -0,0 +1,1374 @@ +/* + * Copyright 2017-2025 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.adobe.testing.s3mock.its + +import com.adobe.testing.s3mock.S3Exception.PRECONDITION_FAILED +import com.adobe.testing.s3mock.util.DigestUtil +import com.adobe.testing.s3mock.util.DigestUtil.hexDigest +import org.apache.commons.codec.digest.DigestUtils +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.assertj.core.api.InstanceOfAssertFactories +import org.assertj.core.util.Files +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.springframework.web.util.UriUtils +import software.amazon.awssdk.awscore.exception.AwsErrorDetails +import software.amazon.awssdk.awscore.exception.AwsServiceException +import software.amazon.awssdk.core.async.AsyncRequestBody +import software.amazon.awssdk.core.checksums.Algorithm.CRC32 +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3AsyncClient +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm +import software.amazon.awssdk.services.s3.model.ChecksumMode +import software.amazon.awssdk.services.s3.model.CompletedPart +import software.amazon.awssdk.services.s3.model.ListPartsRequest +import software.amazon.awssdk.services.s3.model.NoSuchBucketException +import software.amazon.awssdk.services.s3.model.NoSuchKeyException +import software.amazon.awssdk.services.s3.model.S3Exception +import software.amazon.awssdk.services.s3.model.UploadPartRequest +import software.amazon.awssdk.transfer.s3.S3TransferManager +import software.amazon.awssdk.utils.http.SdkHttpUtils +import java.io.ByteArrayInputStream +import java.io.File +import java.io.FileInputStream +import java.nio.charset.StandardCharsets +import java.nio.file.Files.newOutputStream +import java.time.Instant +import java.util.UUID +import java.util.concurrent.CompletionException + + +internal class MultiPartIT : S3TestBase() { + private val s3Client: S3Client = createS3Client() + private val s3AsyncClient: S3AsyncClient = createS3AsyncClient() + private val s3CrtAsyncClient: S3AsyncClient = createS3CrtAsyncClient() + private val autoS3CrtAsyncClient: S3AsyncClient = createAutoS3CrtAsyncClient() + private val transferManager: S3TransferManager = createTransferManager() + + @Test + @S3VerifiedSuccess(year = 2025) + fun testMultipartUpload_asyncClient(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + s3CrtAsyncClient.putObject( + { + it.bucket(bucketName) + it.key(uploadFile.name) + it.checksumAlgorithm(ChecksumAlgorithm.CRC32) + }, + AsyncRequestBody.fromFile(uploadFile) + ).join().also { + assertThat(it.checksumCRC32()).isEqualTo(DigestUtil.checksumFor(uploadFile.toPath(), CRC32)) + } + + s3AsyncClient.waiter().waitUntilObjectExists { + it.bucket(bucketName) + it.key(uploadFile.name) + } + + val uploadDigest = hexDigest(uploadFile) + val downloadedDigest = s3Client.getObject { + it.bucket(bucketName) + it.key(uploadFile.name) + }.use { response -> + Files.newTemporaryFile().let { + response.transferTo(newOutputStream(it.toPath())) + assertThat(it).hasSize(uploadFile.length()) + assertThat(it).hasSameBinaryContentAs(uploadFile) + hexDigest(it) + } + } + assertThat(uploadDigest).isEqualTo(downloadedDigest) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testMultipartUpload_transferManager(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + transferManager + .uploadFile { + it.putObjectRequest { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + } + it.source(uploadFile) + }.completionFuture().join() + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }.use { + assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) + } + + val downloadFile = Files.newTemporaryFile() + transferManager.downloadFile { + it.getObjectRequest { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + } + it.destination(downloadFile) + }.also { download -> + download.completionFuture().join().response().also { + assertThat(it.contentLength()).isEqualTo(uploadFile.length()) + } + } + assertThat(downloadFile.length()).isEqualTo(uploadFile.length()) + assertThat(downloadFile).hasSameBinaryContentAs(uploadFile) + } + + /** + * Tests if user metadata can be passed by multipart upload. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testMultipartUpload_withUserMetadata(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val objectMetadata = mapOf(Pair("key", "value")) + val initiateMultipartUploadResult = s3Client + .createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.metadata(objectMetadata) + } + val uploadId = initiateMultipartUploadResult.uploadId() + val uploadPartResult = s3Client.uploadPart( + { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.partNumber(1) + it.contentLength(uploadFile.length()) + //it.lastPart(true) + }, + RequestBody.fromFile(uploadFile), + ) + + s3Client.completeMultipartUpload { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(initiateMultipartUploadResult.uploadId()) + it.multipartUpload { + it.parts({ + it.eTag(uploadPartResult.eTag()) + it.partNumber(1) + } + ) + } + } + + s3Client.getObject { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + }.use { + assertThat(it.response().metadata()).isEqualTo(objectMetadata) + } + } + + /** + * Tests if a multipart upload with the last part being smaller than 5MB works. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testMultipartUpload(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val objectMetadata = mapOf(Pair("key", "value")) + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.metadata(objectMetadata) + } + val uploadId = initiateMultipartUploadResult.uploadId() + // upload part 1, >5MB + val randomBytes = randomBytes() + val etag1 = uploadPart(bucketName, UPLOAD_FILE_NAME, uploadId, 1, randomBytes) + // upload part 2, <5MB + val etag2 = s3Client.uploadPart( + { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.partNumber(2) + it.contentLength(uploadFile.length()) + //it.lastPart(true) + }, + RequestBody.fromFile(uploadFile), + ).eTag() + + val completeMultipartUpload = s3Client.completeMultipartUpload { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(initiateMultipartUploadResult.uploadId()) + it.multipartUpload { + it.parts( + { + it.eTag(etag1) + it.partNumber(1) + }, + { + it.eTag(etag2) + it.partNumber(2) + } + ) + } + } + + val uploadFileBytes = readStreamIntoByteArray(uploadFile.inputStream()) + + (DigestUtils.md5(randomBytes) + DigestUtils.md5(uploadFileBytes)).also { + // verify special etag + assertThat(completeMultipartUpload.eTag()).isEqualTo("\"${DigestUtils.md5Hex(it)}-2\"") + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }.use { + // verify content size + assertThat(it.response().contentLength()).isEqualTo(randomBytes.size.toLong() + uploadFileBytes.size.toLong()) + // verify contents + assertThat(readStreamIntoByteArray(it.buffered())).isEqualTo(concatByteArrays(randomBytes, uploadFileBytes)) + assertThat(it.response().metadata()).isEqualTo(objectMetadata) + } + + assertThat(completeMultipartUpload.location()) + .isEqualTo("${serviceEndpoint}/$bucketName/${UriUtils.encode(UPLOAD_FILE_NAME, StandardCharsets.UTF_8)}") + } + + + /** + * Tests if a multipart upload with the last part being smaller than 5MB works. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testMultipartUpload_checksum(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(TEST_IMAGE_TIFF) + //construct uploadfile >5MB + val tempFile = Files.newTemporaryFile().also { + (readStreamIntoByteArray(uploadFile.inputStream()) + + readStreamIntoByteArray(uploadFile.inputStream()) + + readStreamIntoByteArray(uploadFile.inputStream())) + .inputStream() + .copyTo(it.outputStream()) + } + + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key(TEST_IMAGE_TIFF) + it.checksumAlgorithm(ChecksumAlgorithm.CRC32) + } + val uploadId = initiateMultipartUploadResult.uploadId() + // upload part 1, <5MB + val partResponse1 = s3Client.uploadPart( + { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.checksumAlgorithm(ChecksumAlgorithm.CRC32) + it.partNumber(1) + it.contentLength(tempFile.length()) + }, + //.lastPart(true) + RequestBody.fromFile(tempFile), + ) + val etag1 = partResponse1.eTag() + val checksum1 = partResponse1.checksumCRC32() + // upload part 2, <5MB + val partResponse2 = s3Client.uploadPart({ + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.checksumAlgorithm(ChecksumAlgorithm.CRC32) + it.partNumber(2) + it.contentLength(uploadFile.length()) + }, + //.lastPart(true) + RequestBody.fromFile(uploadFile), + ) + val etag2 = partResponse2.eTag() + val checksum2 = partResponse2.checksumCRC32() + val localChecksum1 = DigestUtil.checksumFor(tempFile.toPath(), CRC32) + assertThat(checksum1).isEqualTo(localChecksum1) + val localChecksum2 = DigestUtil.checksumFor(uploadFile.toPath(), CRC32) + assertThat(checksum2).isEqualTo(localChecksum2) + + val completeMultipartUpload = s3Client.completeMultipartUpload { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(initiateMultipartUploadResult.uploadId()) + it.multipartUpload { + it.parts( + { + it.eTag(etag1) + it.partNumber(1) + it.checksumCRC32(checksum1) + }, + { + it.eTag(etag2) + it.partNumber(2) + it.checksumCRC32(checksum2) + } + ) + } + } + + (DigestUtils.md5(tempFile.readBytes()) + DigestUtils.md5(readStreamIntoByteArray(uploadFile.inputStream()))).also { + // verify special etag + assertThat(completeMultipartUpload.eTag()).isEqualTo("\"${DigestUtils.md5Hex(it)}-2\"") + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(TEST_IMAGE_TIFF) + it.checksumMode(ChecksumMode.ENABLED) + }.use { + // verify content size + assertThat(it.response().contentLength()).isEqualTo(tempFile.length() + uploadFile.length()) + // verify contents + assertThat(readStreamIntoByteArray(it.buffered())).isEqualTo(tempFile.readBytes() + uploadFile.readBytes()) + assertThat(it.response().checksumCRC32()).isEqualTo("oGk6qg==-2") + } + + assertThat(completeMultipartUpload.location()) + .isEqualTo("${serviceEndpoint}/$bucketName/${UriUtils.encode(TEST_IMAGE_TIFF, StandardCharsets.UTF_8)}") + } + + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["checksumAlgorithms"]) + fun testUploadPart_checksumAlgorithm(checksumAlgorithm: ChecksumAlgorithm, testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.checksumAlgorithm(checksumAlgorithm) + } + val uploadId = initiateMultipartUploadResult.uploadId() + + s3Client.uploadPart({ + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.checksumAlgorithm(checksumAlgorithm) + it.partNumber(1) + it.contentLength(uploadFile.length()).build() + //.lastPart(true) + }, + RequestBody.fromFile(uploadFile), + ).also { + val actualChecksum = it.checksum(checksumAlgorithm) + assertThat(actualChecksum).isNotBlank + assertThat(actualChecksum).isEqualTo(expectedChecksum) + } + s3Client.abortMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.uploadId(uploadId) + } + } + + @S3VerifiedSuccess(year = 2025) + @ParameterizedTest + @MethodSource(value = ["checksumAlgorithms"]) + fun testMultipartUpload_checksum(checksumAlgorithm: ChecksumAlgorithm, testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.checksumAlgorithm(checksumAlgorithm) + } + val uploadId = initiateMultipartUploadResult.uploadId() + + s3Client.uploadPart({ + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.checksum(expectedChecksum, checksumAlgorithm) + it.partNumber(1) + it.contentLength(uploadFile.length()).build() + //.lastPart(true) + }, + RequestBody.fromFile(uploadFile), + ).also { + val actualChecksum = it.checksum(checksumAlgorithm) + assertThat(actualChecksum).isNotBlank + assertThat(actualChecksum).isEqualTo(expectedChecksum) + } + s3Client.abortMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.uploadId(uploadId).build() + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testMultipartUpload_wrongChecksum(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val expectedChecksum = "wrongChecksum" + val checksumAlgorithm = ChecksumAlgorithm.SHA1 + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + } + val uploadId = initiateMultipartUploadResult.uploadId() + + assertThatThrownBy { + s3Client.uploadPart({ + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.checksum(expectedChecksum, checksumAlgorithm) + it.partNumber(1) + it.contentLength(uploadFile.length()).build() + //it.lastPart(true) + }, + RequestBody.fromFile(uploadFile), + ) + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 400") + .hasMessageContaining("Value for x-amz-checksum-sha1 header is invalid.") + } + + private fun UploadPartRequest.Builder.checksum( + checksum: String, + checksumAlgorithm: ChecksumAlgorithm + ): UploadPartRequest.Builder = + when (checksumAlgorithm) { + ChecksumAlgorithm.SHA1 -> this.checksumSHA1(checksum) + ChecksumAlgorithm.SHA256 -> this.checksumSHA256(checksum) + ChecksumAlgorithm.CRC32 -> this.checksumCRC32(checksum) + ChecksumAlgorithm.CRC32_C -> this.checksumCRC32C(checksum) + else -> error("Unknown checksum algorithm") + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testInitiateMultipartAndRetrieveParts(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val objectMetadata = mapOf(Pair("key", "value")) + val hash = DigestUtils.md5Hex(FileInputStream(uploadFile)) + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.metadata(objectMetadata) + } + val uploadId = initiateMultipartUploadResult.uploadId() + val key = initiateMultipartUploadResult.key() + + s3Client.uploadPart( + { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(key) + it.uploadId(uploadId) + it.partNumber(1) + it.contentLength(uploadFile.length()) + //.lastPart(true) + }, + RequestBody.fromFile(uploadFile), + ) + + val partListing = s3Client.listParts { + it.bucket(bucketName) + it.key(key) + it.uploadId(uploadId) + }.also { + assertThat(it.parts()).hasSize(1) + } + + partListing.parts()[0].also { + assertThat(it.eTag()).isEqualTo("\"" + hash + "\"") + assertThat(it.partNumber()).isEqualTo(1) + assertThat(it.lastModified()).isExactlyInstanceOf(Instant::class.java) + } + } + + /** + * Tests if not yet completed / aborted multipart uploads are listed. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testListMultipartUploads_ok(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.uploads() + ).isEmpty() + val uploadId = s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }.uploadId() + + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.also { listing -> + assertThat(listing.uploads()).isNotEmpty + assertThat(listing.bucket()).isEqualTo(bucketName) + assertThat(listing.uploads()).hasSize(1) + + listing.uploads()[0] + .also { + assertThat(it.uploadId()).isEqualTo(uploadId) + assertThat(it.key()).isEqualTo(UPLOAD_FILE_NAME) + } + } + } + + /** + * Tests if empty parts list of not yet completed multipart upload is returned. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testListMultipartUploads_empty(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.uploads() + ).isEmpty() + val uploadId = s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }.uploadId() + + s3Client.listParts { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.uploadId(uploadId) + }.also { + assertThat(it.parts()).isEmpty() + assertThat(it.bucket()).isEqualTo(bucketName) + assertThat(it.uploadId()).isEqualTo(uploadId) + assertThat(SdkHttpUtils.urlDecode(it.key())).isEqualTo(UPLOAD_FILE_NAME) + } + } + + /** + * Tests that an exception is thrown when listing parts if the upload id is unknown. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testListMultipartUploads_throwOnUnknownId(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + + assertThatThrownBy { + s3Client.listParts { + it.bucket(bucketName) + it.key("NON_EXISTENT_KEY") + it.uploadId("NON_EXISTENT_UPLOAD_ID") + } + } + .isInstanceOf(AwsServiceException::class.java) + .hasMessageContaining("Service: S3, Status Code: 404") + } + + /** + * Tests if not yet completed / aborted multipart uploads are listed with prefix filtering. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testListMultipartUploads_withPrefix(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key("key1") + } + s3Client.createMultipartUpload { + it.bucket(bucketName) + it.key("key2") + } + + val listing = s3Client.listMultipartUploads { + it.bucket(bucketName) + it.prefix("key2") + } + assertThat(listing.uploads()).hasSize(1) + assertThat(listing.uploads()[0].key()).isEqualTo("key2") + } + + /** + * Tests if multipart uploads are stored and can be retrieved by bucket. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testListMultipartUploads_multipleBuckets(testInfo: TestInfo) { + // create multipart upload 1 + val bucketName1 = givenBucket(testInfo) + .also { name -> + s3Client.createMultipartUpload { + it.bucket(name) + it.key("key1") + } + } + + // create multipart upload 2 + val bucketName2 = givenBucket() + .also { name -> + s3Client.createMultipartUpload { + it.bucket(name) + it.key("key2") + } + } + + // assert multipart upload 1 + s3Client.listMultipartUploads { + it.bucket(bucketName1) + }.also { + assertThat(it.uploads()).hasSize(1) + assertThat(it.uploads()[0].key()).isEqualTo("key1") + } + + // assert multipart upload 2 + s3Client.listMultipartUploads { + it.bucket(bucketName2) + }.also { + assertThat(it.uploads()).hasSize(1) + assertThat(it.uploads()[0].key()).isEqualTo("key2") + } + } + + /** + * Tests if a multipart upload can be aborted. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testAbortMultipartUpload(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.hasUploads() + ).isFalse + + val uploadId = s3Client + .createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + }.uploadId() + val randomBytes = randomBytes() + + val partETag = uploadPart(bucketName, UPLOAD_FILE_NAME, uploadId, 1, randomBytes) + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.hasUploads() + ).isTrue + + s3Client.listParts { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.uploadId(uploadId) + }.parts() + .also { + assertThat(it).hasSize(1) + assertThat(it[0].eTag()).isEqualTo(partETag) + } + + s3Client.abortMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.uploadId(uploadId) + } + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.hasUploads() + ).isFalse + + // List parts, make sure we find no parts + assertThatThrownBy { + s3Client.listParts { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + it.uploadId(uploadId) + } + } + .isInstanceOf(AwsServiceException::class.java) + .hasMessageContaining("Service: S3, Status Code: 404") + .asInstanceOf(InstanceOfAssertFactories.type(AwsServiceException::class.java)) + .extracting(AwsServiceException::awsErrorDetails) + .extracting(AwsErrorDetails::errorCode) + .isEqualTo("NoSuchUpload") + } + + /** + * Tests if the parts specified in CompleteUploadRequest are adhered + * irrespective of the number of parts uploaded before. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testCompleteMultipartUpload_partLeftOut(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val key = randomName + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.uploads() + ).isEmpty() + + // Initiate upload + val uploadId = s3Client + .createMultipartUpload { + it.bucket(bucketName) + it.key(key) + }.uploadId() + + // Upload 3 parts + val randomBytes1 = randomBytes() + val partETag1 = uploadPart(bucketName, key, uploadId, 1, randomBytes1) + val randomBytes2 = randomBytes() + uploadPart(bucketName, key, uploadId, 2, randomBytes2) //ignore output in this test. + val randomBytes3 = randomBytes() + val partETag3 = uploadPart(bucketName, key, uploadId, 3, randomBytes3) + + // Try to complete with these parts + val result = s3Client.completeMultipartUpload { + it.bucket(bucketName) + it.key(key) + it.uploadId(uploadId) + it.multipartUpload { + it.parts({ + it.eTag(partETag1) + it.partNumber(1) + }, + { + it.eTag(partETag3) + it.partNumber(3) + } + ) + } + } + + // Verify only 1st and 3rd counts + (DigestUtils.md5(randomBytes1) + DigestUtils.md5(randomBytes3)).also { + // verify special etag + assertThat(result.eTag()).isEqualTo("\"${DigestUtils.md5Hex(it)}-2\"") + } + + s3Client.getObject { + it.bucket(bucketName) + it.key(key) + }.use { + // verify content size + assertThat(it.response().contentLength()).isEqualTo(randomBytes1.size.toLong() + randomBytes3.size) + // verify contents + assertThat(readStreamIntoByteArray(it.buffered())).isEqualTo(concatByteArrays(randomBytes1, randomBytes3)) + } + } + + /** + * Tests that uploaded parts can be listed regardless if the MultipartUpload was completed or + * aborted. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testListParts_completeAndAbort(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val key = randomName + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.uploads() + ).isEmpty() + + // Initiate upload + val uploadId = s3Client + .createMultipartUpload { + it.bucket(bucketName) + it.key(key) + }.uploadId() + + // Upload part + val randomBytes = randomBytes() + val partETag = uploadPart(bucketName, key, uploadId, 1, randomBytes) + + // List parts, make sure we find part 1 + s3Client.listParts { + it.bucket(bucketName) + it.key(key) + it.uploadId(uploadId) + }.parts() + .also { + assertThat(it).hasSize(1) + assertThat(it[0].eTag()).isEqualTo(partETag) + } + + // Complete + s3Client.completeMultipartUpload { + it.bucket(bucketName) + it.key(key) + it.uploadId(uploadId) + it.multipartUpload { + it.parts( + { + it.eTag(partETag) + it.partNumber(1) + } + ) + } + } + + // List parts, make sure we find no parts + assertThatThrownBy { + s3Client.listParts { + it.bucket(bucketName) + it.key(key) + it.uploadId(uploadId) + } + } + .isInstanceOf(AwsServiceException::class.java) + .hasMessageContaining("Service: S3, Status Code: 404") + .asInstanceOf(InstanceOfAssertFactories.type(AwsServiceException::class.java)) + .extracting(AwsServiceException::awsErrorDetails) + .extracting(AwsErrorDetails::errorCode) + .isEqualTo("NoSuchUpload") + } + + /** + * Upload two objects, copy as parts without length, complete multipart. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun shouldCopyPartsAndComplete(testInfo: TestInfo) { + //Initiate upload + val bucketName2 = givenBucket() + val multipartUploadKey = UUID.randomUUID().toString() + + val uploadId = s3Client.createMultipartUpload { + it.bucket(bucketName2) + it.key(multipartUploadKey) + }.uploadId() + val parts: MutableList = ArrayList() + + //bucket for test data + val bucketName1 = givenBucket(testInfo) + + //create two objects, initiate copy part with full object length + val sourceKeys = arrayOf(UUID.randomUUID().toString(), UUID.randomUUID().toString()) + val allRandomBytes: MutableList = ArrayList() + for (i in sourceKeys.indices) { + val key = sourceKeys[i] + val partNumber = i + 1 + val randomBytes = randomBytes() + val metadata1 = HashMap().apply { + this["contentLength"] = randomBytes.size.toString() + } + s3Client.putObject({ + it.bucket(bucketName1) + it.key(key) + it.metadata(metadata1) + }, + RequestBody.fromInputStream(ByteArrayInputStream(randomBytes), randomBytes.size.toLong()) + ) + + s3Client.uploadPartCopy { + it.partNumber(partNumber) + it.uploadId(uploadId) + it.destinationBucket(bucketName2) + it.destinationKey(multipartUploadKey) + it.sourceKey(key) + it.sourceBucket(bucketName1) + }.also { + val etag = it.copyPartResult().eTag() + parts.add(CompletedPart.builder().eTag(etag).partNumber(partNumber).build()) + allRandomBytes.add(randomBytes) + } + } + assertThat(allRandomBytes).hasSize(2) + + // Complete with parts + val result = s3Client.completeMultipartUpload { + it.bucket(bucketName2) + it.key(multipartUploadKey) + it.uploadId(uploadId) + it.multipartUpload { + it.parts(parts) + } + } + // Verify parts + (DigestUtils.md5(allRandomBytes[0]) + DigestUtils.md5(allRandomBytes[1])).also { + // verify etag + assertThat(result.eTag()).isEqualTo("\"${DigestUtils.md5Hex(it)}-2\"") + } + + s3Client.getObject { + it.bucket(bucketName2) + it.key(multipartUploadKey) + }.use { + // verify content size + assertThat(it.response().contentLength()).isEqualTo(allRandomBytes[0].size.toLong() + allRandomBytes[1].size) + + // verify contents + assertThat(readStreamIntoByteArray(it.buffered())) + .isEqualTo(concatByteArrays(allRandomBytes[0], allRandomBytes[1])) + } + } + + /** + * Puts an Object; Copies part of that object to a new bucket; + * Requests parts for the uploadId; compares etag of upload response and parts list. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun shouldCopyObjectPart(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val uploadFile = File(sourceKey) + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucket = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val objectMetadata = mapOf(Pair("key", "value")) + + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(destinationBucket) + it.key(destinationKey) + it.metadata(objectMetadata) + } + val uploadId = initiateMultipartUploadResult.uploadId() + + val result = s3Client.uploadPartCopy { + it.uploadId(uploadId) + it.destinationBucket(destinationBucket) + it.destinationKey(destinationKey) + it.sourceKey(sourceKey) + it.sourceBucket(bucketName) + it.partNumber(1) + it.copySourceRange("bytes=0-" + (uploadFile.length() - 1)) + } + val etag = result.copyPartResult().eTag() + + s3Client.listParts { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(initiateMultipartUploadResult.uploadId()) + }.also { + assertThat(it.parts()).hasSize(1) + assertThat(it.parts()[0].eTag()).isEqualTo(etag) + } + } + + /** + * Tries to copy part of a non-existing object to a new bucket. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun shouldThrowNoSuchKeyOnCopyObjectPartForNonExistingKey(testInfo: TestInfo) { + val sourceKey = "NON_EXISTENT_KEY" + val destinationBucket = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val bucketName = givenBucket(testInfo) + val objectMetadata = mapOf(Pair("key", "value")) + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(destinationBucket) + it.key(destinationKey) + it.metadata(objectMetadata) + } + + val uploadId = initiateMultipartUploadResult.uploadId() + assertThatThrownBy { + s3Client.uploadPartCopy { + it.uploadId(uploadId) + it.destinationBucket(destinationBucket) + it.destinationKey(destinationKey) + it.sourceKey(sourceKey) + it.sourceBucket(bucketName) + it.partNumber(1) + it.copySourceRange("bytes=0-5") + } + } + .isInstanceOf(AwsServiceException::class.java) + .hasMessageContaining("Service: S3, Status Code: 404") + .asInstanceOf(InstanceOfAssertFactories.type(AwsServiceException::class.java)) + .extracting(AwsServiceException::awsErrorDetails) + .extracting(AwsErrorDetails::errorCode) + .isEqualTo("NoSuchKey") + } + + @Test + @S3VerifiedSuccess(year = 2022) + fun testUploadPartCopy_successMatch(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val uploadFile = File(sourceKey) + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucket = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val matchingEtag = putObjectResponse.eTag() + + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(destinationBucket) + it.key(destinationKey) + } + val uploadId = initiateMultipartUploadResult.uploadId() + + val result = s3Client.uploadPartCopy { + it.uploadId(uploadId) + it.destinationBucket(destinationBucket) + it.destinationKey(destinationKey) + it.sourceKey(sourceKey) + it.sourceBucket(bucketName) + it.partNumber(1) + it.copySourceRange("bytes=0-" + (uploadFile.length() - 1)) + it.copySourceIfMatch(matchingEtag) + } + val etag = result.copyPartResult().eTag() + + s3Client.listParts( + ListPartsRequest + .builder() + .bucket(initiateMultipartUploadResult.bucket()) + .key(initiateMultipartUploadResult.key()) + .uploadId(initiateMultipartUploadResult.uploadId()) + .build() + ).also { + assertThat(it.parts()).hasSize(1) + assertThat(it.parts()[0].eTag()).isEqualTo(etag) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testUploadPartCopy_successNoneMatch(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val uploadFile = File(sourceKey) + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucket = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val noneMatchingEtag = "\"${randomName}\"" + + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(destinationBucket) + it.key(destinationKey) + } + val uploadId = initiateMultipartUploadResult.uploadId() + + val result = s3Client.uploadPartCopy { + it.uploadId(uploadId) + it.destinationBucket(destinationBucket) + it.destinationKey(destinationKey) + it.sourceKey(sourceKey) + it.sourceBucket(bucketName) + it.partNumber(1) + it.copySourceRange("bytes=0-" + (uploadFile.length() - 1)) + it.copySourceIfNoneMatch(noneMatchingEtag) + } + val etag = result.copyPartResult().eTag() + + s3Client.listParts { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(initiateMultipartUploadResult.uploadId()) + }.also { + assertThat(it.parts()).hasSize(1) + assertThat(it.parts()[0].eTag()).isEqualTo(etag) + } + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testUploadPartCopy_failureMatch(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val uploadFile = File(sourceKey) + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucket = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val noneMatchingEtag = "\"${randomName}\"" + + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(destinationBucket) + it.key(destinationKey) + } + + val uploadId = initiateMultipartUploadResult.uploadId() + assertThatThrownBy { + s3Client.uploadPartCopy { + it.uploadId(uploadId) + it.destinationBucket(destinationBucket) + it.destinationKey(destinationKey) + it.sourceKey(sourceKey) + it.sourceBucket(bucketName) + it.partNumber(1) + it.copySourceRange("bytes=0-" + uploadFile.length()) + it.copySourceIfMatch(noneMatchingEtag) + } + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 412") + .hasMessageContaining(PRECONDITION_FAILED.message) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testUploadPartCopy_failureNoneMatch(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val uploadFile = File(sourceKey) + val (bucketName, putObjectResponse) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucket = givenBucket() + val destinationKey = "copyOf/$sourceKey" + val matchingEtag = putObjectResponse.eTag() + + val initiateMultipartUploadResult = s3Client.createMultipartUpload { + it.bucket(destinationBucket) + it.key(destinationKey) + } + + val uploadId = initiateMultipartUploadResult.uploadId() + assertThatThrownBy { + s3Client.uploadPartCopy { + it.uploadId(uploadId) + it.destinationBucket(destinationBucket) + it.destinationKey(destinationKey) + it.sourceKey(sourceKey) + it.sourceBucket(bucketName) + it.partNumber(1) + it.copySourceRange("bytes=0-" + uploadFile.length()) + it.copySourceIfNoneMatch(matchingEtag) + } + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining("Service: S3, Status Code: 412") + .hasMessageContaining(PRECONDITION_FAILED.message) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun createMultipartUpload_noSuchBucket() { + assertThatThrownBy { + s3Client.createMultipartUpload { + it.bucket(randomName) + it.key(UPLOAD_FILE_NAME) + } + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + + + @Test + @S3VerifiedSuccess(year = 2025) + fun listMultipartUploads_noSuchBucket() { + assertThatThrownBy { + s3Client.listMultipartUploads { + it.bucket(randomName) + } + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun abortMultipartUpload_noSuchBucket() { + assertThatThrownBy { + s3Client.abortMultipartUpload { + it.bucket(randomName) + it.key(UPLOAD_FILE_NAME) + it.uploadId("uploadId") + } + } + .isInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun transferManagerUpload_noSuchSourceBucket() { + val uploadFile = File(UPLOAD_FILE_NAME) + assertThatThrownBy { + transferManager.upload { + it.putObjectRequest { + it.bucket(randomName) + it.key(UPLOAD_FILE_NAME) + } + it.requestBody(AsyncRequestBody.fromFile(uploadFile)) + }.completionFuture().join() + } + .isInstanceOf(CompletionException::class.java) + .hasCauseInstanceOf(NoSuchBucketException::class.java) + .hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun transferManagerCopy_noSuchDestinationBucket(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, putObjectResult) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + assertThatThrownBy { + transferManager.copy { + it.copyObjectRequest { + it.sourceBucket(randomName) + it.sourceKey(UPLOAD_FILE_NAME) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + } + }.completionFuture().join() + } + .isInstanceOf(CompletionException::class.java) + .hasCauseInstanceOf(NoSuchKeyException::class.java) + //TODO: not sure why AWS SDK v2 does not return the correct error message, S3Mock returns the correct message. + //.hasMessageContaining(NO_SUCH_KEY) + //TODO: not sure why AWS SDK v2 does not return the correct exception here, S3Mock returns the correct error message. + //.hasCauseInstanceOf(NoSuchBucketException::class.java) + //.hasMessageContaining(NO_SUCH_BUCKET) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun transferManagerCopy_noSuchSourceKey(testInfo: TestInfo) { + val sourceKey = UPLOAD_FILE_NAME + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) + val destinationBucketName = givenBucket() + val destinationKey = "copyOf/$sourceKey" + + assertThatThrownBy { + transferManager.copy { + it.copyObjectRequest { + it.sourceBucket(bucketName) + it.sourceKey(randomName) + it.destinationBucket(destinationBucketName) + it.destinationKey(destinationKey) + } + }.completionFuture().join() + } + .isInstanceOf(CompletionException::class.java) + .hasCauseInstanceOf(NoSuchKeyException::class.java) + //TODO: not sure why AWS SDK v2 does not return the correct error message, S3Mock returns the correct message. + //.hasMessageContaining(NO_SUCH_KEY) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun uploadMultipart_invalidPartNumber(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val initiateMultipartUploadResult = s3Client + .createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + } + val uploadId = initiateMultipartUploadResult.uploadId() + + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.uploads() + ).isNotEmpty + + val invalidPartNumber = 0 + assertThatThrownBy { + s3Client.uploadPart( + { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.partNumber(invalidPartNumber) + }, + RequestBody.fromFile(uploadFile) + ) + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining(INVALID_PART_NUMBER) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun completeMultipartUpload_nonExistingPartNumber(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val uploadFile = File(UPLOAD_FILE_NAME) + val initiateMultipartUploadResult = s3Client + .createMultipartUpload { + it.bucket(bucketName) + it.key(UPLOAD_FILE_NAME) + } + val uploadId = initiateMultipartUploadResult.uploadId() + + assertThat( + s3Client.listMultipartUploads { + it.bucket(bucketName) + }.uploads() + ).isNotEmpty + + val eTag = s3Client.uploadPart( + { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.partNumber(1) + }, + RequestBody.fromFile(uploadFile) + ).eTag() + + val invalidPartNumber = 10 + assertThatThrownBy { + s3Client.completeMultipartUpload { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.multipartUpload { + it.parts({ + it.eTag(eTag) + it.partNumber(invalidPartNumber) + }) + } + } + } + .isInstanceOf(S3Exception::class.java) + .hasMessageContaining(INVALID_PART) + } + + private fun uploadPart( + bucketName: String, + key: String, + uploadId: String, + partNumber: Int, + randomBytes: ByteArray + ): String { + return s3Client.uploadPart({ + it.bucket(bucketName) + it.key(key) + it.uploadId(uploadId) + it.partNumber(partNumber) + it.contentLength(randomBytes.size.toLong()) + }, + RequestBody.fromInputStream(ByteArrayInputStream(randomBytes), randomBytes.size.toLong()) + ).eTag() + } + + companion object { + private const val NO_SUCH_BUCKET = "The specified bucket does not exist" + private const val INVALID_PART_NUMBER = "Part number must be an integer between 1 and 10000, inclusive" + private const val INVALID_PART = "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tagSet might not have matched the part's entity tagSet." + } +} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartUploadV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartUploadV1IT.kt deleted file mode 100644 index 7e48dda30..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartUploadV1IT.kt +++ /dev/null @@ -1,599 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.AbortMultipartUploadRequest -import com.amazonaws.services.s3.model.AmazonS3Exception -import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest -import com.amazonaws.services.s3.model.CopyPartRequest -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest -import com.amazonaws.services.s3.model.ListMultipartUploadsRequest -import com.amazonaws.services.s3.model.ListPartsRequest -import com.amazonaws.services.s3.model.ObjectMetadata -import com.amazonaws.services.s3.model.PartETag -import com.amazonaws.services.s3.model.PutObjectRequest -import com.amazonaws.services.s3.model.UploadPartRequest -import org.apache.commons.codec.digest.DigestUtils -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import software.amazon.awssdk.utils.http.SdkHttpUtils -import java.io.ByteArrayInputStream -import java.io.File -import java.io.FileInputStream -import java.util.Date -import java.util.UUID - -/** - * Test the application using the AmazonS3 SDK V1. - */ -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class MultiPartUploadV1IT : S3TestBase() { - val s3Client: AmazonS3 = createS3ClientV1() - - /** - * Tests if user metadata can be passed by multipart upload. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testMultipartUpload_withUserMetadata(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val objectMetadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - } - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload( - InitiateMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME, objectMetadata) - ) - val uploadId = initiateMultipartUploadResult.uploadId - val uploadPartResult = s3Client.uploadPart( - UploadPartRequest() - .withBucketName(initiateMultipartUploadResult.bucketName) - .withKey(initiateMultipartUploadResult.key) - .withUploadId(uploadId) - .withFile(uploadFile) - .withFileOffset(0) - .withPartNumber(1) - .withPartSize(uploadFile.length()) - .withLastPart(true) - ) - val partETags = listOf(uploadPartResult.partETag) - s3Client.completeMultipartUpload( - CompleteMultipartUploadRequest( - initiateMultipartUploadResult.bucketName, - initiateMultipartUploadResult.key, - initiateMultipartUploadResult.uploadId, - partETags - ) - ) - - s3Client.getObjectMetadata( - initiateMultipartUploadResult.bucketName, initiateMultipartUploadResult.key - ).also { - assertThat(it.userMetadata).isEqualTo(objectMetadata.userMetadata) - } - } - - /** - * Tests if a multipart upload with the last part being smaller than 5MB works. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldAllowMultipartUploads(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val objectMetadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - } - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload( - InitiateMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME, objectMetadata) - ) - val uploadId = initiateMultipartUploadResult.uploadId - // upload part 1, >5MB - val randomBytes = randomBytes() - val partETag = uploadPart(bucketName, UPLOAD_FILE_NAME, uploadId, 1, randomBytes) - // upload part 2, <5MB - val uploadPartResult = s3Client.uploadPart( - UploadPartRequest() - .withBucketName(initiateMultipartUploadResult.bucketName) - .withKey(initiateMultipartUploadResult.key) - .withUploadId(uploadId) - .withFile(uploadFile) - .withPartNumber(2) - .withPartSize(uploadFile.length()) - .withLastPart(true) - ) - val partETags = listOf(partETag, uploadPartResult.partETag) - val completeMultipartUpload = s3Client.completeMultipartUpload( - CompleteMultipartUploadRequest( - initiateMultipartUploadResult.bucketName, - initiateMultipartUploadResult.key, - initiateMultipartUploadResult.uploadId, - partETags - ) - ) - // Verify only 1st and 3rd counts - val uploadFileBytes = readStreamIntoByteArray(uploadFile.inputStream()) - (DigestUtils.md5(randomBytes) + DigestUtils.md5(uploadFileBytes)).also { - // verify special etag - assertThat(completeMultipartUpload.eTag).isEqualTo("${DigestUtils.md5Hex(it)}-2") - } - - s3Client.getObject(bucketName, UPLOAD_FILE_NAME).use { - // verify content size - assertThat(it.objectMetadata.contentLength).isEqualTo(randomBytes.size.toLong() + uploadFileBytes.size.toLong()) - - // verify contents - assertThat(readStreamIntoByteArray(it.objectContent)).`as`( - "Object contents doesn't match" - ).isEqualTo(concatByteArrays(randomBytes, uploadFileBytes)) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldInitiateMultipartAndRetrieveParts(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val objectMetadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - } - val hash = DigestUtils.md5Hex(FileInputStream(uploadFile)) - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload( - InitiateMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME, objectMetadata) - ) - val uploadId = initiateMultipartUploadResult.uploadId - val key = initiateMultipartUploadResult.key - s3Client.uploadPart( - UploadPartRequest() - .withBucketName(initiateMultipartUploadResult.bucketName) - .withKey(initiateMultipartUploadResult.key) - .withUploadId(uploadId) - .withFile(uploadFile) - .withFileOffset(0) - .withPartNumber(1) - .withPartSize(uploadFile.length()) - .withLastPart(true) - ) - - ListPartsRequest( - bucketName, - key, - uploadId - ).also { listPartsRequest -> - s3Client.listParts(listPartsRequest).also { partListing -> - assertThat(partListing.parts).hasSize(1) - partListing.parts[0].also { - assertThat(it.eTag).isEqualTo(hash) - assertThat(it.partNumber).isEqualTo(1) - assertThat(it.lastModified).isExactlyInstanceOf(Date::class.java) - } - } - } - } - - /** - * Tests if not yet completed / aborted multipart uploads are listed. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldListMultipartUploads(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - assertThat( - s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)).multipartUploads - ).isEmpty() - - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload(InitiateMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME)) - val uploadId = initiateMultipartUploadResult.uploadId - - s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)).also { listing -> - assertThat(listing.multipartUploads).isNotEmpty - assertThat(listing.bucketName).isEqualTo(bucketName) - assertThat(listing.multipartUploads).hasSize(1) - - listing.multipartUploads[0].also { upload -> - assertThat(upload.uploadId).isEqualTo(uploadId) - assertThat(upload.key).isEqualTo(UPLOAD_FILE_NAME) - } - } - } - - /** - * Tests if empty parts list of not yet completed multipart upload is returned. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldListEmptyPartListForMultipartUpload(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - assertThat( - s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)) - .multipartUploads - ).isEmpty() - - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload(InitiateMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME)) - val uploadId = initiateMultipartUploadResult.uploadId - - s3Client.listParts(ListPartsRequest(bucketName, UPLOAD_FILE_NAME, uploadId)).also { listing -> - assertThat(listing.parts).isEmpty() - assertThat(listing.bucketName).isEqualTo(bucketName) - assertThat(listing.uploadId).isEqualTo(uploadId) - assertThat(SdkHttpUtils.urlDecode(listing.key)).isEqualTo(UPLOAD_FILE_NAME) - } - } - - /** - * Tests that an exception is thrown when listing parts if the upload id is unknown. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldThrowOnListMultipartUploadsWithUnknownId(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - assertThatThrownBy { s3Client.listParts(ListPartsRequest(bucketName, "NON_EXISTENT_KEY", - "NON_EXISTENT_UPLOAD_ID")) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 404; Error Code: NoSuchUpload") - } - - /** - * Tests if not yet completed / aborted multipart uploads are listed with prefix filtering. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldListMultipartUploadsWithPrefix(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - s3Client.initiateMultipartUpload( - InitiateMultipartUploadRequest(bucketName, "key1") - ) - s3Client.initiateMultipartUpload( - InitiateMultipartUploadRequest(bucketName, "key2") - ) - val listMultipartUploadsRequest = ListMultipartUploadsRequest(bucketName).apply { - this.prefix = "key2" - } - - s3Client.listMultipartUploads(listMultipartUploadsRequest).also { listing -> - assertThat(listing.multipartUploads).hasSize(1) - assertThat(listing.multipartUploads[0].key).isEqualTo("key2") - } - } - - /** - * Tests if multipart uploads are stored and can be retrieved by bucket. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldListMultipartUploadsWithBucket(testInfo: TestInfo) { - // create multipart upload 1 - val bucketName1 = givenBucketV1(testInfo) - s3Client.initiateMultipartUpload( - InitiateMultipartUploadRequest(bucketName1, "key1") - ) - // create multipart upload 2 - val bucketName2 = givenRandomBucketV1() - s3Client.initiateMultipartUpload( - InitiateMultipartUploadRequest(bucketName2, "key2") - ) - - // assert multipart upload 1 - val listMultipartUploadsRequest1 = ListMultipartUploadsRequest(bucketName1) - s3Client.listMultipartUploads(listMultipartUploadsRequest1).also { listing -> - assertThat(listing.multipartUploads).hasSize(1) - assertThat(listing.multipartUploads[0].key).isEqualTo("key1") - } - - // assert multipart upload 2 - val listMultipartUploadsRequest2 = ListMultipartUploadsRequest(bucketName2) - s3Client.listMultipartUploads(listMultipartUploadsRequest2).also { listing -> - assertThat(listing.multipartUploads).hasSize(1) - assertThat(listing.multipartUploads[0].key).isEqualTo("key2") - } - } - - /** - * Tests if a multipart upload can be aborted. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldAbortMultipartUpload(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - assertThat(s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)).multipartUploads).isEmpty() - val result = s3Client.initiateMultipartUpload(InitiateMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME)) - val uploadId = result.uploadId - val randomBytes = randomBytes() - - val partETag = uploadPart(bucketName, UPLOAD_FILE_NAME, uploadId, 1, randomBytes) - assertThat(s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)).multipartUploads).isNotEmpty - - s3Client.listParts(ListPartsRequest(bucketName, UPLOAD_FILE_NAME, uploadId)).also { listing -> - listing.parts.also { - assertThat(it).hasSize(1) - assertThat(it[0].eTag).isEqualTo(partETag.eTag) - } - } - - s3Client.abortMultipartUpload(AbortMultipartUploadRequest(bucketName, UPLOAD_FILE_NAME, uploadId)) - assertThat(s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)).multipartUploads).isEmpty() - - // List parts, make sure we find no parts - assertThatThrownBy { s3Client.listParts(ListPartsRequest(bucketName, UPLOAD_FILE_NAME, - uploadId)) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 404; Error Code: NoSuchUpload") - } - - /** - * Tests if the parts specified in CompleteUploadRequest are adhered - * irrespective of the number of parts uploaded before. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldAdherePartsInCompleteMultipartUploadRequest(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val key = UUID.randomUUID().toString() - assertThat(s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)).multipartUploads).isEmpty() - - // Initiate upload - val multipartUploadResult = s3Client.initiateMultipartUpload(InitiateMultipartUploadRequest(bucketName, key)) - val uploadId = multipartUploadResult.uploadId - - // Upload 3 parts - val randomBytes1 = randomBytes() - val partETag1 = uploadPart(bucketName, key, uploadId, 1, randomBytes1) - val randomBytes2 = randomBytes() - uploadPart(bucketName, key, uploadId, 2, randomBytes2) //ignore result in this test - val randomBytes3 = randomBytes() - val partETag3 = uploadPart(bucketName, key, uploadId, 3, randomBytes3) - - // Adding to parts list only 1st and 3rd part - val parts: MutableList = ArrayList().apply { - this.add(partETag1) - this.add(partETag3) - } - - // Try to complete with these parts - val result = s3Client.completeMultipartUpload(CompleteMultipartUploadRequest(bucketName, key, uploadId, parts)) - - // Verify only 1st and 3rd counts - (DigestUtils.md5(randomBytes1) + DigestUtils.md5(randomBytes3)).also { - // verify special etag - assertThat(result.eTag).isEqualTo("${DigestUtils.md5Hex(it)}-2") - } - - - s3Client.getObject(bucketName, key).use { - // verify content size - assertThat(it.objectMetadata.contentLength).isEqualTo(randomBytes1.size.toLong() + randomBytes3.size) - // verify contents - assertThat(readStreamIntoByteArray(it.objectContent)).isEqualTo(concatByteArrays(randomBytes1, randomBytes3)) - } - } - - /** - * Tests that uploaded parts can be listed regardless if the MultipartUpload was completed or - * aborted. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldListPartsOnCompleteOrAbort(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val key = randomName - assertThat(s3Client.listMultipartUploads(ListMultipartUploadsRequest(bucketName)).multipartUploads).isEmpty() - - // Initiate upload - val multipartUploadResult = s3Client.initiateMultipartUpload(InitiateMultipartUploadRequest(bucketName, key)) - val uploadId = multipartUploadResult.uploadId - - // Upload part - val randomBytes = randomBytes() - val partETag = uploadPart(bucketName, key, uploadId, 1, randomBytes) - - // List parts, make sure we find part 1 - s3Client.listParts(ListPartsRequest(bucketName, key, uploadId)).also { listing -> - listing.parts.also { - assertThat(it).hasSize(1) - assertThat(it[0].eTag).isEqualTo(partETag.eTag) - } - } - - // Complete, ignore result in this test - s3Client.completeMultipartUpload(CompleteMultipartUploadRequest(bucketName, key, uploadId, listOf(partETag))) - - // List parts, make sure we find no parts - assertThatThrownBy { s3Client.listParts(ListPartsRequest(bucketName, key, uploadId)) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 404; Error Code: NoSuchUpload") - } - - /** - * Upload two objects, copy as parts without length, complete multipart. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyPartsAndComplete(testInfo: TestInfo) { - //Initiate upload in random bucket - val bucketName2 = givenRandomBucketV1() - val multipartUploadKey = randomName - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload(InitiateMultipartUploadRequest(bucketName2, multipartUploadKey)) - val uploadId = initiateMultipartUploadResult.uploadId - val parts: MutableList = ArrayList() - - //bucket for test data - val bucketName1 = givenBucketV1(testInfo) - - //create two objects, initiate copy part with full object length - val sourceKeys = arrayOf(UUID.randomUUID().toString(), UUID.randomUUID().toString()) - val allRandomBytes: MutableList = ArrayList() - for (i in sourceKeys.indices) { - val key = sourceKeys[i] - val partNumber = i + 1 - val randomBytes = randomBytes() - val metadata1 = ObjectMetadata().apply { - this.contentLength = randomBytes.size.toLong() - } - s3Client.putObject(PutObjectRequest(bucketName1, key, ByteArrayInputStream(randomBytes), metadata1)) - val request = CopyPartRequest() - .withPartNumber(partNumber) - .withUploadId(uploadId) - .withDestinationBucketName(bucketName2) - .withDestinationKey(multipartUploadKey) - .withSourceKey(key) - .withSourceBucketName(bucketName1) - val etag = s3Client.copyPart(request).eTag - val partETag = PartETag(partNumber, etag) - parts.add(partETag) - allRandomBytes.add(randomBytes) - } - assertThat(allRandomBytes).hasSize(2) - - // Complete with parts - val result = s3Client.completeMultipartUpload( - CompleteMultipartUploadRequest(bucketName2, multipartUploadKey, uploadId, parts) - ) - - // Verify parts - (DigestUtils.md5(allRandomBytes[0]) + DigestUtils.md5(allRandomBytes[1])).also { - // verify etag - assertThat(result.eTag).isEqualTo("${DigestUtils.md5Hex(it)}-2") - } - - - s3Client.getObject(bucketName2, multipartUploadKey).use { - // verify content size - assertThat(it.objectMetadata.contentLength).isEqualTo(allRandomBytes[0].size.toLong() + allRandomBytes[1].size) - - // verify contents - assertThat(readStreamIntoByteArray(it.objectContent)) - .isEqualTo(concatByteArrays(allRandomBytes[0], allRandomBytes[1])) - } - } - - /** - * Puts an Object; Copies part of that object to a new bucket; - * Requests parts for the uploadId; compares etag of upload response and parts list. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObjectPart(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val (bucketName, putObjectResult) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - val objectMetadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - } - - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload( - InitiateMultipartUploadRequest( - destinationBucketName, destinationKey, - objectMetadata - ) - ) - val uploadId = initiateMultipartUploadResult.uploadId - val copyPartRequest = CopyPartRequest().apply { - this.destinationBucketName = destinationBucketName - this.uploadId = uploadId - this.destinationKey = destinationKey - this.sourceBucketName = bucketName - this.sourceKey = sourceKey - this.firstByte = 0L - this.lastByte = putObjectResult.metadata.contentLength - this.partNumber = 1 - } - val copyPartResult = s3Client.copyPart(copyPartRequest) - - s3Client.listParts( - ListPartsRequest( - initiateMultipartUploadResult.bucketName, - initiateMultipartUploadResult.key, - initiateMultipartUploadResult.uploadId - ) - ).also { - assertThat(it.parts).hasSize(1) - assertThat(it.parts[0].eTag).isEqualTo(copyPartResult.eTag) - } - } - - /** - * Tries to copy part of a non-existing object to a new bucket. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldThrowNoSuchKeyOnCopyObjectPartForNonExistingKey(testInfo: TestInfo) { - val sourceKey = "NON_EXISTENT_KEY" - val destinationBucketName = givenRandomBucketV1() - val destinationKey = "copyOf/$sourceKey" - val bucketName = givenBucketV1(testInfo) - val objectMetadata = ObjectMetadata().apply { - this.addUserMetadata("key", "value") - } - val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload( - InitiateMultipartUploadRequest( - destinationBucketName, destinationKey, - objectMetadata - ) - ) - val uploadId = initiateMultipartUploadResult.uploadId - - val copyPartRequest = CopyPartRequest().apply { - this.destinationBucketName = destinationBucketName - this.uploadId = uploadId - this.destinationKey = destinationKey - this.sourceBucketName = bucketName - this.sourceKey = sourceKey - this.firstByte = 0L - this.lastByte = 5L - this.partNumber = 1 - } - assertThatThrownBy { s3Client.copyPart(copyPartRequest) } - .isInstanceOf(AmazonS3Exception::class.java) - .hasMessageContaining("Status Code: 404; Error Code: NoSuchKey") - } - - private fun uploadPart( - bucketName: String, - key: String, - uploadId: String, - partNumber: Int, - randomBytes: ByteArray - ): PartETag { - return s3Client - .uploadPart( - createUploadPartRequest(bucketName, key, uploadId) - .withPartNumber(partNumber) - .withPartSize(randomBytes.size.toLong()) - .withInputStream(ByteArrayInputStream(randomBytes)) - ) - .partETag - } - - private fun createUploadPartRequest(bucketName: String, key: String, uploadId: String): UploadPartRequest { - return UploadPartRequest() - .withBucketName(bucketName) - .withKey(key) - .withUploadId(uploadId) - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartUploadV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartUploadV2IT.kt deleted file mode 100644 index 22e47c8c2..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartUploadV2IT.kt +++ /dev/null @@ -1,1437 +0,0 @@ -/* - * Copyright 2017-2024 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import com.adobe.testing.s3mock.S3Exception.PRECONDITION_FAILED -import com.adobe.testing.s3mock.util.DigestUtil -import com.adobe.testing.s3mock.util.DigestUtil.hexDigest -import org.apache.commons.codec.digest.DigestUtils -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.assertj.core.api.InstanceOfAssertFactories -import org.assertj.core.util.Files -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource -import org.springframework.web.util.UriUtils -import software.amazon.awssdk.awscore.exception.AwsErrorDetails -import software.amazon.awssdk.awscore.exception.AwsServiceException -import software.amazon.awssdk.core.async.AsyncRequestBody -import software.amazon.awssdk.core.checksums.Algorithm.CRC32 -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.S3AsyncClient -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest -import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm -import software.amazon.awssdk.services.s3.model.ChecksumMode -import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest -import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload -import software.amazon.awssdk.services.s3.model.CompletedPart -import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest -import software.amazon.awssdk.services.s3.model.GetObjectRequest -import software.amazon.awssdk.services.s3.model.HeadObjectRequest -import software.amazon.awssdk.services.s3.model.ListMultipartUploadsRequest -import software.amazon.awssdk.services.s3.model.ListPartsRequest -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.S3Exception -import software.amazon.awssdk.services.s3.model.UploadPartCopyRequest -import software.amazon.awssdk.services.s3.model.UploadPartRequest -import software.amazon.awssdk.transfer.s3.S3TransferManager -import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest -import software.amazon.awssdk.transfer.s3.model.UploadFileRequest -import software.amazon.awssdk.utils.http.SdkHttpUtils -import java.io.ByteArrayInputStream -import java.io.File -import java.io.FileInputStream -import java.nio.charset.StandardCharsets -import java.time.Instant -import java.util.UUID - - -internal class MultiPartUploadV2IT : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() - private val s3AsyncClientV2: S3AsyncClient = createS3AsyncClientV2() - private val s3CrtAsyncClientV2: S3AsyncClient = createS3CrtAsyncClientV2() - private val autoS3CrtAsyncClientV2: S3AsyncClient = createAutoS3CrtAsyncClientV2() - private val transferManagerV2: S3TransferManager = createTransferManagerV2() - - @Test - @S3VerifiedSuccess(year = 2024) - fun testMultipartUpload_asyncClient(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - s3CrtAsyncClientV2.putObject( - PutObjectRequest - .builder() - .bucket(bucketName) - .key(uploadFile.name) - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .build(), - AsyncRequestBody.fromFile(uploadFile) - ).join().also { - assertThat(it.checksumCRC32()).isEqualTo(DigestUtil.checksumFor(uploadFile.toPath(), CRC32)) - } - - s3AsyncClientV2.waiter() - .waitUntilObjectExists( - HeadObjectRequest - .builder() - .bucket(bucketName) - .key(uploadFile.name) - .build() - ) - - s3ClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(uploadFile.name) - .build() - ).use { - val uploadDigest = hexDigest(uploadFile) - val newTemporaryFile = Files.newTemporaryFile() - it.transferTo(java.nio.file.Files.newOutputStream(newTemporaryFile.toPath())) - assertThat(newTemporaryFile).hasSize(uploadFile.length()) - assertThat(newTemporaryFile).hasSameBinaryContentAs(uploadFile) - val downloadedDigest = hexDigest(newTemporaryFile) - assertThat(uploadDigest).isEqualTo(downloadedDigest) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testMultipartUpload_transferManager(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - transferManagerV2 - .uploadFile( - UploadFileRequest - .builder() - .putObjectRequest( - PutObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) - .source(uploadFile) - .build() - ).completionFuture().join() - - s3ClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ).use { - assertThat(it.response().contentLength()).isEqualTo(uploadFile.length()) - } - - val downloadFile = Files.newTemporaryFile() - transferManagerV2.downloadFile( - DownloadFileRequest - .builder() - .getObjectRequest( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) - .destination(downloadFile) - .build() - ).also { download -> - download.completionFuture().join().response().also { - assertThat(it.contentLength()).isEqualTo(uploadFile.length()) - } - } - assertThat(downloadFile.length()).isEqualTo(uploadFile.length()) - assertThat(downloadFile).hasSameBinaryContentAs(uploadFile) - } - - /** - * Tests if user metadata can be passed by multipart upload. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testMultipartUpload_withUserMetadata(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val objectMetadata = mapOf(Pair("key", "value")) - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest.builder().bucket(bucketName).key(UPLOAD_FILE_NAME) - .metadata(objectMetadata).build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - val uploadPartResult = s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .partNumber(1) - .contentLength(uploadFile.length()).build(), - //.lastPart(true) - RequestBody.fromFile(uploadFile), - ) - - s3ClientV2.completeMultipartUpload( - CompleteMultipartUploadRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(initiateMultipartUploadResult.uploadId()) - .multipartUpload( - CompletedMultipartUpload - .builder() - .parts( - CompletedPart - .builder() - .eTag(uploadPartResult.eTag()) - .partNumber(1) - .build() - ) - .build() - ) - .build() - ) - - s3ClientV2.getObject( - GetObjectRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .build() - ).use { - assertThat(it.response().metadata()).isEqualTo(objectMetadata) - } - } - - /** - * Tests if a multipart upload with the last part being smaller than 5MB works. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testMultipartUpload(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val objectMetadata = mapOf(Pair("key", "value")) - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .metadata(objectMetadata) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - // upload part 1, >5MB - val randomBytes = randomBytes() - val etag1 = uploadPart(bucketName, UPLOAD_FILE_NAME, uploadId, 1, randomBytes) - // upload part 2, <5MB - val etag2 = s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .partNumber(2) - .contentLength(uploadFile.length()).build(), - //.lastPart(true) - RequestBody.fromFile(uploadFile), - ).eTag() - - val completeMultipartUpload = s3ClientV2.completeMultipartUpload( - CompleteMultipartUploadRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(initiateMultipartUploadResult.uploadId()) - .multipartUpload( - CompletedMultipartUpload - .builder() - .parts( - CompletedPart - .builder() - .eTag(etag1) - .partNumber(1) - .build(), - CompletedPart - .builder() - .eTag(etag2) - .partNumber(2) - .build() - ) - .build() - ) - .build() - ) - - val uploadFileBytes = readStreamIntoByteArray(uploadFile.inputStream()) - - (DigestUtils.md5(randomBytes) + DigestUtils.md5(uploadFileBytes)).also { - // verify special etag - assertThat(completeMultipartUpload.eTag()).isEqualTo("\"${DigestUtils.md5Hex(it)}-2\"") - } - - s3ClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ).use { - // verify content size - assertThat(it.response().contentLength()).isEqualTo(randomBytes.size.toLong() + uploadFileBytes.size.toLong()) - // verify contents - assertThat(readStreamIntoByteArray(it.buffered())).isEqualTo(concatByteArrays(randomBytes, uploadFileBytes)) - assertThat(it.response().metadata()).isEqualTo(objectMetadata) - } - - assertThat(completeMultipartUpload.location()) - .isEqualTo("${serviceEndpoint}/$bucketName/${UriUtils.encode(UPLOAD_FILE_NAME, StandardCharsets.UTF_8)}") - } - - - /** - * Tests if a multipart upload with the last part being smaller than 5MB works. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testMultipartUpload_checksum(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(TEST_IMAGE_TIFF) - //construct uploadfile >5MB - val tempFile = Files.newTemporaryFile().also { - (readStreamIntoByteArray(uploadFile.inputStream()) + - readStreamIntoByteArray(uploadFile.inputStream()) + - readStreamIntoByteArray(uploadFile.inputStream())) - .inputStream() - .copyTo(it.outputStream()) - } - - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest.builder() - .bucket(bucketName) - .key(TEST_IMAGE_TIFF) - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - // upload part 1, <5MB - val partResponse1 = s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .partNumber(1) - .contentLength(tempFile.length()).build(), - //.lastPart(true) - RequestBody.fromFile(tempFile), - ) - val etag1 = partResponse1.eTag() - val checksum1 = partResponse1.checksumCRC32() - // upload part 2, <5MB - val partResponse2 = s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .checksumAlgorithm(ChecksumAlgorithm.CRC32) - .partNumber(2) - .contentLength(uploadFile.length()).build(), - //.lastPart(true) - RequestBody.fromFile(uploadFile), - ) - val etag2 = partResponse2.eTag() - val checksum2 = partResponse2.checksumCRC32() - val localChecksum1 = DigestUtil.checksumFor(tempFile.toPath(), CRC32) - assertThat(checksum1).isEqualTo(localChecksum1) - val localChecksum2 = DigestUtil.checksumFor(uploadFile.toPath(), CRC32) - assertThat(checksum2).isEqualTo(localChecksum2) - - val completeMultipartUpload = s3ClientV2.completeMultipartUpload( - CompleteMultipartUploadRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(initiateMultipartUploadResult.uploadId()) - .multipartUpload( - CompletedMultipartUpload - .builder() - .parts( - CompletedPart - .builder() - .eTag(etag1) - .partNumber(1) - .checksumCRC32(checksum1) - .build(), - CompletedPart - .builder() - .eTag(etag2) - .partNumber(2) - .checksumCRC32(checksum2) - .build() - ) - .build() - ) - .build() - ) - - (DigestUtils.md5(tempFile.readBytes()) + DigestUtils.md5(readStreamIntoByteArray(uploadFile.inputStream()))).also { - // verify special etag - assertThat(completeMultipartUpload.eTag()).isEqualTo("\"${DigestUtils.md5Hex(it)}-2\"") - } - - s3ClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(TEST_IMAGE_TIFF) - .checksumMode(ChecksumMode.ENABLED) - .build() - ).use { - // verify content size - assertThat(it.response().contentLength()).isEqualTo(tempFile.length() + uploadFile.length()) - // verify contents - assertThat(readStreamIntoByteArray(it.buffered())).isEqualTo(tempFile.readBytes() + uploadFile.readBytes()) - assertThat(it.response().checksumCRC32()).isEqualTo("oGk6qg==-2") - } - - assertThat(completeMultipartUpload.location()) - .isEqualTo("${serviceEndpoint}/$bucketName/${UriUtils.encode(TEST_IMAGE_TIFF, StandardCharsets.UTF_8)}") - } - - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["checksumAlgorithms"]) - fun testUploadPart_checksumAlgorithm(checksumAlgorithm: ChecksumAlgorithm, testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload(CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .checksumAlgorithm(checksumAlgorithm) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .checksumAlgorithm(checksumAlgorithm) - .partNumber(1) - .contentLength(uploadFile.length()).build(), - //.lastPart(true) - RequestBody.fromFile(uploadFile), - ).also { - val actualChecksum = it.checksum(checksumAlgorithm) - assertThat(actualChecksum).isNotBlank - assertThat(actualChecksum).isEqualTo(expectedChecksum) - } - s3ClientV2.abortMultipartUpload( - AbortMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .uploadId(uploadId) - .build() - ) - } - - @S3VerifiedSuccess(year = 2024) - @ParameterizedTest - @MethodSource(value = ["checksumAlgorithms"]) - fun testMultipartUpload_checksum(checksumAlgorithm: ChecksumAlgorithm, testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), checksumAlgorithm.toAlgorithm()) - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload(CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .checksumAlgorithm(checksumAlgorithm) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .checksum(expectedChecksum, checksumAlgorithm) - .partNumber(1) - .contentLength(uploadFile.length()).build(), - //.lastPart(true) - RequestBody.fromFile(uploadFile), - ).also { - val actualChecksum = it.checksum(checksumAlgorithm) - assertThat(actualChecksum).isNotBlank - assertThat(actualChecksum).isEqualTo(expectedChecksum) - } - s3ClientV2.abortMultipartUpload( - AbortMultipartUploadRequest.builder().bucket(bucketName).key(UPLOAD_FILE_NAME) - .uploadId(uploadId).build() - ) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testMultipartUpload_wrongChecksum(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val expectedChecksum = "wrongChecksum" - val checksumAlgorithm = ChecksumAlgorithm.SHA1 - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload(CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - assertThatThrownBy { - s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(uploadId) - .checksum(expectedChecksum, checksumAlgorithm) - .partNumber(1) - .contentLength(uploadFile.length()).build(), - //.lastPart(true) - RequestBody.fromFile(uploadFile), - ) - } - .isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 400") - .hasMessageContaining("Value for x-amz-checksum-sha1 header is invalid.") - } - - private fun UploadPartRequest.Builder.checksum( - checksum: String, - checksumAlgorithm: ChecksumAlgorithm - ): UploadPartRequest.Builder = - when (checksumAlgorithm) { - ChecksumAlgorithm.SHA1 -> this.checksumSHA1(checksum) - ChecksumAlgorithm.SHA256 -> this.checksumSHA256(checksum) - ChecksumAlgorithm.CRC32 -> this.checksumCRC32(checksum) - ChecksumAlgorithm.CRC32_C -> this.checksumCRC32C(checksum) - else -> error("Unknown checksum algorithm") - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testInitiateMultipartAndRetrieveParts(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val objectMetadata = mapOf(Pair("key", "value")) - val hash = DigestUtils.md5Hex(FileInputStream(uploadFile)) - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .metadata(objectMetadata) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - val key = initiateMultipartUploadResult.key() - - s3ClientV2.uploadPart( - UploadPartRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(key) - .uploadId(uploadId) - .partNumber(1) - .contentLength(uploadFile.length()).build(), - //.lastPart(true) - RequestBody.fromFile(uploadFile), - ) - - val listPartsRequest = ListPartsRequest - .builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .build() - val partListing = s3ClientV2.listParts(listPartsRequest) - .also { - assertThat(it.parts()).hasSize(1) - } - - partListing.parts()[0].also { - assertThat(it.eTag()).isEqualTo("\"" + hash + "\"") - assertThat(it.partNumber()).isEqualTo(1) - assertThat(it.lastModified()).isExactlyInstanceOf(Instant::class.java) - } - } - - /** - * Tests if not yet completed / aborted multipart uploads are listed. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testListMultipartUploads_ok(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - assertThat( - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(bucketName) - .build() - ) - .uploads() - ).isEmpty() - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest.builder().bucket(bucketName).build() - ).also { listing -> - assertThat(listing.uploads()).isNotEmpty - assertThat(listing.bucket()).isEqualTo(bucketName) - assertThat(listing.uploads()).hasSize(1) - - listing.uploads()[0] - .also { - assertThat(it.uploadId()).isEqualTo(uploadId) - assertThat(it.key()).isEqualTo(UPLOAD_FILE_NAME) - } - } - } - - /** - * Tests if empty parts list of not yet completed multipart upload is returned. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testListMultipartUploads_empty(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - assertThat( - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(bucketName) - .build() - ).uploads() - ).isEmpty() - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - s3ClientV2 - .listParts( - ListPartsRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .uploadId(uploadId) - .build() - ).also { - assertThat(it.parts()).isEmpty() - assertThat(it.bucket()).isEqualTo(bucketName) - assertThat(it.uploadId()).isEqualTo(uploadId) - assertThat(SdkHttpUtils.urlDecode(it.key())).isEqualTo(UPLOAD_FILE_NAME) - } - } - - /** - * Tests that an exception is thrown when listing parts if the upload id is unknown. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testListMultipartUploads_throwOnUnknownId(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - - assertThatThrownBy { - s3ClientV2.listParts( - ListPartsRequest - .builder() - .bucket(bucketName) - .key("NON_EXISTENT_KEY") - .uploadId("NON_EXISTENT_UPLOAD_ID") - .build() - ) - } - .isInstanceOf(AwsServiceException::class.java) - .hasMessageContaining("Service: S3, Status Code: 404") - } - - /** - * Tests if not yet completed / aborted multipart uploads are listed with prefix filtering. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testListMultipartUploads_withPrefix(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key("key1") - .build() - ) - s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key("key2") - .build() - ) - val listMultipartUploadsRequest = ListMultipartUploadsRequest.builder().bucket(bucketName).prefix("key2").build() - - val listing = s3ClientV2.listMultipartUploads(listMultipartUploadsRequest) - assertThat(listing.uploads()).hasSize(1) - assertThat(listing.uploads()[0].key()).isEqualTo("key2") - } - - /** - * Tests if multipart uploads are stored and can be retrieved by bucket. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testListMultipartUploads_multipleBuckets(testInfo: TestInfo) { - // create multipart upload 1 - val bucketName1 = givenBucketV2(testInfo) - .also { - s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(it) - .key("key1") - .build() - ) - } - - // create multipart upload 2 - val bucketName2 = givenRandomBucketV1() - .also { - s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(it) - .key("key2") - .build() - ) - } - - // assert multipart upload 1 - ListMultipartUploadsRequest - .builder() - .bucket(bucketName1) - .build() - .also { request -> - s3ClientV2.listMultipartUploads(request) - .also { - assertThat(it.uploads()).hasSize(1) - assertThat(it.uploads()[0].key()).isEqualTo("key1") - } - } - - // assert multipart upload 2 - ListMultipartUploadsRequest - .builder() - .bucket(bucketName2) - .build() - .also { request -> - s3ClientV2.listMultipartUploads(request) - .also { - assertThat(it.uploads()).hasSize(1) - assertThat(it.uploads()[0].key()).isEqualTo("key2") - } - } - } - - /** - * Tests if a multipart upload can be aborted. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testAbortMultipartUpload(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - assertThat( - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(bucketName) - .build() - ).hasUploads() - ).isFalse - - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - val randomBytes = randomBytes() - - val partETag = uploadPart(bucketName, UPLOAD_FILE_NAME, uploadId, 1, randomBytes) - assertThat( - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(bucketName) - .build() - ).hasUploads() - ).isTrue - - s3ClientV2.listParts( - ListPartsRequest.builder().bucket(bucketName).key(UPLOAD_FILE_NAME).uploadId(uploadId) - .build() - ) - .parts() - .also { - assertThat(it).hasSize(1) - assertThat(it[0].eTag()).isEqualTo(partETag) - } - - s3ClientV2.abortMultipartUpload( - AbortMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .uploadId(uploadId) - .build() - ) - assertThat( - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(bucketName) - .build() - ).hasUploads() - ).isFalse - - // List parts, make sure we find no parts - assertThatThrownBy { - s3ClientV2.listParts( - ListPartsRequest - .builder() - .bucket(bucketName) - .key(UPLOAD_FILE_NAME) - .uploadId(uploadId) - .build() - ) - } - .isInstanceOf(AwsServiceException::class.java) - .hasMessageContaining("Service: S3, Status Code: 404") - .asInstanceOf(InstanceOfAssertFactories.type(AwsServiceException::class.java)) - .extracting(AwsServiceException::awsErrorDetails) - .extracting(AwsErrorDetails::errorCode) - .isEqualTo("NoSuchUpload") - } - - /** - * Tests if the parts specified in CompleteUploadRequest are adhered - * irrespective of the number of parts uploaded before. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testCompleteMultipartUpload_partLeftOut(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val key = randomName - assertThat( - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(bucketName) - .build() - ).uploads() - ).isEmpty() - - // Initiate upload - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(key) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - // Upload 3 parts - val randomBytes1 = randomBytes() - val partETag1 = uploadPart(bucketName, key, uploadId, 1, randomBytes1) - val randomBytes2 = randomBytes() - uploadPart(bucketName, key, uploadId, 2, randomBytes2) //ignore output in this test. - val randomBytes3 = randomBytes() - val partETag3 = uploadPart(bucketName, key, uploadId, 3, randomBytes3) - - // Try to complete with these parts - val result = s3ClientV2.completeMultipartUpload( - CompleteMultipartUploadRequest.builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .multipartUpload( - CompletedMultipartUpload - .builder() - .parts( - CompletedPart - .builder() - .eTag(partETag1) - .partNumber(1) - .build(), - CompletedPart - .builder() - .eTag(partETag3) - .partNumber(3) - .build() - ) - .build() - ) - .build() - ) - - // Verify only 1st and 3rd counts - (DigestUtils.md5(randomBytes1) + DigestUtils.md5(randomBytes3)).also { - // verify special etag - assertThat(result.eTag()).isEqualTo("\"${DigestUtils.md5Hex(it)}-2\"") - } - - s3ClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName) - .key(key) - .build() - ).use { - // verify content size - assertThat(it.response().contentLength()).isEqualTo(randomBytes1.size.toLong() + randomBytes3.size) - // verify contents - assertThat(readStreamIntoByteArray(it.buffered())).isEqualTo(concatByteArrays(randomBytes1, randomBytes3)) - } - } - - /** - * Tests that uploaded parts can be listed regardless if the MultipartUpload was completed or - * aborted. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testListParts_completeAndAbort(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val key = randomName - assertThat( - s3ClientV2.listMultipartUploads( - ListMultipartUploadsRequest - .builder() - .bucket(bucketName) - .build() - ) - .uploads() - ).isEmpty() - - // Initiate upload - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(key) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - // Upload part - val randomBytes = randomBytes() - val partETag = uploadPart(bucketName, key, uploadId, 1, randomBytes) - - // List parts, make sure we find part 1 - s3ClientV2.listParts( - ListPartsRequest - .builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .build() - ) - .parts() - .also { - assertThat(it).hasSize(1) - assertThat(it[0].eTag()).isEqualTo(partETag) - } - - // Complete - s3ClientV2.completeMultipartUpload( - CompleteMultipartUploadRequest - .builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .multipartUpload( - CompletedMultipartUpload - .builder() - .parts( - CompletedPart - .builder() - .eTag(partETag) - .partNumber(1) - .build() - ) - .build() - ) - .build() - ) - - // List parts, make sure we find no parts - assertThatThrownBy { - s3ClientV2.listParts( - ListPartsRequest - .builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .build() - ) - } - .isInstanceOf(AwsServiceException::class.java) - .hasMessageContaining("Service: S3, Status Code: 404") - .asInstanceOf(InstanceOfAssertFactories.type(AwsServiceException::class.java)) - .extracting(AwsServiceException::awsErrorDetails) - .extracting(AwsErrorDetails::errorCode) - .isEqualTo("NoSuchUpload") - } - - /** - * Upload two objects, copy as parts without length, complete multipart. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyPartsAndComplete(testInfo: TestInfo) { - //Initiate upload - val bucketName2 = givenRandomBucketV2() - val multipartUploadKey = UUID.randomUUID().toString() - - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(bucketName2) - .key(multipartUploadKey) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - val parts: MutableList = ArrayList() - - //bucket for test data - val bucketName1 = givenBucketV2(testInfo) - - //create two objects, initiate copy part with full object length - val sourceKeys = arrayOf(UUID.randomUUID().toString(), UUID.randomUUID().toString()) - val allRandomBytes: MutableList = ArrayList() - for (i in sourceKeys.indices) { - val key = sourceKeys[i] - val partNumber = i + 1 - val randomBytes = randomBytes() - val metadata1 = HashMap().apply { - this["contentLength"] = randomBytes.size.toString() - } - s3ClientV2.putObject( - PutObjectRequest - .builder() - .bucket(bucketName1) - .key(key) - .metadata(metadata1) - .build(), - RequestBody.fromInputStream(ByteArrayInputStream(randomBytes), randomBytes.size.toLong()) - ) - - s3ClientV2.uploadPartCopy( - UploadPartCopyRequest.builder() - .partNumber(partNumber) - .uploadId(uploadId) - .destinationBucket(bucketName2) - .destinationKey(multipartUploadKey) - .sourceKey(key) - .sourceBucket(bucketName1).build() - ).also { - val etag = it.copyPartResult().eTag() - parts.add(CompletedPart.builder().eTag(etag).partNumber(partNumber).build()) - allRandomBytes.add(randomBytes) - } - } - assertThat(allRandomBytes).hasSize(2) - - // Complete with parts - val result = s3ClientV2.completeMultipartUpload( - CompleteMultipartUploadRequest - .builder() - .bucket(bucketName2) - .key(multipartUploadKey) - .uploadId(uploadId) - .multipartUpload( - CompletedMultipartUpload - .builder() - .parts(parts) - .build() - ) - .build() - ) - - // Verify parts - (DigestUtils.md5(allRandomBytes[0]) + DigestUtils.md5(allRandomBytes[1])).also { - // verify etag - assertThat(result.eTag()).isEqualTo("\"${DigestUtils.md5Hex(it)}-2\"") - } - - s3ClientV2.getObject( - GetObjectRequest - .builder() - .bucket(bucketName2) - .key(multipartUploadKey) - .build() - ).use { - // verify content size - assertThat(it.response().contentLength()).isEqualTo(allRandomBytes[0].size.toLong() + allRandomBytes[1].size) - - // verify contents - assertThat(readStreamIntoByteArray(it.buffered())) - .isEqualTo(concatByteArrays(allRandomBytes[0], allRandomBytes[1])) - } - } - - /** - * Puts an Object; Copies part of that object to a new bucket; - * Requests parts for the uploadId; compares etag of upload response and parts list. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldCopyObjectPart(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val uploadFile = File(sourceKey) - val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucket = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val objectMetadata = mapOf(Pair("key", "value")) - - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(destinationBucket) - .key(destinationKey) - .metadata(objectMetadata) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - val result = s3ClientV2.uploadPartCopy( - UploadPartCopyRequest.builder() - .uploadId(uploadId) - .destinationBucket(destinationBucket) - .destinationKey(destinationKey) - .sourceKey(sourceKey) - .sourceBucket(bucketName) - .partNumber(1) - .copySourceRange("bytes=0-" + (uploadFile.length() - 1)) - .build() - ) - val etag = result.copyPartResult().eTag() - - s3ClientV2.listParts( - ListPartsRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(initiateMultipartUploadResult.uploadId()) - .build() - ).also { - assertThat(it.parts()).hasSize(1) - assertThat(it.parts()[0].eTag()).isEqualTo(etag) - } - } - - /** - * Tries to copy part of a non-existing object to a new bucket. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun shouldThrowNoSuchKeyOnCopyObjectPartForNonExistingKey(testInfo: TestInfo) { - val sourceKey = "NON_EXISTENT_KEY" - val destinationBucket = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val bucketName = givenBucketV2(testInfo) - val objectMetadata = mapOf(Pair("key", "value")) - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(destinationBucket) - .key(destinationKey) - .metadata(objectMetadata) - .build() - ) - - val uploadId = initiateMultipartUploadResult.uploadId() - assertThatThrownBy { - s3ClientV2.uploadPartCopy( - UploadPartCopyRequest.builder() - .uploadId(uploadId) - .destinationBucket(destinationBucket) - .destinationKey(destinationKey) - .sourceKey(sourceKey) - .sourceBucket(bucketName) - .partNumber(1) - .copySourceRange("bytes=0-5") - .build() - ) - } - .isInstanceOf(AwsServiceException::class.java) - .hasMessageContaining("Service: S3, Status Code: 404") - .asInstanceOf(InstanceOfAssertFactories.type(AwsServiceException::class.java)) - .extracting(AwsServiceException::awsErrorDetails) - .extracting(AwsErrorDetails::errorCode) - .isEqualTo("NoSuchKey") - } - - @Test - @S3VerifiedSuccess(year = 2022) - fun testUploadPartCopy_successMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val uploadFile = File(sourceKey) - val (bucketName, putObjectResponse) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucket = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val matchingEtag = putObjectResponse.eTag() - - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(destinationBucket) - .key(destinationKey) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - val result = s3ClientV2.uploadPartCopy( - UploadPartCopyRequest.builder() - .uploadId(uploadId) - .destinationBucket(destinationBucket) - .destinationKey(destinationKey) - .sourceKey(sourceKey) - .sourceBucket(bucketName) - .partNumber(1) - .copySourceRange("bytes=0-" + (uploadFile.length() - 1)) - .copySourceIfMatch(matchingEtag) - .build() - ) - val etag = result.copyPartResult().eTag() - - s3ClientV2.listParts( - ListPartsRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(initiateMultipartUploadResult.uploadId()) - .build() - ).also { - assertThat(it.parts()).hasSize(1) - assertThat(it.parts()[0].eTag()).isEqualTo(etag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testUploadPartCopy_successNoneMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val uploadFile = File(sourceKey) - val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucket = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val noneMatchingEtag = "\"${randomName}\"" - - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(destinationBucket) - .key(destinationKey) - .build() - ) - val uploadId = initiateMultipartUploadResult.uploadId() - - val result = s3ClientV2.uploadPartCopy( - UploadPartCopyRequest.builder() - .uploadId(uploadId) - .destinationBucket(destinationBucket) - .destinationKey(destinationKey) - .sourceKey(sourceKey) - .sourceBucket(bucketName) - .partNumber(1) - .copySourceRange("bytes=0-" + (uploadFile.length() - 1)) - .copySourceIfNoneMatch(noneMatchingEtag) - .build() - ) - val etag = result.copyPartResult().eTag() - - s3ClientV2.listParts( - ListPartsRequest - .builder() - .bucket(initiateMultipartUploadResult.bucket()) - .key(initiateMultipartUploadResult.key()) - .uploadId(initiateMultipartUploadResult.uploadId()) - .build() - ).also { - assertThat(it.parts()).hasSize(1) - assertThat(it.parts()[0].eTag()).isEqualTo(etag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testUploadPartCopy_failureMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val uploadFile = File(sourceKey) - val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucket = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val noneMatchingEtag = "\"${randomName}\"" - - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(destinationBucket) - .key(destinationKey) - .build() - ) - - val uploadId = initiateMultipartUploadResult.uploadId() - assertThatThrownBy { - s3ClientV2.uploadPartCopy( - UploadPartCopyRequest.builder() - .uploadId(uploadId) - .destinationBucket(destinationBucket) - .destinationKey(destinationKey) - .sourceKey(sourceKey) - .sourceBucket(bucketName) - .partNumber(1) - .copySourceRange("bytes=0-" + uploadFile.length()) - .copySourceIfMatch(noneMatchingEtag) - .build() - ) - } - .isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 412") - .hasMessageContaining(PRECONDITION_FAILED.message) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testUploadPartCopy_failureNoneMatch(testInfo: TestInfo) { - val sourceKey = UPLOAD_FILE_NAME - val uploadFile = File(sourceKey) - val (bucketName, putObjectResponse) = givenBucketAndObjectV2(testInfo, sourceKey) - val destinationBucket = givenRandomBucketV2() - val destinationKey = "copyOf/$sourceKey" - val matchingEtag = putObjectResponse.eTag() - - val initiateMultipartUploadResult = s3ClientV2 - .createMultipartUpload( - CreateMultipartUploadRequest - .builder() - .bucket(destinationBucket) - .key(destinationKey) - .build() - ) - - val uploadId = initiateMultipartUploadResult.uploadId() - assertThatThrownBy { - s3ClientV2.uploadPartCopy( - UploadPartCopyRequest.builder() - .uploadId(uploadId) - .destinationBucket(destinationBucket) - .destinationKey(destinationKey) - .sourceKey(sourceKey) - .sourceBucket(bucketName) - .partNumber(1) - .copySourceRange("bytes=0-" + uploadFile.length()) - .copySourceIfNoneMatch(matchingEtag) - .build() - ) - } - .isInstanceOf(S3Exception::class.java) - .hasMessageContaining("Service: S3, Status Code: 412") - .hasMessageContaining(PRECONDITION_FAILED.message) - } - - private fun uploadPart( - bucketName: String, - key: String, - uploadId: String, - partNumber: Int, - randomBytes: ByteArray - ): String { - return s3ClientV2 - .uploadPart( - UploadPartRequest.builder() - .bucket(bucketName) - .key(key) - .uploadId(uploadId) - .partNumber(partNumber) - .contentLength(randomBytes.size.toLong()).build(), - RequestBody.fromInputStream(ByteArrayInputStream(randomBytes), randomBytes.size.toLong()) - ) - .eTag() - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingIT.kt new file mode 100644 index 000000000..08ff4b548 --- /dev/null +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingIT.kt @@ -0,0 +1,122 @@ +/* + * Copyright 2017-2025 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.adobe.testing.s3mock.its + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.Tag +import software.amazon.awssdk.services.s3.model.Tagging + +internal class ObjectTaggingIT : S3TestBase() { + private val s3Client: S3Client = createS3Client() + + @Test + @S3VerifiedSuccess(year = 2025) + fun testGetObjectTagging_noTags(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + s3Client.putObject({ + it.bucket(bucketName) + it.key("foo") + }, + RequestBody.fromString("foo") + ) + + assertThat(s3Client.getObjectTagging { + it.bucket(bucketName) + it.key("foo") + }.tagSet()).isEmpty() + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutAndGetObjectTagging(testInfo: TestInfo) { + val key = UPLOAD_FILE_NAME + val (bucketName, _) = givenBucketAndObject(testInfo, key) + val tag1 = Tag.builder().key("tag1").value("foo").build() + val tag2 = Tag.builder().key("tag2").value("bar").build() + + s3Client.putObjectTagging { + it.bucket(bucketName) + it.key(key) + it.tagging { + it.tagSet(tag1, tag2) + } + } + + assertThat( + s3Client.getObjectTagging { + it.bucket(bucketName) + it.key(key) + }.tagSet() + ).contains( + tag1, + tag2 + ) + } + + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObjectAndGetObjectTagging_withTagging(testInfo: TestInfo) { + val key = UPLOAD_FILE_NAME + val bucketName = givenBucket(testInfo) + + s3Client.putObject({ + it.bucket(bucketName) + it.key(key) + it.tagging("msv=foo") + }, + RequestBody.fromString("foo") + ) + + assertThat( + s3Client.getObjectTagging { + it.bucket(bucketName) + it.key(key) + }.tagSet() + ).contains(Tag.builder().key("msv").value("foo").build()) + } + + /** + * Verify that tagging with multiple tags can be obtained and returns expected content. + */ + @Test + @S3VerifiedSuccess(year = 2025) + fun testPutObjectAndGetObjectTagging_multipleTags(testInfo: TestInfo) { + val bucketName = givenBucket(testInfo) + val tag1 = Tag.builder().key("tag1").value("foo").build() + val tag2 = Tag.builder().key("tag2").value("bar").build() + + s3Client.putObject({ + it.bucket(bucketName) + it.key("multipleFoo") + it.tagging(Tagging.builder().tagSet(tag1, tag2).build()) + }, RequestBody.fromString("multipleFoo") + ) + + assertThat( + s3Client.getObjectTagging { + it.bucket(bucketName) + it.key("multipleFoo") + }.tagSet() + ).contains( + tag1, + tag2 + ) + } +} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingV1IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingV1IT.kt deleted file mode 100644 index cd038c1d6..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingV1IT.kt +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2017-2025 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.GetObjectTaggingRequest -import com.amazonaws.services.s3.model.ObjectTagging -import com.amazonaws.services.s3.model.PutObjectRequest -import com.amazonaws.services.s3.model.SetObjectTaggingRequest -import com.amazonaws.services.s3.model.Tag -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import java.io.File - -/** - * Test the application using the AmazonS3 SDK V1. - */ -@Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") -internal class ObjectTaggingV1IT : S3TestBase() { - val s3Client: AmazonS3 = createS3ClientV1() - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutAndGetObjectTagging(testInfo: TestInfo) { - val (bucketName, _) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val s3Object = s3Client.getObject(bucketName, UPLOAD_FILE_NAME) - - val tag = Tag("foo", "bar") - val tagList: MutableList = mutableListOf(tag) - val setObjectTaggingRequest = SetObjectTaggingRequest(bucketName, s3Object.key, ObjectTagging(tagList)) - s3Client.setObjectTagging(setObjectTaggingRequest) - val getObjectTaggingRequest = GetObjectTaggingRequest(bucketName, s3Object.key) - - s3Client.getObjectTagging(getObjectTaggingRequest).also { - // There should be 'foo:bar' here - assertThat(it.tagSet).hasSize(1) - assertThat(it.tagSet).contains(tag) - } - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObjectAndGetObjectTagging_withTagging(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val putObjectRequest = PutObjectRequest( - bucketName, - uploadFile.name, - uploadFile - ) - .withTagging(ObjectTagging(mutableListOf(Tag("foo", "bar")))) - s3Client.putObject(putObjectRequest) - val s3Object = s3Client.getObject(bucketName, uploadFile.name) - val getObjectTaggingRequest = GetObjectTaggingRequest(bucketName, s3Object.key) - - s3Client.getObjectTagging(getObjectTaggingRequest).also { - // There should be 'foo:bar' here - assertThat(it.tagSet).hasSize(1) - assertThat(it.tagSet[0].value).isEqualTo("bar") - } - } - - /** - * Verify that tagging with multiple tags can be obtained and returns expected content. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObjectAndGetObjectTagging_multipleTags(testInfo: TestInfo) { - val bucketName = givenBucketV1(testInfo) - val uploadFile = File(UPLOAD_FILE_NAME) - val tag1 = Tag("foo1", "bar1") - val tag2 = Tag("foo2", "bar2") - val tagList: MutableList = mutableListOf(tag1, tag2) - val putObjectRequest = PutObjectRequest( - bucketName, - uploadFile.name, - uploadFile - ).withTagging(ObjectTagging(tagList)) - s3Client.putObject(putObjectRequest) - val s3Object = s3Client.getObject(bucketName, uploadFile.name) - val getObjectTaggingRequest = GetObjectTaggingRequest(bucketName, s3Object.key) - - s3Client.getObjectTagging(getObjectTaggingRequest).also { - assertThat(it.tagSet).hasSize(2) - assertThat(it.tagSet).contains(tag1, tag2) - } - } - - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObjectTagging_noTags(testInfo: TestInfo) { - val (bucketName, _) = givenBucketAndObjectV1(testInfo, UPLOAD_FILE_NAME) - val s3Object = s3Client.getObject(bucketName, UPLOAD_FILE_NAME) - val getObjectTaggingRequest = GetObjectTaggingRequest(bucketName, s3Object.key) - - s3Client.getObjectTagging(getObjectTaggingRequest).also { - // There shouldn't be any tags here - assertThat(it.tagSet).isEmpty() - } - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingV2IT.kt deleted file mode 100644 index acc019d06..000000000 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/ObjectTaggingV2IT.kt +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2017-2024 Adobe. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.adobe.testing.s3mock.its - -import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo -import software.amazon.awssdk.core.sync.RequestBody -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest -import software.amazon.awssdk.services.s3.model.PutObjectRequest -import software.amazon.awssdk.services.s3.model.PutObjectTaggingRequest -import software.amazon.awssdk.services.s3.model.Tag -import software.amazon.awssdk.services.s3.model.Tagging - -internal class ObjectTaggingV2IT : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() - - @Test - @S3VerifiedSuccess(year = 2024) - fun testGetObjectTagging_noTags(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - s3ClientV2.putObject( - { b: PutObjectRequest.Builder -> b.bucket(bucketName).key("foo") }, - RequestBody.fromString("foo") - ) - - assertThat(s3ClientV2.getObjectTagging { b: GetObjectTaggingRequest.Builder -> - b.bucket( - bucketName - ).key("foo") - } - .tagSet()) - .isEmpty() - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutAndGetObjectTagging(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val key = "foo" - val tag1 = Tag.builder().key("tag1").value("foo").build() - val tag2 = Tag.builder().key("tag2").value("bar").build() - s3ClientV2.putObject( - { b: PutObjectRequest.Builder -> b.bucket(bucketName).key(key) }, - RequestBody.fromString("foo") - ) - - s3ClientV2.putObjectTagging( - PutObjectTaggingRequest.builder().bucket(bucketName).key(key) - .tagging(Tagging.builder().tagSet(tag1, tag2).build()).build() - ) - - assertThat(s3ClientV2.getObjectTagging { b: GetObjectTaggingRequest.Builder -> - b.bucket( - bucketName - ).key(key) - } - .tagSet()) - .contains( - tag1, - tag2 - ) - } - - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObjectAndGetObjectTagging_withTagging(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - s3ClientV2.putObject( - { b: PutObjectRequest.Builder -> b.bucket(bucketName).key("foo").tagging("msv=foo") }, - RequestBody.fromString("foo") - ) - - assertThat(s3ClientV2.getObjectTagging { b: GetObjectTaggingRequest.Builder -> - b.bucket( - bucketName - ).key("foo") - } - .tagSet()) - .contains(Tag.builder().key("msv").value("foo").build()) - } - - /** - * Verify that tagging with multiple tags can be obtained and returns expected content. - */ - @Test - @S3VerifiedSuccess(year = 2024) - fun testPutObjectAndGetObjectTagging_multipleTags(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - val tag1 = Tag.builder().key("tag1").value("foo").build() - val tag2 = Tag.builder().key("tag2").value("bar").build() - - s3ClientV2.putObject( - { b: PutObjectRequest.Builder -> - b.bucket(bucketName).key("multipleFoo") - .tagging(Tagging.builder().tagSet(tag1, tag2).build()) - }, RequestBody.fromString("multipleFoo") - ) - - assertThat(s3ClientV2.getObjectTagging { b: GetObjectTaggingRequest.Builder -> - b.bucket( - bucketName - ).key("multipleFoo") - } - .tagSet()) - .contains( - tag1, - tag2 - ) - } -} diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PlainHttpIT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PlainHttpIT.kt index fe8932633..88d3ac02d 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PlainHttpIT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PlainHttpIT.kt @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Adobe. + * Copyright 2017-2025 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,10 +15,6 @@ */ package com.adobe.testing.s3mock.its -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest -import com.amazonaws.services.s3.model.ObjectMetadata -import com.amazonaws.services.s3.model.UploadPartRequest import org.apache.http.HttpHeaders import org.apache.http.HttpHost import org.apache.http.HttpStatus @@ -38,8 +34,9 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInfo import org.springframework.http.MediaType +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.utils.http.SdkHttpUtils -import java.io.ByteArrayInputStream import java.io.File import java.io.InputStreamReader import java.util.UUID @@ -51,17 +48,18 @@ import java.util.stream.Collectors */ internal class PlainHttpIT : S3TestBase() { private val httpClient: CloseableHttpClient = createHttpClient() - private val s3Client: AmazonS3 = createS3ClientV1() + private val s3Client: S3Client = createS3Client() @Test @S3VerifiedFailure(year = 2022, reason = "No credentials sent in plain HTTP request") fun putObjectReturns200(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) val byteArray = UUID.randomUUID().toString().toByteArray() val putObject = HttpPut("$serviceEndpoint/$targetBucket/testObjectName").apply { this.entity = ByteArrayEntity(byteArray) this.addHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE) + this.params } httpClient.execute(putObject).use { @@ -73,7 +71,7 @@ internal class PlainHttpIT : S3TestBase() { @S3VerifiedFailure(year = 2022, reason = "No credentials sent in plain HTTP request") fun testGetObject_withAcceptHeader(testInfo: TestInfo) { - val (targetBucket, _) = givenBucketAndObjectV2(testInfo, UPLOAD_FILE_NAME) + val (targetBucket, _) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) val getObject = HttpGet("$serviceEndpoint/$targetBucket/$UPLOAD_FILE_NAME").apply { this.addHeader(HttpHeaders.ACCEPT, MediaType.TEXT_PLAIN_VALUE) @@ -88,7 +86,7 @@ internal class PlainHttpIT : S3TestBase() { @S3VerifiedFailure(year = 2023, reason = "No credentials sent in plain HTTP request") fun putHeadObject_withUserMetadata(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) val byteArray = UUID.randomUUID().toString().toByteArray() val amzMetaHeaderKey = "x-amz-meta-my-key" val amzMetaHeaderValue = "MY_DATA" @@ -136,7 +134,7 @@ internal class PlainHttpIT : S3TestBase() { @S3VerifiedFailure(year = 2022, reason = "No credentials sent in plain HTTP request") fun putObjectEncryptedWithAbsentKeyRef(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) HttpPut("$serviceEndpoint/$targetBucket/testObjectName").apply { this.addHeader("x-amz-server-side-encryption", "aws:kms") @@ -152,10 +150,9 @@ internal class PlainHttpIT : S3TestBase() { @S3VerifiedFailure(year = 2022, reason = "No credentials sent in plain HTTP request") fun listWithPrefixAndMissingSlash(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) - s3Client.putObject(targetBucket, "prefix", "Test") + val (targetBucket, _) = givenBucketAndObject(testInfo, UPLOAD_FILE_NAME) - HttpGet("$serviceEndpoint/$targetBucket?prefix=prefix%2F&encoding-type=url").also { + HttpGet("$serviceEndpoint/$targetBucket?prefix=${UPLOAD_FILE_NAME}%2F&encoding-type=url").also { httpClient.execute(it).use { response -> assertThat(response.statusLine.statusCode).isEqualTo(HttpStatus.SC_OK) } @@ -166,7 +163,7 @@ internal class PlainHttpIT : S3TestBase() { @Test @S3VerifiedSuccess(year = 2022) fun objectUsesApplicationXmlContentType(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) HttpGet("$serviceEndpoint/$targetBucket").also { assertApplicationXmlContentType(it) @@ -174,8 +171,10 @@ internal class PlainHttpIT : S3TestBase() { } @Test + @S3VerifiedFailure(year = 2022, + reason = "No credentials sent in plain HTTP request") fun testCorsHeaders_GET_PUT_HEAD(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) arrayOf("GET", "PUT", "HEAD").forEach { method -> val httpOptions = HttpOptions("$serviceEndpoint/$targetBucket").apply { @@ -197,8 +196,10 @@ internal class PlainHttpIT : S3TestBase() { } @Test + @S3VerifiedFailure(year = 2022, + reason = "No credentials sent in plain HTTP request") fun testCorsHeaders_POST(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) arrayOf("POST").forEach { method -> val httpOptions = HttpOptions("$serviceEndpoint/$targetBucket?delete").apply { @@ -224,16 +225,17 @@ internal class PlainHttpIT : S3TestBase() { @Test @S3VerifiedSuccess(year = 2022) fun listBucketsUsesApplicationXmlContentType(testInfo: TestInfo) { - givenBucketV2(testInfo) + givenBucket(testInfo) HttpGet("$serviceEndpoint$SLASH").also { assertApplicationXmlContentType(it) } } @Test - @S3VerifiedSuccess(year = 2022) + @S3VerifiedFailure(year = 2022, + reason = "No credentials sent in plain HTTP request") fun batchDeleteUsesApplicationXmlContentType(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) HttpPost("$serviceEndpoint/$targetBucket?delete").apply { this.entity = StringEntity( @@ -249,33 +251,36 @@ internal class PlainHttpIT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2022) + @S3VerifiedFailure(year = 2022, + reason = "No credentials sent in plain HTTP request") fun completeMultipartUsesApplicationXmlContentType(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) val uploadFile = File(UPLOAD_FILE_NAME) val initiateMultipartUploadResult = s3Client - .initiateMultipartUpload( - InitiateMultipartUploadRequest(targetBucket, UPLOAD_FILE_NAME) - ) - val uploadId = initiateMultipartUploadResult.uploadId + .createMultipartUpload { + it.bucket(targetBucket) + it.key(UPLOAD_FILE_NAME) + } + val uploadId = initiateMultipartUploadResult.uploadId() + val uploadPartResult = s3Client.uploadPart( - UploadPartRequest() - .withBucketName(initiateMultipartUploadResult.bucketName) - .withKey(initiateMultipartUploadResult.key) - .withUploadId(uploadId) - .withFile(uploadFile) - .withFileOffset(0) - .withPartNumber(1) - .withPartSize(uploadFile.length()) - .withLastPart(true) + { + it.bucket(initiateMultipartUploadResult.bucket()) + it.key(initiateMultipartUploadResult.key()) + it.uploadId(uploadId) + it.partNumber(1) + it.contentLength(uploadFile.length()) + }, + RequestBody.fromFile(uploadFile) ) + HttpPost("$serviceEndpoint/$targetBucket/$UPLOAD_FILE_NAME?uploadId=$uploadId").apply { this.entity = StringEntity( """ - ${uploadPartResult.partETag.eTag} + ${uploadPartResult.eTag()} 1 """.trimMargin(), @@ -292,7 +297,7 @@ internal class PlainHttpIT : S3TestBase() { fun putObjectWithSpecialCharactersInTheName(testInfo: TestInfo) { val fileNameWithSpecialCharacters = ("file=name\$Dollar;Semicolon" + "&Ampersand@At:Colon Space,Comma?Question-mark") - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) HttpPut( "$serviceEndpoint/$targetBucket/${SdkHttpUtils.urlEncodeIgnoreSlashes(fileNameWithSpecialCharacters)}" @@ -305,10 +310,10 @@ internal class PlainHttpIT : S3TestBase() { } assertThat( - s3Client - .listObjects(targetBucket) - .objectSummaries[0] - .key + s3Client.listObjects { + it.bucket(targetBucket) + it.prefix(fileNameWithSpecialCharacters) + }.contents()[0].key() ).isEqualTo(fileNameWithSpecialCharacters) } @@ -316,7 +321,7 @@ internal class PlainHttpIT : S3TestBase() { @S3VerifiedFailure(year = 2022, reason = "No credentials sent in plain HTTP request") fun deleteNonExistingObjectReturns204(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) HttpDelete("$serviceEndpoint/$targetBucket/${UUID.randomUUID()}").also { httpClient.execute(it).use { response -> @@ -330,7 +335,7 @@ internal class PlainHttpIT : S3TestBase() { @S3VerifiedFailure(year = 2022, reason = "No credentials sent in plain HTTP request") fun batchDeleteObjects(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) HttpPost("$serviceEndpoint/$targetBucket?delete").apply { this.entity = StringEntity( @@ -353,16 +358,16 @@ internal class PlainHttpIT : S3TestBase() { @S3VerifiedFailure(year = 2022, reason = "No credentials sent in plain HTTP request") fun headObjectWithUnknownContentType(testInfo: TestInfo) { - val targetBucket = givenBucketV2(testInfo) + val targetBucket = givenBucket(testInfo) val contentAsBytes = ByteArray(0) - val md = ObjectMetadata().apply { - this.contentLength = contentAsBytes.size.toLong() - this.contentType = UUID.randomUUID().toString() - } val blankContentTypeFilename = UUID.randomUUID().toString() - s3Client.putObject( - targetBucket, blankContentTypeFilename, - ByteArrayInputStream(contentAsBytes), md + s3Client.putObject({ + it.bucket(targetBucket) + it.key(blankContentTypeFilename) + it.contentType(UUID.randomUUID().toString()) + it.contentLength(contentAsBytes.size.toLong()) + }, + RequestBody.fromBytes(contentAsBytes) ) HttpHead("$serviceEndpoint/$targetBucket/$blankContentTypeFilename").also { diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PresignedUrlV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PresignedUrlIT.kt similarity index 74% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PresignedUrlV2IT.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PresignedUrlIT.kt index 0f97adc63..e55576957 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PresignedUrlV2IT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/PresignedUrlIT.kt @@ -38,17 +38,18 @@ import java.io.File import java.nio.file.Files import java.nio.file.Path import java.time.Duration +import java.time.Instant -internal class PresignedUrlV2IT : S3TestBase() { +internal class PresignedUrlIT : S3TestBase() { private val httpClient: CloseableHttpClient = createHttpClient() - private val s3ClientV2: S3Client = createS3ClientV2() + private val s3Client: S3Client = createS3Client() private val s3Presigner: S3Presigner = createS3Presigner() @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testPresignedUrl_getObject(testInfo: TestInfo) { val key = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV2(testInfo, key) + val (bucketName, _) = givenBucketAndObject(testInfo, key) val presignedUrlString = s3Presigner.presignGetObject { it.getObjectRequest { @@ -73,10 +74,53 @@ internal class PresignedUrlV2IT : S3TestBase() { } @Test - @S3VerifiedTodo + @S3VerifiedSuccess(year = 2025) + fun testPresignedUrl_getObject_responseHeaderOverrides(testInfo: TestInfo) { + val key = UPLOAD_FILE_NAME + val (bucketName, _) = givenBucketAndObject(testInfo, key) + + val responseExpires = Instant.now() + + val presignedUrlString = s3Presigner.presignGetObject { + it.getObjectRequest { + it.bucket(bucketName) + it.key(key) + it.responseExpires(responseExpires) + it.responseCacheControl("no-cache") + it.responseContentDisposition("attachment; filename=\"$key\"") + it.responseContentEncoding("encoding") + it.responseContentType("application/json") + it.responseContentLanguage("en") + } + it.signatureDuration(Duration.ofMinutes(1L)) + }.url().toString() + + assertThat(presignedUrlString).isNotBlank() + + HttpGet(presignedUrlString).also { get -> + httpClient.execute( + get + ).use { + assertThat(it.statusLine.statusCode).isEqualTo(HttpStatus.SC_OK) + val expectedEtag = "\"${DigestUtil.hexDigest(Files.newInputStream(Path.of(UPLOAD_FILE_NAME)))}\"" + val actualEtag = "\"${DigestUtil.hexDigest(it.entity.content)}\"" + assertThat(actualEtag).isEqualTo(expectedEtag) + //TODO: S3 SDK serializes date as 'Sun, 20 Apr 2025 22:07:04 GMT' + //assertThat(it.getFirstHeader(HttpHeaders.EXPIRES).value).isEqualTo(responseExpires) + assertThat(it.getFirstHeader(HttpHeaders.CACHE_CONTROL).value).isEqualTo("no-cache") + assertThat(it.getFirstHeader("Content-Disposition").value).isEqualTo("attachment; filename=\"$key\"") + assertThat(it.getFirstHeader(HttpHeaders.CONTENT_ENCODING).value).isEqualTo("encoding") + assertThat(it.getFirstHeader(HttpHeaders.CONTENT_TYPE).value).isEqualTo("application/json") + assertThat(it.getFirstHeader(HttpHeaders.CONTENT_LANGUAGE).value).isEqualTo("en") + } + } + } + + @Test + @S3VerifiedSuccess(year = 2025) fun testPresignedUrl_getObject_range(testInfo: TestInfo) { val key = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV2(testInfo, key) + val (bucketName, _) = givenBucketAndObject(testInfo, key) val presignedUrlString = s3Presigner.presignGetObject { it.getObjectRequest{ @@ -101,13 +145,13 @@ internal class PresignedUrlV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testPresignedUrl_putObject(testInfo: TestInfo) { val key = UPLOAD_FILE_NAME - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) - val presignedUrlString = s3Presigner.presignGetObject { - it.getObjectRequest{ + val presignedUrlString = s3Presigner.presignPutObject { + it.putObjectRequest { it.bucket(bucketName) it.key(key) } @@ -126,7 +170,7 @@ internal class PresignedUrlV2IT : S3TestBase() { } } - s3ClientV2.getObject { + s3Client.getObject { it.bucket(bucketName) it.key(key) }.use { @@ -137,10 +181,10 @@ internal class PresignedUrlV2IT : S3TestBase() { } @Test - @S3VerifiedFailure(year = 2024, reason = "S3 returns no multipart uploads.") + @S3VerifiedSuccess(year = 2025) fun testPresignedUrl_createMultipartUpload(testInfo: TestInfo) { val key = UPLOAD_FILE_NAME - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) val presignedUrlString = s3Presigner.presignCreateMultipartUpload { it.createMultipartUploadRequest{ @@ -164,29 +208,28 @@ internal class PresignedUrlV2IT : S3TestBase() { }.uploadId } - s3ClientV2.listMultipartUploads { + s3Client.listMultipartUploads { it.bucket(bucketName) - it.keyMarker(key) - it.uploadIdMarker(uploadId) }.also { assertThat(it.uploads()).hasSize(1) + assertThat(it.uploads()[0].uploadId()).isEqualTo(uploadId) } } @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testPresignedUrl_abortMultipartUpload(testInfo: TestInfo) { val key = UPLOAD_FILE_NAME - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) val file = File(UPLOAD_FILE_NAME) - val createMultipartUpload = s3ClientV2.createMultipartUpload { + val createMultipartUpload = s3Client.createMultipartUpload { it.bucket(bucketName) it.key(key) } val uploadId = createMultipartUpload.uploadId() - s3ClientV2.uploadPart( + s3Client.uploadPart( { it.bucket(createMultipartUpload.bucket()) it.key(createMultipartUpload.key()) @@ -215,7 +258,7 @@ internal class PresignedUrlV2IT : S3TestBase() { } } - s3ClientV2.listMultipartUploads { + s3Client.listMultipartUploads { it.bucket(bucketName) it.keyMarker(key) }.also { @@ -224,19 +267,19 @@ internal class PresignedUrlV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testPresignedUrl_completeMultipartUpload(testInfo: TestInfo) { val key = UPLOAD_FILE_NAME - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) val file = File(UPLOAD_FILE_NAME) - val createMultipartUpload = s3ClientV2.createMultipartUpload { + val createMultipartUpload = s3Client.createMultipartUpload { it.bucket(bucketName) it.key(key) } val uploadId = createMultipartUpload.uploadId() - val uploadPartResult = s3ClientV2.uploadPart( + val uploadPartResult = s3Client.uploadPart( { it.bucket(createMultipartUpload.bucket()) it.key(createMultipartUpload.key()) @@ -276,7 +319,7 @@ internal class PresignedUrlV2IT : S3TestBase() { } } - s3ClientV2.listMultipartUploads { + s3Client.listMultipartUploads { it.bucket(bucketName) it.keyMarker(key) }.also { @@ -286,13 +329,13 @@ internal class PresignedUrlV2IT : S3TestBase() { @Test - @S3VerifiedSuccess(year = 2024) + @S3VerifiedSuccess(year = 2025) fun testPresignedUrl_uploadPart(testInfo: TestInfo) { val key = UPLOAD_FILE_NAME - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) val file = File(UPLOAD_FILE_NAME) - val createMultipartUpload = s3ClientV2.createMultipartUpload { + val createMultipartUpload = s3Client.createMultipartUpload { it.bucket(bucketName) it.key(key) } @@ -319,7 +362,7 @@ internal class PresignedUrlV2IT : S3TestBase() { put ).use { response -> assertThat(response.statusLine.statusCode).isEqualTo(HttpStatus.SC_OK) - s3ClientV2.completeMultipartUpload { + s3Client.completeMultipartUpload { it.bucket(bucketName) it.key(key) it.uploadId(uploadId) @@ -336,7 +379,7 @@ internal class PresignedUrlV2IT : S3TestBase() { } } - s3ClientV2.listMultipartUploads { + s3Client.listMultipartUploads { it.bucket(bucketName) it.keyMarker(key) }.also { diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/RetentionV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/RetentionIT.kt similarity index 87% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/RetentionV2IT.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/RetentionIT.kt index 27ccacccd..418a37f29 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/RetentionV2IT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/RetentionIT.kt @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Adobe. + * Copyright 2017-2025 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ package com.adobe.testing.s3mock.its -import org.assertj.core.api.Assertions import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatThrownBy import org.assertj.core.api.Assertions.within @@ -36,17 +35,17 @@ import java.time.Instant import java.time.temporal.ChronoUnit.DAYS import java.time.temporal.ChronoUnit.MILLIS -internal class RetentionV2IT : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() +internal class RetentionIT : S3TestBase() { + private val s3Client: S3Client = createS3Client() @Test - @S3VerifiedSuccess(year = 2022) + @S3VerifiedSuccess(year = 2025) fun testGetRetentionNoBucketLockConfiguration(testInfo: TestInfo) { val sourceKey = UPLOAD_FILE_NAME - val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey) + val (bucketName, _) = givenBucketAndObject(testInfo, sourceKey) assertThatThrownBy { - s3ClientV2.getObjectRetention( + s3Client.getObjectRetention( GetObjectRetentionRequest .builder() .bucket(bucketName) @@ -59,19 +58,19 @@ internal class RetentionV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2022) + @S3VerifiedSuccess(year = 2025) fun testGetRetentionNoObjectLockConfiguration(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) val sourceKey = UPLOAD_FILE_NAME val bucketName = bucketName(testInfo) - s3ClientV2.createBucket( + s3Client.createBucket( CreateBucketRequest .builder() .bucket(bucketName) .objectLockEnabledForBucket(true) .build() ) - s3ClientV2.putObject( + s3Client.putObject( PutObjectRequest .builder() .bucket(bucketName) @@ -81,7 +80,7 @@ internal class RetentionV2IT : S3TestBase() { ) assertThatThrownBy { - s3ClientV2.getObjectRetention( + s3Client.getObjectRetention( GetObjectRetentionRequest .builder() .bucket(bucketName) @@ -94,19 +93,20 @@ internal class RetentionV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2022) + @S3VerifiedFailure(year = 2025, + reason = "S3 Object Lock makes it impossible to delete the object until the retention period is over.") fun testPutAndGetRetention(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) val sourceKey = UPLOAD_FILE_NAME val bucketName = bucketName(testInfo) - s3ClientV2.createBucket( + s3Client.createBucket( CreateBucketRequest .builder() .bucket(bucketName) .objectLockEnabledForBucket(true) .build() ) - s3ClientV2.putObject( + s3Client.putObject( PutObjectRequest .builder() .bucket(bucketName) @@ -116,7 +116,7 @@ internal class RetentionV2IT : S3TestBase() { ) val retainUntilDate = Instant.now().plus(1, DAYS) - s3ClientV2.putObjectRetention( + s3Client.putObjectRetention( PutObjectRetentionRequest .builder() .bucket(bucketName) @@ -130,7 +130,7 @@ internal class RetentionV2IT : S3TestBase() { .build() ) - s3ClientV2.getObjectRetention( + s3Client.getObjectRetention( GetObjectRetentionRequest .builder() .bucket(bucketName) @@ -147,19 +147,19 @@ internal class RetentionV2IT : S3TestBase() { } @Test - @S3VerifiedSuccess(year = 2022) + @S3VerifiedSuccess(year = 2025) fun testPutInvalidRetentionUntilDate(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) val sourceKey = UPLOAD_FILE_NAME val bucketName = bucketName(testInfo) - s3ClientV2.createBucket( + s3Client.createBucket( CreateBucketRequest .builder() .bucket(bucketName) .objectLockEnabledForBucket(true) .build() ) - s3ClientV2.putObject( + s3Client.putObject( PutObjectRequest .builder() .bucket(bucketName) @@ -170,7 +170,7 @@ internal class RetentionV2IT : S3TestBase() { val invalidRetainUntilDate = Instant.now().minus(1, DAYS) assertThatThrownBy { - s3ClientV2.putObjectRetention( + s3Client.putObjectRetention( PutObjectRetentionRequest .builder() .bucket(bucketName) diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt index 703265379..09ade8396 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt @@ -15,21 +15,8 @@ */ package com.adobe.testing.s3mock.its -import com.adobe.testing.s3mock.util.DigestUtil -import com.amazonaws.ClientConfiguration -import com.amazonaws.auth.AWSStaticCredentialsProvider -import com.amazonaws.auth.BasicAWSCredentials -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.AmazonS3ClientBuilder -import com.amazonaws.services.s3.internal.Constants.MB -import com.amazonaws.services.s3.model.PutObjectRequest -import com.amazonaws.services.s3.model.PutObjectResult -import com.amazonaws.services.s3.transfer.TransferManager -import com.amazonaws.services.s3.transfer.TransferManagerBuilder import com.fasterxml.jackson.dataformat.xml.XmlMapper -import org.apache.http.conn.ssl.NoopHostnameVerifier -import org.apache.http.conn.ssl.SSLConnectionSocketFactory +import org.apache.http.client.config.RequestConfig import org.apache.http.impl.client.CloseableHttpClient import org.apache.http.impl.client.HttpClientBuilder import org.assertj.core.api.Assertions.assertThat @@ -57,7 +44,6 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectResponse import software.amazon.awssdk.services.s3.model.EncodingType import software.amazon.awssdk.services.s3.model.GetObjectAttributesResponse import software.amazon.awssdk.services.s3.model.GetObjectResponse -import software.amazon.awssdk.services.s3.model.HeadBucketRequest import software.amazon.awssdk.services.s3.model.HeadObjectResponse import software.amazon.awssdk.services.s3.model.ObjectLockEnabled import software.amazon.awssdk.services.s3.model.ObjectLockLegalHoldStatus @@ -66,14 +52,12 @@ import software.amazon.awssdk.services.s3.model.S3Exception import software.amazon.awssdk.services.s3.model.S3Response import software.amazon.awssdk.services.s3.model.StorageClass import software.amazon.awssdk.services.s3.model.UploadPartResponse -import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration import software.amazon.awssdk.services.s3.presigner.S3Presigner import software.amazon.awssdk.transfer.s3.S3TransferManager import software.amazon.awssdk.utils.AttributeMap import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.File -import java.io.FileInputStream import java.io.IOException import java.io.InputStream import java.net.Socket @@ -85,8 +69,6 @@ import java.security.cert.X509Certificate import java.time.Duration import java.time.Instant import java.util.UUID -import java.util.concurrent.Executors -import java.util.concurrent.ThreadFactory import java.util.stream.Stream import javax.net.ssl.SSLContext import javax.net.ssl.SSLEngine @@ -99,60 +81,26 @@ import kotlin.random.Random * Base type for S3 Mock integration tests. Sets up S3 Client, Certificates, initial Buckets, etc. */ internal abstract class S3TestBase { - private val _s3Client: AmazonS3 = createS3ClientV1() - private val _s3ClientV2: S3Client = createS3ClientV2() + private val _s3Client: S3Client = createS3Client() protected fun createHttpClient(): CloseableHttpClient { return HttpClientBuilder .create() .setSSLContext(createBlindlyTrustingSslContext()) + .setDefaultRequestConfig(RequestConfig.custom().setExpectContinueEnabled(true).build()) .build() } - @Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") - protected fun createS3ClientV1(endpoint: String = serviceEndpoint): AmazonS3 { - return defaultTestAmazonS3ClientBuilder(endpoint).build() - } - - @Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") - protected fun defaultTestAmazonS3ClientBuilder(endpoint: String = serviceEndpoint): AmazonS3ClientBuilder { - return AmazonS3ClientBuilder.standard() - .withCredentials(AWSStaticCredentialsProvider(BasicAWSCredentials(s3AccessKeyId, s3SecretAccessKey))) - .withClientConfiguration(ignoringInvalidSslCertificates(ClientConfiguration())) - .withEndpointConfiguration( - EndpointConfiguration(endpoint, s3Region) - ) - .enablePathStyleAccess() - } - - @Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") - protected fun createTransferManagerV1(endpoint: String = serviceEndpoint, - s3Client: AmazonS3 = createS3ClientV1(endpoint)): TransferManager { - val threadFactory: ThreadFactory = object : ThreadFactory { - private var threadCount = 1 - override fun newThread(r: Runnable): Thread { - val thread = Thread(r) - thread.name = "s3-transfer-${threadCount++}" - return thread - } - } - return TransferManagerBuilder - .standard() - .withS3Client(s3Client) - .withExecutorFactory { Executors.newFixedThreadPool(THREAD_COUNT, threadFactory) } - .build() - } - - protected fun createS3ClientV2(endpoint: String = serviceEndpoint): S3Client { + protected fun createS3Client(endpoint: String = serviceEndpoint, chunkedEncodingEnabled: Boolean? = null): S3Client { return S3Client.builder() .region(Region.of(s3Region)) .credentialsProvider( StaticCredentialsProvider.create(AwsBasicCredentials.create(s3AccessKeyId, s3SecretAccessKey)) ) - .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .serviceConfiguration { + it.pathStyleAccessEnabled(true) + it.chunkedEncodingEnabled(chunkedEncodingEnabled) + } .endpointOverride(URI.create(endpoint)) .httpClient( ApacheHttpClient.builder().buildWithDefaults( @@ -176,7 +124,7 @@ internal abstract class S3TestBase { } } - protected fun createS3AsyncClientV2(endpoint: String = serviceEndpoint): S3AsyncClient { + protected fun createS3AsyncClient(endpoint: String = serviceEndpoint): S3AsyncClient { return S3AsyncClient.builder() .region(Region.of(s3Region)) .credentialsProvider( @@ -195,15 +143,11 @@ internal abstract class S3TestBase { ) ) .multipartEnabled(true) - .multipartConfiguration(MultipartConfiguration - .builder() - .thresholdInBytes((8 * MB).toLong()) - .build()) .build() } - protected fun createTransferManagerV2(endpoint: String = serviceEndpoint, - s3AsyncClient: S3AsyncClient = createAutoS3CrtAsyncClientV2(endpoint)): S3TransferManager { + protected fun createTransferManager(endpoint: String = serviceEndpoint, + s3AsyncClient: S3AsyncClient = createAutoS3CrtAsyncClient(endpoint)): S3TransferManager { return S3TransferManager.builder() .s3Client(s3AsyncClient) .build() @@ -212,7 +156,7 @@ internal abstract class S3TestBase { /** * Uses manual CRT client setup through AwsCrtAsyncHttpClient.builder() */ - protected fun createS3CrtAsyncClientV2(endpoint: String = serviceEndpoint): S3AsyncClient { + protected fun createS3CrtAsyncClient(endpoint: String = serviceEndpoint): S3AsyncClient { return S3AsyncClient.builder() .region(Region.of(s3Region)) .credentialsProvider( @@ -231,16 +175,13 @@ internal abstract class S3TestBase { ) ) .multipartEnabled(true) - .multipartConfiguration(MultipartConfiguration.builder() - .thresholdInBytes((8 * MB).toLong()) - .build()) .build() } /** * Uses automated CRT client setup through S3AsyncClient.crtBuilder() */ - protected fun createAutoS3CrtAsyncClientV2(endpoint: String = serviceEndpoint): S3CrtAsyncClient { + protected fun createAutoS3CrtAsyncClient(endpoint: String = serviceEndpoint): S3CrtAsyncClient { //using S3AsyncClient.crtBuilder does not work, can't get it to ignore custom SSL certificates. return S3AsyncClient.crtBuilder() .httpConfiguration { @@ -254,10 +195,6 @@ internal abstract class S3TestBase { .forcePathStyle(true) //set endpoint to http(!) .endpointOverride(URI.create(endpoint)) - .targetThroughputInGbps(20.0) - .minimumPartSizeInBytes((8 * MB).toLong()) - //S3Mock currently does not support checksum validation. See #1123 - .checksumValidationEnabled(false) .build() as S3CrtAsyncClient } @@ -277,7 +214,8 @@ internal abstract class S3TestBase { */ @AfterEach fun cleanupStores() { - for (bucket in _s3ClientV2.listBuckets().buckets()) { + for (bucket in _s3Client.listBuckets().buckets()) { + if(bucket.name() == "testputandgetretention-545488000") {return} //Empty all buckets deleteMultipartUploads(bucket) deleteObjectsInBucket(bucket, isObjectLockEnabled(bucket)) @@ -290,123 +228,102 @@ internal abstract class S3TestBase { protected fun bucketName(testInfo: TestInfo): String { val normalizedName = testInfo.testMethod.get().name.let { - it.lowercase().replace('_', '-').let { - if (it.length > 50) { - //max bucket name length is 63, shorten name to 50 since we add the timestamp below. - it.substring(0,50) - } else { - it + it.lowercase() + .replace('_', '-') + .replace(' ', '-') + .replace(',', '-') + .replace('\'', '-') + .let { + if (it.length > 50) { + //max bucket name length is 63, shorten name to 50 since we add the timestamp below. + it.substring(0, 50) + } else { + it + } } - } } val bucketName = "$normalizedName-${Instant.now().nano}" LOG.info("Bucketname=$bucketName") return bucketName } - @Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") - fun givenBucketV1(testInfo: TestInfo): String { + fun givenBucket(testInfo: TestInfo): String { val bucketName = bucketName(testInfo) - return givenBucketV1(bucketName) + return givenBucket(bucketName) } - @Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") - private fun givenBucketV1(bucketName: String): String { - _s3Client.createBucket(bucketName) + fun givenBucket(bucketName: String = randomName): String { + _s3Client.createBucket { it.bucket(bucketName) } + val bucketCreated = _s3Client.waiter().waitUntilBucketExists { it.bucket(bucketName) } + val bucketCreatedResponse = bucketCreated.matched().response().get() + assertThat(bucketCreatedResponse).isNotNull return bucketName } - @Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") - fun givenRandomBucketV1(): String { - return givenBucketV1(randomName) - } - - @Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") - private fun givenObjectV1(bucketName: String, key: String): PutObjectResult { - val uploadFile = File(key) - return _s3Client.putObject(PutObjectRequest(bucketName, key, uploadFile)) - } - - @Deprecated("* AWS has deprecated SDK for Java v1, and will remove support EOY 2025.\n" + - " * S3Mock will remove usage of Java v1 early 2026.") - fun givenBucketAndObjectV1(testInfo: TestInfo, key: String): Pair { - val bucketName = givenBucketV1(testInfo) - val putObjectResult = givenObjectV1(bucketName, key) - return Pair(bucketName, putObjectResult) - } - - fun givenBucketV2(testInfo: TestInfo): String { - val bucketName = bucketName(testInfo) - return givenBucketV2(bucketName) - } - - fun givenBucketV2(bucketName: String): String { - _s3ClientV2.createBucket { it.bucket(bucketName) } - return bucketName - } - - fun givenRandomBucketV2(): String { - return givenBucketV2(randomName) - } - - fun givenObjectV2(bucketName: String, key: String): PutObjectResponse { - val uploadFile = File(key) - return _s3ClientV2.putObject({ + fun givenObject(bucketName: String, key: String, fileName: String? = null): PutObjectResponse { + val uploadFile = File(fileName ?: key) + return _s3Client.putObject({ it.bucket(bucketName) it.key(key) }, RequestBody.fromFile(uploadFile) ) } - fun deleteObjectV2(bucketName: String, key: String): DeleteObjectResponse { - return _s3ClientV2.deleteObject { + fun deleteObject(bucketName: String, key: String): DeleteObjectResponse { + return _s3Client.deleteObject { it.bucket(bucketName) it.key(key) } } - fun getObjectV2(bucketName: String, key: String): ResponseInputStream { - return _s3ClientV2.getObject { + fun getObject(bucketName: String, key: String): ResponseInputStream { + return _s3Client.getObject { it.bucket(bucketName) it.key(key) } } - fun givenBucketAndObjectV2(testInfo: TestInfo, key: String): Pair { - val bucketName = givenBucketV2(testInfo) - val putObjectResponse = givenObjectV2(bucketName, key) - return Pair(bucketName, putObjectResponse) + fun givenBucketAndObject(testInfo: TestInfo, key: String): Pair { + val bucketName = givenBucket(testInfo) + val putObjectResponse = givenObject(bucketName, key) + return bucketName to putObjectResponse + } + + fun givenBucketAndObjects(testInfo: TestInfo, count: Int): Pair> { + val keys = mutableListOf() + val baseKey = randomName + val bucketName = givenBucket(testInfo) + for (i in 0 until count) { + val key = "$baseKey-$i" + keys.add(key) + givenObject(bucketName, key, UPLOAD_FILE_NAME) + } + return bucketName to keys } private fun deleteBucket(bucket: Bucket) { - _s3ClientV2.deleteBucket { + _s3Client.deleteBucket { it.bucket(bucket.name()) } - val bucketDeleted = _s3ClientV2 + val bucketDeleted = _s3Client .waiter() - .waitUntilBucketNotExists(HeadBucketRequest - .builder() - .bucket(bucket.name()) - .build() - ) + .waitUntilBucketNotExists { + it.bucket(bucket.name()) + } bucketDeleted.matched().exception().get().also { assertThat(it).isNotNull } } private fun deleteObjectsInBucket(bucket: Bucket, objectLockEnabled: Boolean) { - _s3ClientV2.listObjectVersions { + _s3Client.listObjectVersions { it.bucket(bucket.name()) it.encodingType(EncodingType.URL) }.also { it.versions().forEach { objectVersion -> if (objectLockEnabled) { //must remove potential legal hold, otherwise object can't be deleted - _s3ClientV2.putObjectLegalHold { + _s3Client.putObjectLegalHold { it.bucket(bucket.name()) it.key(objectVersion.key()) it.versionId(objectVersion.versionId()) @@ -415,7 +332,7 @@ internal abstract class S3TestBase { } } } - _s3ClientV2.deleteObject { + _s3Client.deleteObject { it.bucket(bucket.name()) it.key(objectVersion.key()) it.versionId(objectVersion.versionId()) @@ -424,7 +341,7 @@ internal abstract class S3TestBase { it.deleteMarkers().forEach { marker -> if (objectLockEnabled) { //must remove potential legal hold, otherwise object can't be deleted - _s3ClientV2.putObjectLegalHold { + _s3Client.putObjectLegalHold { it.bucket(bucket.name()) it.key(marker.key()) it.versionId(marker.versionId()) @@ -433,7 +350,7 @@ internal abstract class S3TestBase { } } } - _s3ClientV2.deleteObject { + _s3Client.deleteObject { it.bucket(bucket.name()) it.key(marker.key()) it.versionId(marker.versionId()) @@ -444,7 +361,7 @@ internal abstract class S3TestBase { private fun isObjectLockEnabled(bucket: Bucket): Boolean { return try { - ObjectLockEnabled.ENABLED == _s3ClientV2.getObjectLockConfiguration { + ObjectLockEnabled.ENABLED == _s3Client.getObjectLockConfiguration { it.bucket(bucket.name()) }.objectLockConfiguration().objectLockEnabled() } catch (e: S3Exception) { @@ -454,10 +371,10 @@ internal abstract class S3TestBase { } private fun deleteMultipartUploads(bucket: Bucket) { - _s3ClientV2.listMultipartUploads { + _s3Client.listMultipartUploads { it.bucket(bucket.name()) }.uploads().forEach { upload -> - _s3ClientV2.abortMultipartUpload { + _s3Client.abortMultipartUpload { it.bucket(bucket.name()) it.key(upload.key()) it.uploadId(upload.uploadId()) @@ -486,18 +403,6 @@ internal abstract class S3TestBase { protected val httpPort: Int get() = Integer.getInteger("it.s3mock.port_http", 9090) - private fun ignoringInvalidSslCertificates(clientConfiguration: ClientConfiguration): - ClientConfiguration { - clientConfiguration.apacheHttpClientConfig - .withSslSocketFactory( - SSLConnectionSocketFactory( - createBlindlyTrustingSslContext(), - NoopHostnameVerifier.INSTANCE - ) - ) - return clientConfiguration - } - protected fun createBlindlyTrustingSslContext(): SSLContext { return try { val sc = SSLContext.getInstance("TLS") @@ -510,31 +415,20 @@ internal abstract class S3TestBase { // no-op } - override fun checkClientTrusted( - arg0: Array, arg1: String, - arg2: SSLEngine - ) { + override fun checkClientTrusted(arg0: Array, arg1: String, arg2: SSLEngine) { // no-op } - override fun checkClientTrusted( - arg0: Array, arg1: String, - arg2: Socket + override fun checkClientTrusted(arg0: Array, arg1: String, arg2: Socket ) { // no-op } - override fun checkServerTrusted( - arg0: Array, arg1: String, - arg2: SSLEngine - ) { + override fun checkServerTrusted(arg0: Array, arg1: String, arg2: SSLEngine) { // no-op } - override fun checkServerTrusted( - arg0: Array, arg1: String, - arg2: Socket - ) { + override fun checkServerTrusted(arg0: Array, arg1: String, arg2: Socket) { // no-op } @@ -557,20 +451,6 @@ internal abstract class S3TestBase { return ByteArrayInputStream(content) } - fun verifyObjectContent(uploadFile: File, s3Object: com.amazonaws.services.s3.model.S3Object) { - val uploadDigest = FileInputStream(uploadFile).use { - DigestUtil.hexDigest(it) - } - - s3Object.use { - val downloadedDigest = DigestUtil.hexDigest(s3Object.objectContent) - assertThat(uploadDigest) - .isEqualTo(downloadedDigest) - .`as`("Up- and downloaded Files should have equal digests") - } - } - - /** * Creates 5+MB of random bytes to upload as a valid part * (all parts but the last must be at least 5MB in size) diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsIT.kt similarity index 82% rename from integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsV2IT.kt rename to integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsIT.kt index 321cfef39..6534317f5 100644 --- a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsV2IT.kt +++ b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsIT.kt @@ -31,15 +31,15 @@ import software.amazon.awssdk.services.s3.model.S3Exception import software.amazon.awssdk.services.s3.model.StorageClass import java.io.File -internal class VersionsV2IT : S3TestBase() { - private val s3ClientV2: S3Client = createS3ClientV2() +internal class VersionsIT : S3TestBase() { + private val s3Client: S3Client = createS3Client() @Test @S3VerifiedSuccess(year = 2025) fun testListObjectVersions_nonVersionEnabledBucket(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - givenObjectV2(bucketName, UPLOAD_FILE_NAME) - val listObjectVersions = s3ClientV2.listObjectVersions { it.bucket(bucketName) } + val bucketName = givenBucket(testInfo) + givenObject(bucketName, UPLOAD_FILE_NAME) + val listObjectVersions = s3Client.listObjectVersions { it.bucket(bucketName) } assertThat(listObjectVersions.hasVersions()).isTrue assertThat(listObjectVersions.versions()[0].key()).isEqualTo(UPLOAD_FILE_NAME) assertThat(listObjectVersions.versions()[0].versionId()).isEqualTo("null") @@ -48,17 +48,17 @@ internal class VersionsV2IT : S3TestBase() { @Test @S3VerifiedSuccess(year = 2025) fun testListObjectVersions_versionEnabledBucket(testInfo: TestInfo) { - val bucketName = givenBucketV2(testInfo) - s3ClientV2.putBucketVersioning { + val bucketName = givenBucket(testInfo) + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - val versionId = givenObjectV2(bucketName, UPLOAD_FILE_NAME).versionId() + val versionId = givenObject(bucketName, UPLOAD_FILE_NAME).versionId() - val listObjectVersions = s3ClientV2.listObjectVersions { it.bucket(bucketName) } + val listObjectVersions = s3Client.listObjectVersions { it.bucket(bucketName) } assertThat(listObjectVersions.hasVersions()).isTrue assertThat(listObjectVersions.versions()[0].key()).isEqualTo(UPLOAD_FILE_NAME) assertThat(listObjectVersions.versions()[0].versionId()).isEqualTo(versionId) @@ -69,23 +69,23 @@ internal class VersionsV2IT : S3TestBase() { fun testPutGetObject_withVersion(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), Algorithm.SHA1) - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) - s3ClientV2.putBucketVersioning { + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - val versionId = s3ClientV2.putObject( + val versionId = s3Client.putObject( { it.bucket(bucketName).key(UPLOAD_FILE_NAME) it.checksumAlgorithm(ChecksumAlgorithm.SHA1) }, RequestBody.fromFile(uploadFile) ).versionId() - s3ClientV2.getObjectAttributes { + s3Client.getObjectAttributes { it.bucket(bucketName) it.key(UPLOAD_FILE_NAME) it.versionId(versionId) @@ -109,30 +109,30 @@ internal class VersionsV2IT : S3TestBase() { @S3VerifiedSuccess(year = 2025) fun testPutGetObject_withMultipleVersions(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) - s3ClientV2.putBucketVersioning { + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - val versionId1 = s3ClientV2.putObject( + val versionId1 = s3Client.putObject( { it.bucket(bucketName).key(UPLOAD_FILE_NAME) it.checksumAlgorithm(ChecksumAlgorithm.SHA1) }, RequestBody.fromFile(uploadFile) ).versionId() - val versionId2 = s3ClientV2.putObject( + val versionId2 = s3Client.putObject( { it.bucket(bucketName).key(UPLOAD_FILE_NAME) it.checksumAlgorithm(ChecksumAlgorithm.SHA1) }, RequestBody.fromFile(uploadFile) ).versionId() - s3ClientV2.getObject { + s3Client.getObject { it.bucket(bucketName) it.key(UPLOAD_FILE_NAME) it.versionId(versionId2) @@ -140,7 +140,7 @@ internal class VersionsV2IT : S3TestBase() { assertThat(it.response().versionId()).isEqualTo(versionId2) } - s3ClientV2.getObject { + s3Client.getObject { it.bucket(bucketName) it.key(UPLOAD_FILE_NAME) it.versionId(versionId1) @@ -148,7 +148,7 @@ internal class VersionsV2IT : S3TestBase() { assertThat(it.response().versionId()).isEqualTo(versionId1) } - s3ClientV2.getObject { + s3Client.getObject { it.bucket(bucketName) it.key(UPLOAD_FILE_NAME) }.also { @@ -160,36 +160,36 @@ internal class VersionsV2IT : S3TestBase() { @S3VerifiedSuccess(year = 2025) fun testPutGetDeleteObject_withVersion(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) - s3ClientV2.putBucketVersioning { + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - val versionId1 = s3ClientV2.putObject( + val versionId1 = s3Client.putObject( { it.bucket(bucketName).key(UPLOAD_FILE_NAME) it.checksumAlgorithm(ChecksumAlgorithm.SHA1) }, RequestBody.fromFile(uploadFile) ).versionId() - val versionId2 = s3ClientV2.putObject( + val versionId2 = s3Client.putObject( { it.bucket(bucketName).key(UPLOAD_FILE_NAME) it.checksumAlgorithm(ChecksumAlgorithm.SHA1) }, RequestBody.fromFile(uploadFile) ).versionId() - s3ClientV2.deleteObject { + s3Client.deleteObject { it.bucket(bucketName) it.key(UPLOAD_FILE_NAME) it.versionId(versionId2) } - s3ClientV2.getObject { + s3Client.getObject { it.bucket(bucketName) it.key(UPLOAD_FILE_NAME) }.also { @@ -201,28 +201,28 @@ internal class VersionsV2IT : S3TestBase() { @S3VerifiedSuccess(year = 2025) fun testPutGetDeleteObject_withDeleteMarker(testInfo: TestInfo) { val uploadFile = File(UPLOAD_FILE_NAME) - val bucketName = givenBucketV2(testInfo) + val bucketName = givenBucket(testInfo) - s3ClientV2.putBucketVersioning { + s3Client.putBucketVersioning { it.bucket(bucketName) it.versioningConfiguration { it.status(BucketVersioningStatus.ENABLED) } } - s3ClientV2.putObject( + s3Client.putObject( { it.bucket(bucketName).key(UPLOAD_FILE_NAME) }, RequestBody.fromFile(uploadFile) ).versionId() - s3ClientV2.putObject( + s3Client.putObject( { it.bucket(bucketName).key(UPLOAD_FILE_NAME) }, RequestBody.fromFile(uploadFile) ).versionId() - s3ClientV2.deleteObject { + s3Client.deleteObject { it.bucket(bucketName) it.key(UPLOAD_FILE_NAME) }.also { @@ -230,7 +230,7 @@ internal class VersionsV2IT : S3TestBase() { } assertThatThrownBy { - s3ClientV2.getObject { + s3Client.getObject { it.bucket(bucketName) it.key(UPLOAD_FILE_NAME) } diff --git a/server/src/main/java/com/adobe/testing/s3mock/KmsValidationFilter.java b/server/src/main/java/com/adobe/testing/s3mock/KmsValidationFilter.java index d5e876faa..1fe96240f 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/KmsValidationFilter.java +++ b/server/src/main/java/com/adobe/testing/s3mock/KmsValidationFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2023 Adobe. + * Copyright 2017-2025 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -82,7 +82,7 @@ protected void doFilterInternal(@NonNull HttpServletRequest request, var errorResponse = new ErrorResponse( "KMS.NotFoundException", - "Key ID " + encryptionKeyId + " does not exist!", + "Invalid keyId '" + encryptionKeyId + "'", null, null ); diff --git a/server/src/main/java/com/adobe/testing/s3mock/ObjectController.java b/server/src/main/java/com/adobe/testing/s3mock/ObjectController.java index 19d6426c4..42ab0a49f 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/ObjectController.java +++ b/server/src/main/java/com/adobe/testing/s3mock/ObjectController.java @@ -195,33 +195,28 @@ public ResponseEntity headObject(@PathVariable String bucketName, @RequestParam(value = VERSION_ID, required = false) String versionId, @RequestParam Map queryParams) { var bucket = bucketService.verifyBucketExists(bucketName); - var s3ObjectMetadata = objectService.verifyObjectExists(bucketName, key.key(), versionId); + objectService.verifyObjectMatching(match, noneMatch, + ifModifiedSince, ifUnmodifiedSince, s3ObjectMetadata); - if (s3ObjectMetadata != null) { - objectService.verifyObjectMatching(match, noneMatch, - ifModifiedSince, ifUnmodifiedSince, s3ObjectMetadata); - return ResponseEntity.ok() - .eTag(s3ObjectMetadata.etag()) - .header(HttpHeaders.ACCEPT_RANGES, RANGES_BYTES) - .lastModified(s3ObjectMetadata.lastModified()) - .contentLength(Long.parseLong(s3ObjectMetadata.size())) - .contentType(mediaTypeFrom(s3ObjectMetadata.contentType())) - .headers(h -> { - if (bucket.isVersioningEnabled() && s3ObjectMetadata.versionId() != null) { - h.set(X_AMZ_VERSION_ID, s3ObjectMetadata.versionId()); - } - }) - .headers(h -> h.setAll(s3ObjectMetadata.storeHeaders())) - .headers(h -> h.setAll(userMetadataHeadersFrom(s3ObjectMetadata))) - .headers(h -> h.setAll(s3ObjectMetadata.encryptionHeaders())) - .headers(h -> h.setAll(checksumHeaderFrom(s3ObjectMetadata))) - .headers(h -> h.setAll(storageClassHeadersFrom(s3ObjectMetadata))) - .headers(h -> h.setAll(overrideHeadersFrom(queryParams))) - .build(); - } else { - return ResponseEntity.status(NOT_FOUND).build(); - } + return ResponseEntity.ok() + .eTag(s3ObjectMetadata.etag()) + .header(HttpHeaders.ACCEPT_RANGES, RANGES_BYTES) + .lastModified(s3ObjectMetadata.lastModified()) + .contentLength(Long.parseLong(s3ObjectMetadata.size())) + .contentType(mediaTypeFrom(s3ObjectMetadata.contentType())) + .headers(h -> { + if (bucket.isVersioningEnabled() && s3ObjectMetadata.versionId() != null) { + h.set(X_AMZ_VERSION_ID, s3ObjectMetadata.versionId()); + } + }) + .headers(h -> h.setAll(s3ObjectMetadata.storeHeaders())) + .headers(h -> h.setAll(userMetadataHeadersFrom(s3ObjectMetadata))) + .headers(h -> h.setAll(s3ObjectMetadata.encryptionHeaders())) + .headers(h -> h.setAll(checksumHeaderFrom(s3ObjectMetadata))) + .headers(h -> h.setAll(storageClassHeadersFrom(s3ObjectMetadata))) + .headers(h -> h.setAll(overrideHeadersFrom(queryParams))) + .build(); } /** diff --git a/server/src/main/java/com/adobe/testing/s3mock/S3MockConfiguration.java b/server/src/main/java/com/adobe/testing/s3mock/S3MockConfiguration.java index 465a81e6e..f805b2990 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/S3MockConfiguration.java +++ b/server/src/main/java/com/adobe/testing/s3mock/S3MockConfiguration.java @@ -68,12 +68,12 @@ public class S3MockConfiguration implements WebMvcConfigurer { @Bean ServletWebServerFactory webServerFactory(S3MockProperties properties) { var factory = new TomcatServletWebServerFactory(); - factory.addAdditionalTomcatConnectors(createHttpConnector(properties.httpPort())); factory.addConnectorCustomizers(connector -> { // Allow encoded slashes in URL connector.setEncodedSolidusHandling(EncodedSolidusHandling.DECODE.getValue()); connector.setAllowBackslash(true); }); + factory.addAdditionalTomcatConnectors(createHttpConnector(properties.httpPort())); return factory; } diff --git a/server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java b/server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java index 521fde25b..ed07ab2eb 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java +++ b/server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java @@ -298,6 +298,11 @@ public ListBucketResultV2 listObjectsV2(String bucketName, Integer maxKeys, String continuationToken) { + if (maxKeys == 0) { + return new ListBucketResultV2(bucketName, prefix, maxKeys, false, List.of(), List.of(), + continuationToken, "0", null, null, encodingType); + } + var contents = getS3Objects(bucketName, prefix); var nextContinuationToken = (String) null; var isTruncated = false; @@ -354,8 +359,10 @@ public ListBucketResultV2 listObjectsV2(String bucketName, public ListBucketResult listObjectsV1(String bucketName, String prefix, String delimiter, String marker, String encodingType, Integer maxKeys) { - verifyMaxKeys(maxKeys); - verifyEncodingType(encodingType); + if (maxKeys == 0) { + return new ListBucketResult(bucketName, prefix, marker, maxKeys, false, encodingType, + null, List.of(), List.of(), null); + } var contents = getS3Objects(bucketName, prefix); contents = filterObjectsBy(contents, marker); diff --git a/server/src/main/java/com/adobe/testing/s3mock/service/MultipartService.java b/server/src/main/java/com/adobe/testing/s3mock/service/MultipartService.java index 45de7f3ac..d2bc56ba4 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/service/MultipartService.java +++ b/server/src/main/java/com/adobe/testing/s3mock/service/MultipartService.java @@ -317,6 +317,7 @@ public void verifyMultipartParts(String bucketName, UUID id, String uploadId) th if (!uploadedParts.isEmpty()) { for (int i = 0; i < uploadedParts.size() - 1; i++) { var part = uploadedParts.get(i); + verifyPartNumberLimits(part.partNumber().toString()); if (part.size() < MINIMUM_PART_SIZE) { LOG.error("Multipart part size too small. bucket={}, id={}, uploadId={}, size={}", bucketMetadata, id, uploadId, part.size()); diff --git a/server/src/main/java/com/adobe/testing/s3mock/service/ObjectService.java b/server/src/main/java/com/adobe/testing/s3mock/service/ObjectService.java index 74d3dc27e..838f0bc6d 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/service/ObjectService.java +++ b/server/src/main/java/com/adobe/testing/s3mock/service/ObjectService.java @@ -310,7 +310,7 @@ public void verifyObjectMatching(List match, List noneMatch, var setModifiedSince = ifModifiedSince != null && !ifModifiedSince.isEmpty(); if (setModifiedSince) { if (ifModifiedSince.get(0).isAfter(lastModified)) { - throw PRECONDITION_FAILED; + throw NOT_MODIFIED; } } diff --git a/server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java b/server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java index 9f4766cab..e3fb14b60 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java +++ b/server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java @@ -1,5 +1,5 @@ /* - * Copyright 2017-2024 Adobe. + * Copyright 2017-2025 Adobe. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,14 +34,17 @@ import com.adobe.testing.s3mock.util.DigestUtil; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpHeaders; abstract class ServiceBase { + private static final Logger LOG = LoggerFactory.getLogger(ServiceBase.class); + public void verifyChecksum(Path path, String checksum, ChecksumAlgorithm checksumAlgorithm) { String checksumFor = DigestUtil.checksumFor(path, checksumAlgorithm.toAlgorithm()); if (!checksum.equals(checksumFor)) { @@ -69,6 +72,7 @@ public Pair toTempFile(InputStream inputStream, HttpHeaders httpHe return Pair.of(tempFile, null); } } catch (IOException e) { + LOG.error("Error reading from InputStream", e); throw BAD_REQUEST_CONTENT; } } diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/ObjectControllerTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/ObjectControllerTest.kt index c70d97d1c..c80cddf4d 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/ObjectControllerTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/ObjectControllerTest.kt @@ -370,6 +370,8 @@ internal class ObjectControllerTest : BaseControllerTest() { fun testHeadObject_NotFound() { givenBucket() val key = "name" + whenever(objectService.verifyObjectExists("test-bucket", key, null)) + .thenThrow(S3Exception.NO_SUCH_KEY) val headers = HttpHeaders().apply { this.accept = listOf(MediaType.APPLICATION_XML) diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/service/ObjectServiceTest.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/service/ObjectServiceTest.kt index c7b9120b9..698f8ce3a 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/service/ObjectServiceTest.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/service/ObjectServiceTest.kt @@ -181,7 +181,7 @@ internal class ObjectServiceTest : ServiceTestBase() { val now = Instant.now().plusSeconds(10) assertThatThrownBy { iut.verifyObjectMatching(null, null, listOf(now), null, s3ObjectMetadata) } - .isEqualTo(S3Exception.PRECONDITION_FAILED) + .isEqualTo(S3Exception.NOT_MODIFIED) } @Test diff --git a/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt b/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt index eb8b363c5..ba9d37896 100644 --- a/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt +++ b/server/src/test/kotlin/com/adobe/testing/s3mock/service/ServiceTestBase.kt @@ -141,7 +141,7 @@ internal abstract class ServiceTestBase { fun givenParts(count: Int, size: Long): List { val parts = mutableListOf() - for (i in 0 until count) { + for (i in 1 .. count) { val lastModified = Date() parts.add(Part(i, "\"${UUID.randomUUID()}\"", lastModified, size)) }