Skip to content

Commit 30fd0ac

Browse files
afrankenafranken
committed
Support delete markers and version deletion
Fixes #64
1 parent 7a39002 commit 30fd0ac

File tree

17 files changed

+340
-56
lines changed

17 files changed

+340
-56
lines changed

‎integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/VersionsV2IT.kt‎

Lines changed: 90 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package com.adobe.testing.s3mock.its
1818

1919
import com.adobe.testing.s3mock.util.DigestUtil
2020
import org.assertj.core.api.Assertions.assertThat
21+
import org.assertj.core.api.Assertions.assertThatThrownBy
2122
import org.junit.jupiter.api.Test
2223
import org.junit.jupiter.api.TestInfo
2324
import software.amazon.awssdk.core.checksums.Algorithm
@@ -26,14 +27,15 @@ import software.amazon.awssdk.services.s3.S3Client
2627
import software.amazon.awssdk.services.s3.model.BucketVersioningStatus
2728
import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm
2829
import software.amazon.awssdk.services.s3.model.ObjectAttributes
30+
import software.amazon.awssdk.services.s3.model.S3Exception
2931
import software.amazon.awssdk.services.s3.model.StorageClass
3032
import java.io.File
3133

3234
internal class VersionsV2IT : S3TestBase() {
3335
private val s3ClientV2: S3Client = createS3ClientV2()
3436

3537
@Test
36-
@S3VerifiedSuccess(year = 2024)
38+
@S3VerifiedTodo
3739
fun testPutGetObject_withVersion(testInfo: TestInfo) {
3840
val uploadFile = File(UPLOAD_FILE_NAME)
3941
val expectedChecksum = DigestUtil.checksumFor(uploadFile.toPath(), Algorithm.SHA1)
@@ -74,7 +76,7 @@ internal class VersionsV2IT : S3TestBase() {
7476
}
7577

7678
@Test
77-
@S3VerifiedSuccess(year = 2024)
79+
@S3VerifiedTodo
7880
fun testPutGetObject_withMultipleVersions(testInfo: TestInfo) {
7981
val uploadFile = File(UPLOAD_FILE_NAME)
8082
val bucketName = givenBucketV2(testInfo)
@@ -123,4 +125,90 @@ internal class VersionsV2IT : S3TestBase() {
123125
assertThat(it.response().versionId()).isEqualTo(versionId2)
124126
}
125127
}
128+
129+
@Test
130+
@S3VerifiedTodo
131+
fun testPutGetDeleteObject_withVersion(testInfo: TestInfo) {
132+
val uploadFile = File(UPLOAD_FILE_NAME)
133+
val bucketName = givenBucketV2(testInfo)
134+
135+
s3ClientV2.putBucketVersioning {
136+
it.bucket(bucketName)
137+
it.versioningConfiguration {
138+
it.status(BucketVersioningStatus.ENABLED)
139+
}
140+
}
141+
142+
val versionId1 = s3ClientV2.putObject(
143+
{
144+
it.bucket(bucketName).key(UPLOAD_FILE_NAME)
145+
it.checksumAlgorithm(ChecksumAlgorithm.SHA1)
146+
}, RequestBody.fromFile(uploadFile)
147+
).versionId()
148+
149+
val versionId2 = s3ClientV2.putObject(
150+
{
151+
it.bucket(bucketName).key(UPLOAD_FILE_NAME)
152+
it.checksumAlgorithm(ChecksumAlgorithm.SHA1)
153+
}, RequestBody.fromFile(uploadFile)
154+
).versionId()
155+
156+
s3ClientV2.deleteObject {
157+
it.bucket(bucketName)
158+
it.key(UPLOAD_FILE_NAME)
159+
it.versionId(versionId2)
160+
}.also {
161+
assertThat(it.deleteMarker()).isEqualTo(false)
162+
}
163+
164+
s3ClientV2.getObject {
165+
it.bucket(bucketName)
166+
it.key(UPLOAD_FILE_NAME)
167+
}.also {
168+
assertThat(it.response().versionId()).isEqualTo(versionId1)
169+
}
170+
}
171+
172+
@Test
173+
@S3VerifiedTodo
174+
fun testPutGetDeleteObject_withDeleteMarker(testInfo: TestInfo) {
175+
val uploadFile = File(UPLOAD_FILE_NAME)
176+
val bucketName = givenBucketV2(testInfo)
177+
178+
s3ClientV2.putBucketVersioning {
179+
it.bucket(bucketName)
180+
it.versioningConfiguration {
181+
it.status(BucketVersioningStatus.ENABLED)
182+
}
183+
}
184+
185+
val versionId1 = s3ClientV2.putObject(
186+
{
187+
it.bucket(bucketName).key(UPLOAD_FILE_NAME)
188+
it.checksumAlgorithm(ChecksumAlgorithm.SHA1)
189+
}, RequestBody.fromFile(uploadFile)
190+
).versionId()
191+
192+
s3ClientV2.putObject(
193+
{
194+
it.bucket(bucketName).key(UPLOAD_FILE_NAME)
195+
it.checksumAlgorithm(ChecksumAlgorithm.SHA1)
196+
}, RequestBody.fromFile(uploadFile)
197+
).versionId()
198+
199+
s3ClientV2.deleteObject {
200+
it.bucket(bucketName)
201+
it.key(UPLOAD_FILE_NAME)
202+
}.also {
203+
assertThat(it.deleteMarker()).isEqualTo(true)
204+
}
205+
206+
assertThatThrownBy {
207+
s3ClientV2.getObject {
208+
it.bucket(bucketName)
209+
it.key(UPLOAD_FILE_NAME)
210+
}
211+
}.isInstanceOf(S3Exception::class.java)
212+
.hasMessageContaining("Service: S3, Status Code: 404")
213+
}
126214
}

‎server/src/main/java/com/adobe/testing/s3mock/ObjectController.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.adobe.testing.s3mock;
1818

19+
import static com.adobe.testing.s3mock.S3Exception.NO_SUCH_KEY_DELETE_MARKER;
1920
import static com.adobe.testing.s3mock.dto.StorageClass.STANDARD;
2021
import static com.adobe.testing.s3mock.service.ObjectService.getChecksum;
2122
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.CONTENT_MD5;
@@ -257,6 +258,18 @@ public ResponseEntity<Void> deleteObject(@PathVariable String bucketName,
257258
h.set(X_AMZ_VERSION_ID, s3ObjectMetadataVersionId);
258259
}
259260
})
261+
.headers(h -> {
262+
if (bucket.isVersioningEnabled()) {
263+
try {
264+
objectService.verifyObjectExists(bucketName, key.key(), versionId);
265+
} catch (S3Exception e) {
266+
//ignore all other exceptions here
267+
if (e == NO_SUCH_KEY_DELETE_MARKER) {
268+
h.set(X_AMZ_DELETE_MARKER, "true");
269+
}
270+
}
271+
}
272+
})
260273
.build();
261274
}
262275

‎server/src/main/java/com/adobe/testing/s3mock/S3Exception.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2017-2024 Adobe.
2+
* Copyright 2017-2025 Adobe.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -59,6 +59,8 @@ public class S3Exception extends RuntimeException {
5959
"The lifecycle configuration does not exist.");
6060
public static final S3Exception NO_SUCH_KEY =
6161
new S3Exception(NOT_FOUND.value(), "NoSuchKey", "The specified key does not exist.");
62+
public static final S3Exception NO_SUCH_KEY_DELETE_MARKER =
63+
new S3Exception(NOT_FOUND.value(), "NoSuchKey", "The specified key does not exist.");
6264
public static final S3Exception NOT_MODIFIED =
6365
new S3Exception(HttpStatus.NOT_MODIFIED.value(), "NotModified", "Not Modified");
6466
public static final S3Exception PRECONDITION_FAILED =

‎server/src/main/java/com/adobe/testing/s3mock/S3MockConfiguration.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2017-2024 Adobe.
2+
* Copyright 2017-2025 Adobe.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
1616

1717
package com.adobe.testing.s3mock;
1818

19+
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_DELETE_MARKER;
1920
import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
2021

2122
import com.adobe.testing.s3mock.dto.ErrorResponse;
@@ -228,6 +229,9 @@ public ResponseEntity<ErrorResponse> handleS3Exception(final S3Exception s3Excep
228229

229230
var headers = new HttpHeaders();
230231
headers.setContentType(MediaType.APPLICATION_XML);
232+
if (s3Exception == S3Exception.NO_SUCH_KEY_DELETE_MARKER) {
233+
headers.set(X_AMZ_DELETE_MARKER, "true");
234+
}
231235

232236
return ResponseEntity.status(s3Exception.getStatus()).headers(headers).body(errorResponse);
233237
}

‎server/src/main/java/com/adobe/testing/s3mock/dto/ObjectVersion.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2017-2024 Adobe.
2+
* Copyright 2017-2025 Adobe.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -58,7 +58,7 @@ public static ObjectVersion from(S3ObjectMetadata s3ObjectMetadata) {
5858
s3ObjectMetadata.owner(),
5959
s3ObjectMetadata.checksumAlgorithm(),
6060
true,
61-
"staticVersion");
61+
s3ObjectMetadata.versionId());
6262
}
6363

6464
public static ObjectVersion from(S3Object s3Object) {

‎server/src/main/java/com/adobe/testing/s3mock/service/BucketService.java‎

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,24 @@ public BucketService(BucketStore bucketStore, ObjectStore objectStore) {
6666
}
6767

6868
public boolean isBucketEmpty(String bucketName) {
69-
return bucketStore.isBucketEmpty(bucketName);
69+
var bucketMetadata = bucketStore.getBucketMetadata(bucketName);
70+
if (bucketMetadata != null) {
71+
var objects = bucketMetadata.objects();
72+
if (!objects.isEmpty()) {
73+
for (var id : objects.values()) {
74+
var s3ObjectMetadata = objectStore.getS3ObjectMetadata(bucketMetadata, id, null);
75+
if (s3ObjectMetadata != null) {
76+
if (!s3ObjectMetadata.deleteMarker()) {
77+
return false;
78+
}
79+
}
80+
}
81+
return true;
82+
}
83+
return bucketMetadata.objects().isEmpty();
84+
} else {
85+
throw new IllegalStateException("Requested Bucket does not exist: " + bucketName);
86+
}
7087
}
7188

7289
public boolean doesBucketExist(String bucketName) {
@@ -112,7 +129,32 @@ public Bucket createBucket(String bucketName,
112129
}
113130

114131
public boolean deleteBucket(String bucketName) {
115-
return bucketStore.deleteBucket(bucketName);
132+
var bucketMetadata = bucketStore.getBucketMetadata(bucketName);
133+
if (bucketMetadata != null) {
134+
var objects = bucketMetadata.objects();
135+
if (!objects.isEmpty()) {
136+
for (var entry : objects.entrySet()) {
137+
var s3ObjectMetadata =
138+
objectStore.getS3ObjectMetadata(bucketMetadata, entry.getValue(), null);
139+
if (s3ObjectMetadata != null) {
140+
if (s3ObjectMetadata.deleteMarker()) {
141+
//yes, we really want to delete the objects here, if they are delete markers, they
142+
//do not officially exist.
143+
objectStore.doDeleteObject(bucketMetadata, entry.getValue());
144+
bucketStore.removeFromBucket(entry.getKey(), bucketName);
145+
}
146+
}
147+
}
148+
}
149+
//check again if bucket is empty
150+
bucketMetadata = bucketStore.getBucketMetadata(bucketName);
151+
if (!bucketMetadata.objects().isEmpty()) {
152+
throw new IllegalStateException("Bucket is not empty: " + bucketName);
153+
}
154+
return bucketStore.deleteBucket(bucketName);
155+
} else {
156+
throw new IllegalStateException("Requested Bucket does not exist: " + bucketName);
157+
}
116158
}
117159

118160
public void setVersioningConfiguration(String bucketName, VersioningConfiguration configuration) {
@@ -176,9 +218,9 @@ public BucketLifecycleConfiguration getBucketLifecycleConfiguration(String bucke
176218
public List<S3Object> getS3Objects(String bucketName, String prefix) {
177219
var bucketMetadata = bucketStore.getBucketMetadata(bucketName);
178220
var uuids = bucketStore.lookupKeysInBucket(prefix, bucketName);
179-
//TODO: versionId?
180221
return uuids
181222
.stream()
223+
.filter(Objects::nonNull)
182224
.map(uuid -> objectStore.getS3ObjectMetadata(bucketMetadata, uuid, null))
183225
.filter(Objects::nonNull)
184226
.map(S3Object::from)
@@ -346,7 +388,7 @@ public void verifyBucketNameIsAllowed(String bucketName) {
346388
}
347389

348390
public void verifyBucketIsEmpty(String bucketName) {
349-
if (!bucketStore.isBucketEmpty(bucketName)) {
391+
if (!isBucketEmpty(bucketName)) {
350392
throw BUCKET_NOT_EMPTY;
351393
}
352394
}

‎server/src/main/java/com/adobe/testing/s3mock/service/ObjectService.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@
2222
import static com.adobe.testing.s3mock.S3Exception.NOT_FOUND_OBJECT_LOCK;
2323
import static com.adobe.testing.s3mock.S3Exception.NOT_MODIFIED;
2424
import static com.adobe.testing.s3mock.S3Exception.NO_SUCH_KEY;
25+
import static com.adobe.testing.s3mock.S3Exception.NO_SUCH_KEY_DELETE_MARKER;
2526
import static com.adobe.testing.s3mock.S3Exception.PRECONDITION_FAILED;
2627

2728
import com.adobe.testing.s3mock.S3Exception;
2829
import com.adobe.testing.s3mock.dto.AccessControlPolicy;
2930
import com.adobe.testing.s3mock.dto.Checksum;
3031
import com.adobe.testing.s3mock.dto.ChecksumAlgorithm;
31-
import com.adobe.testing.s3mock.dto.CopyObjectResult;
3232
import com.adobe.testing.s3mock.dto.Delete;
3333
import com.adobe.testing.s3mock.dto.DeleteResult;
3434
import com.adobe.testing.s3mock.dto.DeletedS3Object;
@@ -348,6 +348,8 @@ public S3ObjectMetadata verifyObjectExists(String bucketName, String key, String
348348
var s3ObjectMetadata = objectStore.getS3ObjectMetadata(bucketMetadata, uuid, versionId);
349349
if (s3ObjectMetadata == null) {
350350
throw NO_SUCH_KEY;
351+
} else if (s3ObjectMetadata.deleteMarker()) {
352+
throw NO_SUCH_KEY_DELETE_MARKER;
351353
}
352354
return s3ObjectMetadata;
353355
}

‎server/src/main/java/com/adobe/testing/s3mock/store/BucketMetadata.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.adobe.testing.s3mock.store;
1818

1919
import static com.adobe.testing.s3mock.dto.VersioningConfiguration.Status.ENABLED;
20+
import static com.adobe.testing.s3mock.dto.VersioningConfiguration.Status.SUSPENDED;
2021

2122
import com.adobe.testing.s3mock.dto.BucketLifecycleConfiguration;
2223
import com.adobe.testing.s3mock.dto.ObjectLockConfiguration;
@@ -120,4 +121,11 @@ public boolean isVersioningEnabled() {
120121
&& this.versioningConfiguration().status() != null
121122
&& this.versioningConfiguration().status() == ENABLED;
122123
}
124+
125+
@JsonIgnore
126+
public boolean isVersioningSuspended() {
127+
return this.versioningConfiguration() != null
128+
&& this.versioningConfiguration().status() != null
129+
&& this.versioningConfiguration().status() == SUSPENDED;
130+
}
123131
}

‎server/src/main/java/com/adobe/testing/s3mock/store/BucketStore.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2017-2024 Adobe.
2+
* Copyright 2017-2025 Adobe.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -275,9 +275,6 @@ public boolean deleteBucket(String bucketName) {
275275
synchronized (lockStore.get(bucketName)) {
276276
var bucketMetadata = getBucketMetadata(bucketName);
277277
if (bucketMetadata != null && bucketMetadata.objects().isEmpty()) {
278-
//TODO: this currently does not work, since we store objects below their prefixes, which
279-
// are not deleted when deleting the object, leaving empty directories in the S3Mock
280-
// filesystem should be: return Files.deleteIfExists(bucket.getPath())
281278
FileUtils.deleteDirectory(bucketMetadata.path().toFile());
282279
lockStore.remove(bucketName);
283280
return true;
@@ -286,7 +283,7 @@ public boolean deleteBucket(String bucketName) {
286283
}
287284
}
288285
} catch (final IOException e) {
289-
throw new IllegalStateException("Can't create bucket directory!", e);
286+
throw new IllegalStateException("Can't delete bucket directory!", e);
290287
}
291288
}
292289

0 commit comments

Comments
 (0)