Skip to content

Commit b30d8bc

Browse files
afrankenafranken
authored
Merge pull request #2173 from adobe/2143-fix-file-deletion
Refactor deletion of all files on exit
2 parents 069d87b + 8a742fa commit b30d8bc

File tree

12 files changed

+111
-101
lines changed

12 files changed

+111
-101
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,13 +138,17 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
138138
* Features and fixes
139139
* Allow overriding headers in head object
140140
* Implement If-(Un)modified-Since handling (fixes #829)
141+
* Close all InputStreams and OutputStreams
141142
* Refactorings
142143
* Use Tomcat instead of Jetty as the application container (fixes #2136)
143144
* "FROM" in Dockerfile did not match "as"
145+
* Delete files on shutdown using a `DisposableBean` instead of `File#deleteOnExit()`
144146
* Version updates (deliverable dependencies)
145-
* none
147+
* Bump spring-boot.version from 3.3.5 to 3.4.1
146148
* Version updates (build dependencies)
147149
* Bump github/codeql-action from 3.27.6 to 3.27.9
150+
* Bump actions/upload-artifact from 4.4.3 to 4.5.0
151+
* Bump actions/setup-java from 4.5.0 to 4.6.0
148152
* Bump com.puppycrawl.tools:checkstyle from 10.20.2 to 10.21.0
149153
* Jackson 2.18.2 to 2.17.2 (remove override, use Spring-Boot supplied version)
150154

‎server/src/main/java/com/adobe/testing/s3mock/ObjectController.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@
9999
import java.util.List;
100100
import java.util.Map;
101101
import org.apache.commons.io.FileUtils;
102-
import org.apache.commons.io.IOUtils;
103102
import org.apache.commons.io.input.BoundedInputStream;
104103
import org.springframework.http.HttpHeaders;
105104
import org.springframework.http.HttpRange;
@@ -785,7 +784,13 @@ private static void extractBytesToOutputStream(HttpRange range, S3ObjectMetadata
785784
try (var fis = Files.newInputStream(s3ObjectMetadata.dataPath())) {
786785
var skip = fis.skip(range.getRangeStart(fileSize));
787786
if (skip == range.getRangeStart(fileSize)) {
788-
IOUtils.copy(new BoundedInputStream(fis, bytesToRead), outputStream);
787+
try (var bis = BoundedInputStream
788+
.builder()
789+
.setInputStream(fis)
790+
.setMaxCount(bytesToRead)
791+
.get()) {
792+
bis.transferTo(outputStream);
793+
}
789794
} else {
790795
throw new IllegalStateException("Could not skip exact byte range");
791796
}

‎server/src/main/java/com/adobe/testing/s3mock/service/ServiceBase.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ public void verifyChecksum(Path path, String checksum, ChecksumAlgorithm checksu
5858
public Pair<Path, String> toTempFile(InputStream inputStream, HttpHeaders httpHeaders) {
5959
try {
6060
var tempFile = Files.createTempFile("ObjectService", "toTempFile");
61-
try (OutputStream os = Files.newOutputStream(tempFile)) {
62-
InputStream wrappedStream = wrapStream(inputStream, httpHeaders);
61+
try (var os = Files.newOutputStream(tempFile);
62+
var wrappedStream = wrapStream(inputStream, httpHeaders)) {
6363
wrappedStream.transferTo(os);
6464
ChecksumAlgorithm algorithmFromSdk = checksumAlgorithmFromSdk(httpHeaders);
6565
if (algorithmFromSdk != null

‎server/src/main/java/com/adobe/testing/s3mock/store/BucketStore.java‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,13 @@ public class BucketStore {
5151
*/
5252
private final Map<String, Object> lockStore = new ConcurrentHashMap<>();
5353
private final File rootFolder;
54-
private final boolean retainFilesOnExit;
5554
private final DateTimeFormatter s3ObjectDateFormat;
5655
private final ObjectMapper objectMapper;
5756

5857
public BucketStore(File rootFolder,
59-
boolean retainFilesOnExit,
6058
DateTimeFormatter s3ObjectDateFormat,
6159
ObjectMapper objectMapper) {
6260
this.rootFolder = rootFolder;
63-
this.retainFilesOnExit = retainFilesOnExit;
6461
this.s3ObjectDateFormat = s3ObjectDateFormat;
6562
this.objectMapper = objectMapper;
6663
}
@@ -309,9 +306,6 @@ List<UUID> loadBuckets(List<String> bucketNames) {
309306
private void writeToDisk(BucketMetadata bucketMetadata) {
310307
try {
311308
var metaFile = getMetaFilePath(bucketMetadata.name()).toFile();
312-
if (!retainFilesOnExit) {
313-
metaFile.deleteOnExit();
314-
}
315309
synchronized (lockStore.get(bucketMetadata.name())) {
316310
objectMapper.writeValue(metaFile, bucketMetadata);
317311
}
@@ -328,9 +322,6 @@ private File createBucketFolder(String bucketName) {
328322
try {
329323
var bucketFolder = getBucketFolderPath(bucketName).toFile();
330324
FileUtils.forceMkdir(bucketFolder);
331-
if (!retainFilesOnExit) {
332-
bucketFolder.deleteOnExit();
333-
}
334325
return bucketFolder;
335326
} catch (final IOException e) {
336327
throw new IllegalStateException("Can't create bucket directory!", e);

‎server/src/main/java/com/adobe/testing/s3mock/store/MultipartStore.java‎

Lines changed: 33 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050
import java.util.concurrent.ConcurrentHashMap;
5151
import java.util.stream.StreamSupport;
5252
import org.apache.commons.io.FileUtils;
53-
import org.apache.commons.io.IOUtils;
5453
import org.apache.commons.io.input.BoundedInputStream;
5554
import org.apache.commons.lang3.stream.Streams;
5655
import org.slf4j.Logger;
@@ -71,14 +70,10 @@ public class MultipartStore extends StoreBase {
7170
* Any method modifying the underlying file must acquire the lock object before the modification.
7271
*/
7372
private final Map<UUID, Object> lockStore = new ConcurrentHashMap<>();
74-
private final boolean retainFilesOnExit;
7573
private final ObjectStore objectStore;
7674
private final ObjectMapper objectMapper;
7775

78-
public MultipartStore(boolean retainFilesOnExit,
79-
ObjectStore objectStore,
80-
ObjectMapper objectMapper) {
81-
this.retainFilesOnExit = retainFilesOnExit;
76+
public MultipartStore(ObjectStore objectStore, ObjectMapper objectMapper) {
8277
this.objectStore = objectStore;
8378
this.objectMapper = objectMapper;
8479
}
@@ -217,9 +212,7 @@ public String putPart(BucketMetadata bucket,
217212
String partNumber,
218213
Path path,
219214
Map<String, String> encryptionHeaders) {
220-
var file = inputPathToFile(path,
221-
getPartPath(bucket, uploadId, partNumber),
222-
retainFilesOnExit);
215+
var file = inputPathToFile(path, getPartPath(bucket, uploadId, partNumber));
223216

224217
return hexDigest(encryptionHeaders.get(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID), file);
225218
}
@@ -255,29 +248,33 @@ public CompleteMultipartUploadResult completeMultipartUpload(BucketMetadata buck
255248
)
256249
.toList();
257250
Path tempFile = null;
258-
try (var inputStream = toInputStream(partsPaths)) {
251+
try {
259252
tempFile = Files.createTempFile("completeMultipartUpload", "");
260-
inputStream.transferTo(Files.newOutputStream(tempFile));
261-
var checksumFor = checksumFor(partsPaths, uploadInfo);
262-
var etag = hexDigestMultipart(partsPaths);
263-
objectStore.storeS3ObjectMetadata(bucket,
264-
id,
265-
key,
266-
uploadInfo.contentType(),
267-
uploadInfo.storeHeaders(),
268-
tempFile,
269-
uploadInfo.userMetadata(),
270-
encryptionHeaders,
271-
etag,
272-
Collections.emptyList(), //TODO: no tags for multi part uploads?
273-
uploadInfo.checksumAlgorithm(),
274-
checksumFor,
275-
uploadInfo.upload().owner(),
276-
uploadInfo.storageClass()
277-
);
278-
FileUtils.deleteDirectory(partFolder.toFile());
279-
return new CompleteMultipartUploadResult(location, uploadInfo.bucket(),
280-
key, etag, uploadInfo, checksumFor);
253+
254+
try (var is = toInputStream(partsPaths);
255+
var os = newOutputStream(tempFile)) {
256+
is.transferTo(os);
257+
var checksumFor = checksumFor(partsPaths, uploadInfo);
258+
var etag = hexDigestMultipart(partsPaths);
259+
objectStore.storeS3ObjectMetadata(bucket,
260+
id,
261+
key,
262+
uploadInfo.contentType(),
263+
uploadInfo.storeHeaders(),
264+
tempFile,
265+
uploadInfo.userMetadata(),
266+
encryptionHeaders,
267+
etag,
268+
Collections.emptyList(), //TODO: no tags for multi part uploads?
269+
uploadInfo.checksumAlgorithm(),
270+
checksumFor,
271+
uploadInfo.upload().owner(),
272+
uploadInfo.storageClass()
273+
);
274+
FileUtils.deleteDirectory(partFolder.toFile());
275+
return new CompleteMultipartUploadResult(location, uploadInfo.bucket(),
276+
key, etag, uploadInfo, checksumFor);
277+
}
281278
} catch (IOException e) {
282279
throw new IllegalStateException(String.format(
283280
"Error finishing multipart upload bucket=%s, key=%s, id=%s, uploadId=%s",
@@ -387,13 +384,13 @@ private String copyPartToFile(BucketMetadata bucket,
387384
var targetStream = newOutputStream(partFile.toPath())) {
388385
var skip = sourceStream.skip(from);
389386
if (skip == from) {
390-
IOUtils.copy(BoundedInputStream
387+
try (var bis = BoundedInputStream
391388
.builder()
392389
.setInputStream(sourceStream)
393390
.setMaxCount(len)
394-
.get(),
395-
targetStream
396-
);
391+
.get()) {
392+
bis.transferTo(targetStream);
393+
}
397394
} else {
398395
throw new IllegalStateException("Could not skip exact byte range");
399396
}
@@ -447,11 +444,7 @@ private void verifyMultipartUploadPreparation(BucketMetadata bucket, UUID id, St
447444

448445
private boolean createPartsFolder(BucketMetadata bucket, String uploadId) {
449446
var partsFolder = getPartsFolder(bucket, uploadId).toFile();
450-
var created = partsFolder.mkdirs();
451-
if (created && !retainFilesOnExit) {
452-
partsFolder.deleteOnExit();
453-
}
454-
return created;
447+
return partsFolder.mkdirs();
455448
}
456449

457450
private Path getMultipartsFolder(BucketMetadata bucket) {
@@ -490,9 +483,6 @@ private void writeMetafile(BucketMetadata bucket, MultipartUploadInfo uploadInfo
490483
try {
491484
synchronized (lockStore.get(UUID.fromString(uploadId))) {
492485
var metaFile = getUploadMetadataPath(bucket, uploadId).toFile();
493-
if (!retainFilesOnExit) {
494-
metaFile.deleteOnExit();
495-
}
496486
objectMapper.writeValue(metaFile, uploadInfo);
497487
}
498488
} catch (IOException e) {

‎server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,12 @@ public class ObjectStore extends StoreBase {
6363
*/
6464
private final Map<UUID, Object> lockStore = new ConcurrentHashMap<>();
6565

66-
private final boolean retainFilesOnExit;
6766
private final DateTimeFormatter s3ObjectDateFormat;
6867

6968
private final ObjectMapper objectMapper;
7069

71-
public ObjectStore(boolean retainFilesOnExit,
72-
DateTimeFormatter s3ObjectDateFormat,
70+
public ObjectStore(DateTimeFormatter s3ObjectDateFormat,
7371
ObjectMapper objectMapper) {
74-
this.retainFilesOnExit = retainFilesOnExit;
7572
this.s3ObjectDateFormat = s3ObjectDateFormat;
7673
this.objectMapper = objectMapper;
7774
}
@@ -109,7 +106,7 @@ public S3ObjectMetadata storeS3ObjectMetadata(BucketMetadata bucket,
109106
lockStore.putIfAbsent(id, new Object());
110107
synchronized (lockStore.get(id)) {
111108
createObjectRootFolder(bucket, id);
112-
var dataFile = inputPathToFile(path, getDataFilePath(bucket, id), retainFilesOnExit);
109+
var dataFile = inputPathToFile(path, getDataFilePath(bucket, id));
113110
var now = Instant.now();
114111
var s3ObjectMetadata = new S3ObjectMetadata(
115112
id,
@@ -445,9 +442,7 @@ void loadObjects(BucketMetadata bucketMetadata, Collection<UUID> ids) {
445442
*/
446443
private void createObjectRootFolder(BucketMetadata bucket, UUID id) {
447444
var objectRootFolder = getObjectFolderPath(bucket, id).toFile();
448-
if (objectRootFolder.mkdirs() && !retainFilesOnExit) {
449-
objectRootFolder.deleteOnExit();
450-
}
445+
objectRootFolder.mkdirs();
451446
}
452447

453448
private Path getObjectFolderPath(BucketMetadata bucket, UUID id) {
@@ -471,9 +466,6 @@ private void writeMetafile(BucketMetadata bucket, S3ObjectMetadata s3ObjectMetad
471466
try {
472467
synchronized (lockStore.get(id)) {
473468
var metaFile = getMetaFilePath(bucket, id).toFile();
474-
if (!retainFilesOnExit) {
475-
metaFile.deleteOnExit();
476-
}
477469
objectMapper.writeValue(metaFile, s3ObjectMetadata);
478470
}
479471
} catch (IOException e) {
@@ -500,9 +492,6 @@ private void writeAclFile(BucketMetadata bucket, UUID id, AccessControlPolicy po
500492
try {
501493
synchronized (lockStore.get(id)) {
502494
var aclFile = getAclFilePath(bucket, id).toFile();
503-
if (!retainFilesOnExit) {
504-
aclFile.deleteOnExit();
505-
}
506495
FileUtils.write(aclFile, objectMapper.writeValueAsString(policy), Charset.defaultCharset());
507496
}
508497
} catch (IOException e) {

‎server/src/main/java/com/adobe/testing/s3mock/store/StoreBase.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,10 @@ abstract class StoreBase {
3434
*
3535
* @return the newly created File.
3636
*/
37-
File inputPathToFile(Path inputPath, Path filePath, boolean retainFilesOnExit) {
37+
File inputPathToFile(Path inputPath, Path filePath) {
3838
var targetFile = filePath.toFile();
3939
try {
40-
if (targetFile.createNewFile() && (!retainFilesOnExit)) {
41-
targetFile.deleteOnExit();
42-
}
43-
40+
targetFile.createNewFile();
4441
try (var is = newInputStream(inputPath);
4542
var os = newOutputStream(targetFile.toPath())) {
4643
is.transferTo(os);

‎server/src/main/java/com/adobe/testing/s3mock/store/StoreCleaner.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright 2017-2024 Adobe.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.adobe.testing.s3mock.store;
18+
19+
import static org.apache.commons.io.FileUtils.cleanDirectory;
20+
import static org.apache.commons.io.FileUtils.deleteDirectory;
21+
import static org.apache.commons.io.FileUtils.isSymlink;
22+
23+
import java.io.File;
24+
import org.springframework.beans.factory.DisposableBean;
25+
26+
public class StoreCleaner implements DisposableBean {
27+
28+
private final File rootFolder;
29+
private final boolean retainFilesOnExit;
30+
31+
public StoreCleaner(File rootFolder, boolean retainFilesOnExit) {
32+
this.rootFolder = rootFolder;
33+
this.retainFilesOnExit = retainFilesOnExit;
34+
}
35+
36+
@Override
37+
public void destroy() throws Exception {
38+
if (!retainFilesOnExit && rootFolder.exists()) {
39+
cleanDirectory(rootFolder);
40+
}
41+
}
42+
}

‎server/src/main/java/com/adobe/testing/s3mock/store/StoreConfiguration.java‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,7 @@ public class StoreConfiguration {
4646
@Bean
4747
ObjectStore objectStore(StoreProperties properties, List<String> bucketNames,
4848
BucketStore bucketStore, ObjectMapper objectMapper) {
49-
var objectStore = new ObjectStore(properties.retainFilesOnExit(),
50-
S3_OBJECT_DATE_FORMAT, objectMapper);
49+
var objectStore = new ObjectStore(S3_OBJECT_DATE_FORMAT, objectMapper);
5150
for (var bucketName : bucketNames) {
5251
var bucketMetadata = bucketStore.getBucketMetadata(bucketName);
5352
if (bucketMetadata != null) {
@@ -60,8 +59,7 @@ ObjectStore objectStore(StoreProperties properties, List<String> bucketNames,
6059
@Bean
6160
BucketStore bucketStore(StoreProperties properties, File rootFolder, List<String> bucketNames,
6261
ObjectMapper objectMapper) {
63-
var bucketStore = new BucketStore(rootFolder, properties.retainFilesOnExit(),
64-
S3_OBJECT_DATE_FORMAT, objectMapper);
62+
var bucketStore = new BucketStore(rootFolder, S3_OBJECT_DATE_FORMAT, objectMapper);
6563
//load existing buckets first
6664
bucketStore.loadBuckets(bucketNames);
6765

@@ -112,7 +110,7 @@ List<String> bucketNames(File rootFolder) {
112110
MultipartStore multipartStore(StoreProperties properties,
113111
ObjectStore objectStore,
114112
ObjectMapper objectMapper) {
115-
return new MultipartStore(properties.retainFilesOnExit(), objectStore, objectMapper);
113+
return new MultipartStore(objectStore, objectMapper);
116114
}
117115

118116
@Bean
@@ -151,11 +149,11 @@ File rootFolder(StoreProperties properties) {
151149
root.getAbsolutePath(), properties.retainFilesOnExit());
152150
}
153151
}
154-
155-
if (!properties.retainFilesOnExit()) {
156-
root.deleteOnExit();
157-
}
158-
159152
return root;
160153
}
154+
155+
@Bean
156+
StoreCleaner storeCleaner(File rootFolder, StoreProperties storeProperties) {
157+
return new StoreCleaner(rootFolder, storeProperties.retainFilesOnExit());
158+
}
161159
}

0 commit comments

Comments
 (0)