Skip to content

Commit 4d360f4

Browse files
ivandika3ivandika3
authored
HDDS-12061. Cleanup the allocated but uncommitted blocks for multipart upload (apache#8848)
1 parent 3a0e6d7 commit 4d360f4

File tree

12 files changed

+643
-98
lines changed

12 files changed

+643
-98
lines changed

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -351,16 +351,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
351351
// which will be deleted as RepeatedOmKeyInfo
352352
final OmKeyInfo pseudoKeyInfo = isHSync ? null
353353
: wrapUncommittedBlocksAsPseudoKey(uncommitted, omKeyInfo);
354-
if (pseudoKeyInfo != null) {
355-
long pseudoObjId = ozoneManager.getObjectIdFromTxId(trxnLogIndex);
356-
String delKeyName = omMetadataManager.getOzoneDeletePathKey(
357-
pseudoObjId, dbOzoneKey);
358-
if (null == oldKeyVersionsToDeleteMap) {
359-
oldKeyVersionsToDeleteMap = new HashMap<>();
360-
}
361-
oldKeyVersionsToDeleteMap.computeIfAbsent(delKeyName,
362-
key -> new RepeatedOmKeyInfo()).addOmKeyInfo(pseudoKeyInfo);
363-
}
354+
oldKeyVersionsToDeleteMap = addKeyInfoToDeleteMap(ozoneManager, trxnLogIndex, dbOzoneKey,
355+
pseudoKeyInfo, oldKeyVersionsToDeleteMap);
364356

365357
// Add to cache of open key table and key table.
366358
if (!isHSync) {

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.util.ArrayList;
4141
import java.util.Collections;
4242
import java.util.EnumSet;
43+
import java.util.HashMap;
4344
import java.util.HashSet;
4445
import java.util.Iterator;
4546
import java.util.List;
@@ -1174,6 +1175,21 @@ protected OmKeyInfo wrapUncommittedBlocksAsPseudoKey(
11741175
return pseudoKeyInfo;
11751176
}
11761177

1178+
protected static Map<String, RepeatedOmKeyInfo> addKeyInfoToDeleteMap(OzoneManager om,
1179+
long trxnLogIndex, String ozoneKey, OmKeyInfo keyInfo, Map<String, RepeatedOmKeyInfo> deleteMap) {
1180+
if (keyInfo == null) {
1181+
return deleteMap;
1182+
}
1183+
final long pseudoObjId = om.getObjectIdFromTxId(trxnLogIndex);
1184+
final String delKeyName = om.getMetadataManager().getOzoneDeletePathKey(pseudoObjId, ozoneKey);
1185+
if (deleteMap == null) {
1186+
deleteMap = new HashMap<>();
1187+
}
1188+
deleteMap.computeIfAbsent(delKeyName, key -> new RepeatedOmKeyInfo())
1189+
.addOmKeyInfo(keyInfo);
1190+
return deleteMap;
1191+
}
1192+
11771193
/**
11781194
* Remove blocks in-place from keysToBeFiltered that exist in referenceKey.
11791195
* <p>

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import com.google.common.annotations.VisibleForTesting;
2424
import java.io.IOException;
2525
import java.nio.file.InvalidPathException;
26+
import java.util.HashMap;
27+
import java.util.List;
2628
import java.util.Map;
2729
import java.util.stream.Collectors;
2830
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -39,6 +41,7 @@
3941
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
4042
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
4143
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
44+
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
4245
import org.apache.hadoop.ozone.om.request.key.OMKeyRequest;
4346
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
4447
import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
@@ -162,7 +165,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
162165

163166
// set the data size and location info list
164167
omKeyInfo.setDataSize(keyArgs.getDataSize());
165-
omKeyInfo.updateLocationInfoList(keyArgs.getKeyLocationsList().stream()
168+
List<OmKeyLocationInfo> uncommitted = omKeyInfo.updateLocationInfoList(
169+
keyArgs.getKeyLocationsList().stream()
166170
.map(OmKeyLocationInfo::getFromProtobuf)
167171
.collect(Collectors.toList()), true);
168172
// Set Modification time
@@ -220,16 +224,37 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
220224

221225
omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName);
222226

227+
// This map should contain maximum of two entries
228+
// 1. Overwritten part
229+
// 2. Uncommitted pseudo part key
230+
Map<String, RepeatedOmKeyInfo> keyVersionsToDeleteMap = null;
231+
223232
long correctedSpace = omKeyInfo.getReplicatedSize();
224233
if (null != oldPartKeyInfo) {
225234
OmKeyInfo partKeyToBeDeleted =
226235
OmKeyInfo.getFromProtobuf(oldPartKeyInfo.getPartKeyInfo());
227236
correctedSpace -= partKeyToBeDeleted.getReplicatedSize();
237+
RepeatedOmKeyInfo oldVerKeyInfo = getOldVersionsToCleanUp(partKeyToBeDeleted, trxnLogIndex);
238+
// Unlike normal key commit, we can reuse the objectID for MPU part key because MPU part key
239+
// always use a new object ID regardless whether there is an existing key.
240+
String delKeyName = omMetadataManager.getOzoneDeletePathKey(
241+
partKeyToBeDeleted.getObjectID(), multipartKey);
242+
243+
if (!oldVerKeyInfo.getOmKeyInfoList().isEmpty()) {
244+
keyVersionsToDeleteMap = new HashMap<>();
245+
keyVersionsToDeleteMap.put(delKeyName, oldVerKeyInfo);
246+
}
228247
}
229248
checkBucketQuotaInBytes(omMetadataManager, omBucketInfo,
230249
correctedSpace);
231250
omBucketInfo.incrUsedBytes(correctedSpace);
232251

252+
// let the uncommitted blocks pretend as key's old version blocks
253+
// which will be deleted as RepeatedOmKeyInfo
254+
final OmKeyInfo pseudoKeyInfo = wrapUncommittedBlocksAsPseudoKey(uncommitted, omKeyInfo);
255+
keyVersionsToDeleteMap = addKeyInfoToDeleteMap(ozoneManager, trxnLogIndex, ozoneKey,
256+
pseudoKeyInfo, keyVersionsToDeleteMap);
257+
233258
MultipartCommitUploadPartResponse.Builder commitResponseBuilder = MultipartCommitUploadPartResponse.newBuilder()
234259
.setPartName(partName);
235260
String eTag = omKeyInfo.getMetadata().get(OzoneConsts.ETAG);
@@ -238,7 +263,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
238263
}
239264
omResponse.setCommitMultiPartUploadResponse(commitResponseBuilder);
240265
omClientResponse =
241-
getOmClientResponse(ozoneManager, oldPartKeyInfo, openKey,
266+
getOmClientResponse(ozoneManager, keyVersionsToDeleteMap, openKey,
242267
omKeyInfo, multipartKey, multipartKeyInfo, omResponse.build(),
243268
omBucketInfo.copyObject());
244269

@@ -247,7 +272,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
247272
result = Result.FAILURE;
248273
exception = ex;
249274
omClientResponse =
250-
getOmClientResponse(ozoneManager, oldPartKeyInfo, openKey,
275+
getOmClientResponse(ozoneManager, null, openKey,
251276
omKeyInfo, multipartKey, multipartKeyInfo,
252277
createErrorOMResponse(omResponse, exception), copyBucketInfo);
253278
} finally {
@@ -275,14 +300,13 @@ public static String getPartName(String ozoneKey, String uploadID,
275300

276301
@SuppressWarnings("checkstyle:ParameterNumber")
277302
protected S3MultipartUploadCommitPartResponse getOmClientResponse(
278-
OzoneManager ozoneManager,
279-
OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo, String openKey,
280-
OmKeyInfo omKeyInfo, String multipartKey,
303+
OzoneManager ozoneManager, Map<String, RepeatedOmKeyInfo> keyToDeleteMap,
304+
String openKey, OmKeyInfo omKeyInfo, String multipartKey,
281305
OmMultipartKeyInfo multipartKeyInfo, OMResponse build,
282306
OmBucketInfo omBucketInfo) {
283307

284308
return new S3MultipartUploadCommitPartResponse(build, multipartKey, openKey,
285-
multipartKeyInfo, oldPartKeyInfo, omKeyInfo,
309+
multipartKeyInfo, keyToDeleteMap, omKeyInfo,
286310
omBucketInfo, getBucketLayout());
287311
}
288312

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequestWithFSO.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@
1818
package org.apache.hadoop.ozone.om.request.s3.multipart;
1919

2020
import java.io.IOException;
21+
import java.util.Map;
2122
import org.apache.hadoop.ozone.om.OMMetadataManager;
2223
import org.apache.hadoop.ozone.om.OzoneManager;
2324
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
2425
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
2526
import org.apache.hadoop.ozone.om.helpers.OmFSOFile;
2627
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
2728
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
29+
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
2830
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
2931
import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCommitPartResponse;
3032
import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCommitPartResponseWithFSO;
@@ -67,13 +69,13 @@ protected OmKeyInfo getOmKeyInfo(OMMetadataManager omMetadataManager,
6769
@SuppressWarnings("checkstyle:ParameterNumber")
6870
protected S3MultipartUploadCommitPartResponse getOmClientResponse(
6971
OzoneManager ozoneManager,
70-
OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo, String openKey,
72+
Map<String, RepeatedOmKeyInfo> keyToDeleteMap, String openKey,
7173
OmKeyInfo omKeyInfo, String multipartKey,
7274
OmMultipartKeyInfo multipartKeyInfo,
7375
OzoneManagerProtocolProtos.OMResponse build, OmBucketInfo omBucketInfo) {
7476

7577
return new S3MultipartUploadCommitPartResponseWithFSO(build, multipartKey,
76-
openKey, multipartKeyInfo, oldPartKeyInfo, omKeyInfo,
78+
openKey, multipartKeyInfo, keyToDeleteMap, omKeyInfo,
7779
omBucketInfo, getBucketLayout());
7880
}
7981
}

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponse.java‎

Lines changed: 22 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@
2424
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.NO_SUCH_MULTIPART_UPLOAD_ERROR;
2525
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
2626

27+
import com.google.common.annotations.VisibleForTesting;
2728
import jakarta.annotation.Nonnull;
2829
import jakarta.annotation.Nullable;
2930
import java.io.IOException;
31+
import java.util.Map;
3032
import org.apache.hadoop.hdds.utils.db.BatchOperation;
3133
import org.apache.hadoop.ozone.OmUtils;
3234
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -37,7 +39,6 @@
3739
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
3840
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
3941
import org.apache.hadoop.ozone.om.response.key.OmKeyResponse;
40-
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
4142
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
4243

4344
/**
@@ -47,39 +48,32 @@
4748
MULTIPART_INFO_TABLE, BUCKET_TABLE})
4849
public class S3MultipartUploadCommitPartResponse extends OmKeyResponse {
4950

50-
private String multipartKey;
51-
private String openKey;
52-
private OmMultipartKeyInfo omMultipartKeyInfo;
53-
private OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo;
54-
private OmKeyInfo openPartKeyInfoToBeDeleted;
55-
private OmBucketInfo omBucketInfo;
51+
private final String multipartKey;
52+
private final String openKey;
53+
private final OmMultipartKeyInfo omMultipartKeyInfo;
54+
private final Map<String, RepeatedOmKeyInfo> keyToDeleteMap;
55+
private final OmKeyInfo openPartKeyInfoToBeDeleted;
56+
private final OmBucketInfo omBucketInfo;
5657

5758
/**
5859
* Regular response.
5960
* 1. Update MultipartKey in MultipartInfoTable with new PartKeyInfo
6061
* 2. Delete openKey from OpenKeyTable
61-
* 3. If old PartKeyInfo exists, put it in DeletedKeyTable
62-
* @param omResponse
63-
* @param multipartKey
64-
* @param openKey
65-
* @param omMultipartKeyInfo
66-
* @param oldPartKeyInfo
67-
* @param openPartKeyInfoToBeDeleted
68-
* @param omBucketInfo
62+
* 3. If old key or uncommitted (pseudo) key exists, put it in DeletedTable
6963
*/
7064
@SuppressWarnings("checkstyle:ParameterNumber")
7165
public S3MultipartUploadCommitPartResponse(@Nonnull OMResponse omResponse,
7266
String multipartKey, String openKey,
7367
@Nullable OmMultipartKeyInfo omMultipartKeyInfo,
74-
@Nullable OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo,
68+
@Nullable Map<String, RepeatedOmKeyInfo> keyToDeleteMap,
7569
@Nullable OmKeyInfo openPartKeyInfoToBeDeleted,
7670
@Nonnull OmBucketInfo omBucketInfo,
7771
@Nonnull BucketLayout bucketLayout) {
7872
super(omResponse, bucketLayout);
7973
this.multipartKey = multipartKey;
8074
this.openKey = openKey;
8175
this.omMultipartKeyInfo = omMultipartKeyInfo;
82-
this.oldPartKeyInfo = oldPartKeyInfo;
76+
this.keyToDeleteMap = keyToDeleteMap;
8377
this.openPartKeyInfoToBeDeleted = openPartKeyInfoToBeDeleted;
8478
this.omBucketInfo = omBucketInfo;
8579
}
@@ -112,29 +106,12 @@ public void checkAndUpdateDB(OMMetadataManager omMetadataManager,
112106
@Override
113107
public void addToDBBatch(OMMetadataManager omMetadataManager,
114108
BatchOperation batchOperation) throws IOException {
115-
116-
// If we have old part info:
117-
// Need to do 3 steps:
118-
// 0. Strip GDPR related metadata from multipart info
119-
// 1. add old part to delete table
120-
// 2. Commit multipart info which has information about this new part.
121-
// 3. delete this new part entry from open key table.
122-
123-
// This means for this multipart upload part upload, we have an old
124-
// part information, so delete it.
125-
if (oldPartKeyInfo != null) {
126-
OmKeyInfo partKeyToBeDeleted =
127-
OmKeyInfo.getFromProtobuf(oldPartKeyInfo.getPartKeyInfo());
128-
129-
RepeatedOmKeyInfo repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(
130-
partKeyToBeDeleted,
131-
omMultipartKeyInfo.getUpdateID());
132-
// multi-part key format is volumeName/bucketName/keyName/uploadId
133-
String deleteKey = omMetadataManager.getOzoneDeletePathKey(
134-
partKeyToBeDeleted.getObjectID(), multipartKey);
135-
136-
omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
137-
deleteKey, repeatedOmKeyInfo);
109+
// Delete old (overwritten) part upload and uncommitted parts
110+
if (keyToDeleteMap != null) {
111+
for (Map.Entry<String, RepeatedOmKeyInfo> entry : keyToDeleteMap.entrySet()) {
112+
omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
113+
entry.getKey(), entry.getValue());
114+
}
138115
}
139116

140117
omMetadataManager.getMultipartInfoTable().putWithBatch(batchOperation,
@@ -150,5 +127,10 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
150127
omMetadataManager.getBucketKey(omBucketInfo.getVolumeName(),
151128
omBucketInfo.getBucketName()), omBucketInfo);
152129
}
130+
131+
@VisibleForTesting
132+
public Map<String, RepeatedOmKeyInfo> getKeyToDelete() {
133+
return keyToDeleteMap;
134+
}
153135
}
154136

‎hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCommitPartResponseWithFSO.java‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,51 +17,46 @@
1717

1818
package org.apache.hadoop.ozone.om.response.s3.multipart;
1919

20+
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE;
2021
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
2122
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.MULTIPART_INFO_TABLE;
2223
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.OPEN_FILE_TABLE;
2324

2425
import jakarta.annotation.Nonnull;
2526
import jakarta.annotation.Nullable;
27+
import java.util.Map;
2628
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
2729
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
2830
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
2931
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
32+
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
3033
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
31-
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
3234
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
3335

3436
/**
3537
* Response for S3MultipartUploadCommitPartWithFSO request.
3638
*/
3739
@CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE, DELETED_TABLE,
38-
MULTIPART_INFO_TABLE})
40+
MULTIPART_INFO_TABLE, BUCKET_TABLE})
3941
public class S3MultipartUploadCommitPartResponseWithFSO
4042
extends S3MultipartUploadCommitPartResponse {
4143

4244
/**
4345
* Regular response.
4446
* 1. Update MultipartKey in MultipartInfoTable with new PartKeyInfo
4547
* 2. Delete openKey from OpenKeyTable
46-
* 3. If old PartKeyInfo exists, put it in DeletedKeyTable
47-
* @param omResponse
48-
* @param multipartKey
49-
* @param openKey
50-
* @param omMultipartKeyInfo
51-
* @param oldPartKeyInfo
52-
* @param openPartKeyInfoToBeDeleted
53-
* @param omBucketInfo
48+
* 3. If old key or uncommitted (pseudo) key exists, put it in DeletedTable
5449
*/
5550
@SuppressWarnings("checkstyle:ParameterNumber")
5651
public S3MultipartUploadCommitPartResponseWithFSO(
5752
@Nonnull OMResponse omResponse, String multipartKey, String openKey,
5853
@Nullable OmMultipartKeyInfo omMultipartKeyInfo,
59-
@Nullable OzoneManagerProtocolProtos.PartKeyInfo oldPartKeyInfo,
54+
@Nullable Map<String, RepeatedOmKeyInfo> keyToDeleteMap,
6055
@Nullable OmKeyInfo openPartKeyInfoToBeDeleted,
6156
@Nonnull OmBucketInfo omBucketInfo, @Nonnull BucketLayout bucketLayout) {
6257

6358
super(omResponse, multipartKey, openKey, omMultipartKeyInfo,
64-
oldPartKeyInfo, openPartKeyInfoToBeDeleted,
59+
keyToDeleteMap, openPartKeyInfoToBeDeleted,
6560
omBucketInfo, bucketLayout);
6661
}
6762
}

0 commit comments

Comments
 (0)