Skip to content

Commit 8ed142a

Browse files
sandeepvinayaksandeepvinayak
authored
blob store sync for all providers (#9)
1 parent f5844c6 commit 8ed142a

File tree

20 files changed

+1620
-315
lines changed

20 files changed

+1620
-315
lines changed

‎blob/blob-ali/src/main/java/com/salesforce/multicloudj/blob/ali/AliBlobStore.java‎

Lines changed: 114 additions & 225 deletions
Large diffs are not rendered by default.

‎blob/blob-ali/src/main/java/com/salesforce/multicloudj/blob/ali/AliTransformer.java‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
package com.salesforce.multicloudj.blob.ali;
2+
3+
import com.aliyun.oss.HttpMethod;
4+
import com.aliyun.oss.internal.OSSHeaders;
5+
import com.aliyun.oss.model.AbortMultipartUploadRequest;
6+
import com.aliyun.oss.model.CompleteMultipartUploadRequest;
7+
import com.aliyun.oss.model.CopyObjectRequest;
8+
import com.aliyun.oss.model.CopyObjectResult;
9+
import com.aliyun.oss.model.DeleteObjectsRequest;
10+
import com.aliyun.oss.model.DeleteVersionsRequest;
11+
import com.aliyun.oss.model.GeneratePresignedUrlRequest;
12+
import com.aliyun.oss.model.GenericRequest;
13+
import com.aliyun.oss.model.GetObjectRequest;
14+
import com.aliyun.oss.model.InitiateMultipartUploadRequest;
15+
import com.aliyun.oss.model.InitiateMultipartUploadResult;
16+
import com.aliyun.oss.model.ListPartsRequest;
17+
import com.aliyun.oss.model.OSSObject;
18+
import com.aliyun.oss.model.ObjectMetadata;
19+
import com.aliyun.oss.model.PartETag;
20+
import com.aliyun.oss.model.PartListing;
21+
import com.aliyun.oss.model.PartSummary;
22+
import com.aliyun.oss.model.PutObjectRequest;
23+
import com.aliyun.oss.model.PutObjectResult;
24+
import com.aliyun.oss.model.UploadPartRequest;
25+
import com.aliyun.oss.model.UploadPartResult;
26+
import com.salesforce.multicloudj.blob.driver.BlobIdentifier;
27+
import com.salesforce.multicloudj.blob.driver.BlobMetadata;
28+
import com.salesforce.multicloudj.blob.driver.CopyRequest;
29+
import com.salesforce.multicloudj.blob.driver.CopyResponse;
30+
import com.salesforce.multicloudj.blob.driver.DownloadRequest;
31+
import com.salesforce.multicloudj.blob.driver.DownloadResponse;
32+
import com.salesforce.multicloudj.blob.driver.MultipartPart;
33+
import com.salesforce.multicloudj.blob.driver.MultipartUpload;
34+
import com.salesforce.multicloudj.blob.driver.MultipartUploadRequest;
35+
import com.salesforce.multicloudj.blob.driver.PresignedUrlRequest;
36+
import com.salesforce.multicloudj.blob.driver.UploadPartResponse;
37+
import com.salesforce.multicloudj.blob.driver.UploadRequest;
38+
import com.salesforce.multicloudj.blob.driver.UploadResponse;
39+
import lombok.Getter;
40+
import org.apache.commons.lang3.tuple.ImmutablePair;
41+
import org.apache.commons.lang3.tuple.Pair;
42+
43+
import java.io.File;
44+
import java.io.InputStream;
45+
import java.time.Instant;
46+
import java.util.ArrayList;
47+
import java.util.Collection;
48+
import java.util.Comparator;
49+
import java.util.Date;
50+
import java.util.List;
51+
import java.util.Map;
52+
import java.util.stream.Collectors;
53+
54+
@Getter
55+
public class AliTransformer {
56+
57+
private final String bucket;
58+
59+
public AliTransformer(String bucket) {
60+
this.bucket = bucket;
61+
}
62+
63+
public PutObjectRequest toPutObjectRequest(UploadRequest uploadRequest, InputStream inputStream) {
64+
return new PutObjectRequest(bucket, uploadRequest.getKey(), inputStream, generateObjectMetadata(uploadRequest));
65+
}
66+
67+
public PutObjectRequest toPutObjectRequest(UploadRequest uploadRequest, File file) {
68+
return new PutObjectRequest(bucket, uploadRequest.getKey(), file, generateObjectMetadata(uploadRequest));
69+
}
70+
71+
protected ObjectMetadata generateObjectMetadata(UploadRequest uploadRequest) {
72+
ObjectMetadata metadata = new ObjectMetadata();
73+
metadata.setUserMetadata(uploadRequest.getMetadata());
74+
metadata.setObjectTagging(uploadRequest.getTags());
75+
return metadata;
76+
}
77+
78+
public UploadResponse toUploadResponse(UploadRequest uploadRequest, PutObjectResult result) {
79+
return UploadResponse.builder()
80+
.key(uploadRequest.getKey())
81+
.versionId(result.getVersionId())
82+
.eTag(result.getETag())
83+
.build();
84+
}
85+
86+
public GetObjectRequest toGetObjectRequest(DownloadRequest downloadRequest) {
87+
GetObjectRequest request = new GetObjectRequest(bucket, downloadRequest.getKey(), downloadRequest.getVersionId());
88+
if(downloadRequest.getStart() != null || downloadRequest.getEnd() != null) {
89+
Pair<Long, Long> range = computeRange(downloadRequest.getStart(), downloadRequest.getEnd());
90+
request.withRange(range.getLeft(), range.getRight());
91+
}
92+
return request;
93+
}
94+
95+
/**
96+
* Reading the first 500 bytes - computeRange(0, 500) -> (0, 500)
97+
* Reading a middle 500 bytes - computeRange(123, 623) -> (123, 623)
98+
* Reading the last 500 bytes - computeRange(null, 500) -> (-1, 500)
99+
* Reading everything but first 500 bytes - computeRange(500, null) -> (500, -1)
100+
*/
101+
protected Pair<Long, Long> computeRange(Long start, Long end) {
102+
return new ImmutablePair<>(start==null ? -1 : start, end==null ? -1 : end);
103+
}
104+
105+
public DownloadResponse toDownloadResponse(OSSObject ossObject) {
106+
return DownloadResponse.builder()
107+
.key(ossObject.getKey())
108+
.metadata(BlobMetadata.builder()
109+
.key(ossObject.getKey())
110+
.versionId(ossObject.getObjectMetadata().getVersionId())
111+
.eTag(ossObject.getObjectMetadata().getETag())
112+
.lastModified(ossObject.getObjectMetadata().getLastModified().toInstant())
113+
.metadata(ossObject.getObjectMetadata().getUserMetadata())
114+
.objectSize(ossObject.getObjectMetadata().getContentLength())
115+
.build())
116+
.build();
117+
}
118+
119+
public DeleteObjectsRequest toDeleteObjectsRequest(Collection<BlobIdentifier> objects) {
120+
return new DeleteObjectsRequest(bucket)
121+
.withKeys(
122+
objects.stream()
123+
.map(BlobIdentifier::getKey)
124+
.collect(Collectors.toList()));
125+
}
126+
127+
public DeleteVersionsRequest toDeleteVersionsRequest(Collection<BlobIdentifier> objects) {
128+
List<DeleteVersionsRequest.KeyVersion> objectsToDelete = new ArrayList<>();
129+
for(BlobIdentifier object : objects) {
130+
objectsToDelete.add(new DeleteVersionsRequest.KeyVersion(object.getKey(), object.getVersionId()));
131+
}
132+
return new DeleteVersionsRequest(bucket).withKeys(objectsToDelete);
133+
}
134+
135+
public CopyObjectRequest toCopyObjectRequest(CopyRequest request) {
136+
return new CopyObjectRequest(
137+
bucket,
138+
request.getSrcKey(),
139+
request.getSrcVersionId(),
140+
request.getDestBucket(),
141+
request.getDestKey());
142+
}
143+
144+
public CopyResponse toCopyResponse(String destKey, CopyObjectResult result) {
145+
return CopyResponse.builder()
146+
.key(destKey)
147+
.versionId(result.getVersionId())
148+
.eTag(result.getETag())
149+
.lastModified(result.getLastModified().toInstant())
150+
.build();
151+
}
152+
153+
public GenericRequest toMetadataRequest(String key, String versionId) {
154+
return new GenericRequest()
155+
.withBucketName(bucket)
156+
.withKey(key)
157+
.withVersionId(versionId);
158+
}
159+
160+
public BlobMetadata toBlobMetadata(String key, ObjectMetadata metadata) {
161+
long objectSize = metadata.getContentLength();
162+
Map<String, String> rawMetadata = metadata.getUserMetadata();
163+
return BlobMetadata.builder()
164+
.key(key)
165+
.versionId(metadata.getVersionId())
166+
.eTag(metadata.getETag())
167+
.objectSize(objectSize)
168+
.metadata(rawMetadata)
169+
.lastModified(metadata.getLastModified().toInstant())
170+
.build();
171+
}
172+
173+
public InitiateMultipartUploadRequest toInitiateMultipartUploadRequest(MultipartUploadRequest request) {
174+
ObjectMetadata metadata = new ObjectMetadata();
175+
metadata.setUserMetadata(request.getMetadata());
176+
return new InitiateMultipartUploadRequest(getBucket(), request.getKey(), metadata);
177+
}
178+
179+
public MultipartUpload toMultipartUpload(InitiateMultipartUploadResult initiateMultipartUploadResult) {
180+
return new MultipartUpload(
181+
initiateMultipartUploadResult.getBucketName(),
182+
initiateMultipartUploadResult.getKey(),
183+
initiateMultipartUploadResult.getUploadId());
184+
}
185+
186+
public UploadPartRequest toUploadPartRequest(MultipartUpload mpu, MultipartPart mpp){
187+
return new UploadPartRequest(
188+
getBucket(),
189+
mpu.getKey(),
190+
mpu.getId(),
191+
mpp.getPartNumber(),
192+
mpp.getInputStream(),
193+
mpp.getContentLength());
194+
}
195+
196+
public UploadPartResponse toUploadPartResponse(MultipartPart mpp, UploadPartResult uploadPartResult) {
197+
return new UploadPartResponse(mpp.getPartNumber(), uploadPartResult.getPartETag().getETag(), mpp.getContentLength());
198+
}
199+
200+
public CompleteMultipartUploadRequest toCompleteMultipartUploadRequest(MultipartUpload mpu, List<UploadPartResponse> parts) {
201+
List<PartETag> completedParts = parts.stream()
202+
.sorted(Comparator.comparingInt(UploadPartResponse::getPartNumber))
203+
.map(part -> new PartETag(part.getPartNumber(), part.getEtag()))
204+
.collect(Collectors.toList());
205+
return new CompleteMultipartUploadRequest(getBucket(), mpu.getKey(), mpu.getId(), completedParts);
206+
}
207+
208+
public ListPartsRequest toListPartsRequest(MultipartUpload mpu) {
209+
return new ListPartsRequest(bucket, mpu.getKey(), mpu.getId());
210+
}
211+
212+
public List<UploadPartResponse> toListUploadPartResponse(PartListing partListing) {
213+
return partListing.getParts().stream()
214+
.sorted(Comparator.comparingInt(PartSummary::getPartNumber))
215+
.map((part) -> new UploadPartResponse(part.getPartNumber(), part.getETag(), part.getSize()))
216+
.collect(Collectors.toList());
217+
}
218+
219+
public AbortMultipartUploadRequest toAbortMultipartUploadRequest(MultipartUpload mpu) {
220+
return new AbortMultipartUploadRequest(bucket, mpu.getKey(), mpu.getId());
221+
}
222+
223+
public GeneratePresignedUrlRequest toPresignedUrlUploadRequest(PresignedUrlRequest request) {
224+
Date expirationDate = Date.from(Instant.now().plus(request.getDuration()));
225+
GeneratePresignedUrlRequest presignedUrlRequest = new GeneratePresignedUrlRequest(getBucket(), request.getKey());
226+
presignedUrlRequest.setExpiration(expirationDate);
227+
presignedUrlRequest.setMethod(HttpMethod.PUT);
228+
presignedUrlRequest.setUserMetadata(request.getMetadata());
229+
230+
// Note: Tagging is not supported by default for OSS presigned uploads so we have to manually append it
231+
ObjectMetadata metadata = new ObjectMetadata();
232+
metadata.setObjectTagging(request.getTags());
233+
Object encodedTagging = metadata.getRawMetadata().get(OSSHeaders.OSS_TAGGING);
234+
if(encodedTagging instanceof String) {
235+
presignedUrlRequest.addHeader(OSSHeaders.OSS_TAGGING, (String)encodedTagging);
236+
}
237+
return presignedUrlRequest;
238+
}
239+
240+
public GeneratePresignedUrlRequest toPresignedUrlDownloadRequest(PresignedUrlRequest request) {
241+
Date expirationDate = Date.from(Instant.now().plus(request.getDuration()));
242+
GeneratePresignedUrlRequest presignedUrlRequest = new GeneratePresignedUrlRequest(getBucket(), request.getKey());
243+
presignedUrlRequest.setExpiration(expirationDate);
244+
presignedUrlRequest.setMethod(HttpMethod.GET);
245+
return presignedUrlRequest;
246+
}
247+
}

‎blob/blob-ali/src/main/java/com/salesforce/multicloudj/blob/ali/AliTransformerSupplier.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package com.salesforce.multicloudj.blob.ali;
2+
3+
/**
4+
* Super-simple class to provide instances of AliTransformer that are specific to a given bucket.
5+
* This helps remove the need to validate that the buckets line up between the client and their transformer.
6+
*/
7+
public class AliTransformerSupplier {
8+
9+
/**
10+
* Produces a {@link AliTransformer} specific to the supplied bucket
11+
* @param bucket the bucket to assign the transformer to.
12+
* @return the bucket-specific transformer
13+
*/
14+
public AliTransformer get(String bucket) {
15+
return new AliTransformer(bucket);
16+
}
17+
}

‎blob/blob-ali/src/test/java/com/salesforce/multicloudj/blob/ali/AliBlobStoreTest.java‎

Lines changed: 35 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import com.aliyun.oss.model.CompleteMultipartUploadResult;
1212
import com.aliyun.oss.model.CopyObjectRequest;
1313
import com.aliyun.oss.model.CopyObjectResult;
14+
import com.aliyun.oss.model.DeleteObjectsRequest;
1415
import com.aliyun.oss.model.DeleteVersionsRequest;
1516
import com.aliyun.oss.model.GeneratePresignedUrlRequest;
1617
import com.aliyun.oss.model.GenericRequest;
@@ -53,7 +54,6 @@
5354
import com.salesforce.multicloudj.sts.model.CredentialsOverrider;
5455
import com.salesforce.multicloudj.sts.model.CredentialsType;
5556
import com.salesforce.multicloudj.sts.model.StsCredentials;
56-
import org.apache.commons.lang3.tuple.Pair;
5757
import org.junit.jupiter.api.AfterEach;
5858
import org.junit.jupiter.api.Assertions;
5959
import org.junit.jupiter.api.BeforeEach;
@@ -78,7 +78,6 @@
7878
import java.util.List;
7979
import java.util.Map;
8080
import java.util.NoSuchElementException;
81-
import java.util.stream.Collectors;
8281
import java.util.stream.IntStream;
8382

8483
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -108,6 +107,7 @@ void setup() {
108107
staticMockBuilder.when(OSSClientBuilder::create).thenReturn(mockBuilder);
109108
when(mockBuilder.region(any())).thenReturn(mockBuilder);
110109
when(mockBuilder.endpoint(any())).thenReturn(mockBuilder);
110+
when(mockBuilder.clientConfiguration(any())).thenReturn(mockBuilder);
111111
when(mockBuilder.credentialsProvider(any())).thenReturn(mockBuilder);
112112
when(mockBuilder.build()).thenReturn(mockOssClient);
113113

@@ -198,25 +198,6 @@ void testDoUploadPath() throws IOException {
198198
verifyUploadTestResults(ali.doUpload(getTestUploadRequest(), path));
199199
}
200200

201-
@Test
202-
void testComputeRange() {
203-
Pair<Long, Long> result = ali.computeRange(0L, 500L);
204-
assertEquals(result.getLeft(), 0);
205-
assertEquals(result.getRight(), 500);
206-
207-
result = ali.computeRange(100L, 600L);
208-
assertEquals(result.getLeft(), 100);
209-
assertEquals(result.getRight(), 600);
210-
211-
result = ali.computeRange(null, 500L);
212-
assertEquals(result.getLeft(), -1);
213-
assertEquals(result.getRight(), 500);
214-
215-
result = ali.computeRange(500L, null);
216-
assertEquals(result.getLeft(), 500);
217-
assertEquals(result.getRight(), -1);
218-
}
219-
220201
void verifyUploadTestResults(UploadResponse uploadResponse) {
221202

222203
// Verify the parameters passed into the SDK
@@ -320,23 +301,48 @@ void testDoDelete() {
320301
assertEquals("bucket-1", bucketCaptor.getValue());
321302
assertEquals("object-1", keyCaptor.getValue());
322303
assertEquals("version-1", versionCaptor.getValue());
304+
305+
ali.doDelete("object-1", null);
306+
bucketCaptor = ArgumentCaptor.forClass(String.class);
307+
keyCaptor = ArgumentCaptor.forClass(String.class);
308+
verify(mockOssClient, times(1)).deleteObject(bucketCaptor.capture(), keyCaptor.capture());
309+
assertEquals("bucket-1", bucketCaptor.getValue());
310+
assertEquals("object-1", keyCaptor.getValue());
323311
}
324312

325313
@Test
326314
void testDoBulkDelete() {
327315
List<BlobIdentifier> objects = List.of(new BlobIdentifier("object-1","version-1"),
328-
new BlobIdentifier("object-2","version-2"),
329-
new BlobIdentifier("object-3","version-3"));
316+
new BlobIdentifier("object-2",null),
317+
new BlobIdentifier("object-3","version-3"),
318+
new BlobIdentifier("object-4",null));
330319
ali.doDelete(objects);
331320

321+
// Verify it sends a delete request for the objects that have versionIds
332322
ArgumentCaptor<DeleteVersionsRequest> deleteVersionsRequestCaptor = ArgumentCaptor.forClass(DeleteVersionsRequest.class);
333323
verify(mockOssClient, times(1)).deleteVersions(deleteVersionsRequestCaptor.capture());
334324
DeleteVersionsRequest actualDeleteVersionsRequest = deleteVersionsRequestCaptor.getValue();
335325
assertEquals("bucket-1", actualDeleteVersionsRequest.getBucketName());
336-
Map<String, String> objectsMap = objects.stream().collect(Collectors.toMap(BlobIdentifier::getKey, BlobIdentifier::getVersionId));
337-
for(DeleteVersionsRequest.KeyVersion key : actualDeleteVersionsRequest.getKeys()){
338-
assertEquals(objectsMap.get(key.getKey()), key.getVersion());
339-
}
326+
List<DeleteVersionsRequest.KeyVersion> keyVersions = actualDeleteVersionsRequest.getKeys();
327+
assertEquals(2, keyVersions.size());
328+
assertEquals("object-1", keyVersions.get(0).getKey());
329+
assertEquals("version-1", keyVersions.get(0).getVersion());
330+
assertEquals("object-3", keyVersions.get(1).getKey());
331+
assertEquals("version-3", keyVersions.get(1).getVersion());
332+
333+
// Verify it sends a delete request for the objects that don't have versionIds
334+
ArgumentCaptor<DeleteObjectsRequest> deleteObjectsRequestCaptor = ArgumentCaptor.forClass(DeleteObjectsRequest.class);
335+
verify(mockOssClient, times(1)).deleteObjects(deleteObjectsRequestCaptor.capture());
336+
DeleteObjectsRequest actualDeleteObjectsRequest = deleteObjectsRequestCaptor.getValue();
337+
List<String> keys = actualDeleteObjectsRequest.getKeys();
338+
assertEquals(2, keys.size());
339+
assertEquals("object-2", keys.get(0));
340+
assertEquals("object-4", keys.get(1));
341+
342+
// Test that edge cases are properly processed
343+
ali.doDelete(List.of(new BlobIdentifier("object-1","version-1")));
344+
ali.doDelete(List.of(new BlobIdentifier("object-1",null)));
345+
ali.doDelete(List.of());
340346
}
341347

342348
@Test
@@ -579,7 +585,7 @@ void testDoGeneratePresignedUploadUrl() {
579585
.duration(duration)
580586
.build();
581587

582-
ali.doGeneratePresignedUploadUrl(presignedUploadRequest);
588+
ali.doGeneratePresignedUrl(presignedUploadRequest);
583589

584590
ArgumentCaptor<GeneratePresignedUrlRequest> generatePresignedUrlRequestCaptor = ArgumentCaptor.forClass(GeneratePresignedUrlRequest.class);
585591
verify(mockOssClient, times(1)).generatePresignedUrl(generatePresignedUrlRequestCaptor.capture());
@@ -602,7 +608,7 @@ void testDoGeneratePresignedDownloadUrl() {
602608
.duration(duration)
603609
.build();
604610

605-
ali.doGeneratePresignedDownloadUrl(presignedDownloadRequest);
611+
ali.doGeneratePresignedUrl(presignedDownloadRequest);
606612

607613
ArgumentCaptor<GeneratePresignedUrlRequest> generatePresignedUrlRequestCaptor = ArgumentCaptor.forClass(GeneratePresignedUrlRequest.class);
608614
verify(mockOssClient, times(1)).generatePresignedUrl(generatePresignedUrlRequestCaptor.capture());

0 commit comments

Comments
 (0)