Skip to content

Commit 088f0a2

Browse files
sandeepvinayaksandeepvinayak
authored
blobstore: fix GCP async client builder for configs (#123)
1 parent a57e09d commit 088f0a2

File tree

7 files changed

+788
-714
lines changed

7 files changed

+788
-714
lines changed

‎blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStore.java‎

Lines changed: 61 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,22 @@
1919
import com.google.common.io.ByteStreams;
2020
import com.salesforce.multicloudj.blob.driver.AbstractBlobStore;
2121
import com.salesforce.multicloudj.blob.driver.BlobIdentifier;
22-
import com.salesforce.multicloudj.common.provider.Provider;
2322
import com.salesforce.multicloudj.blob.driver.BlobInfo;
2423
import com.salesforce.multicloudj.blob.driver.BlobMetadata;
24+
import com.salesforce.multicloudj.blob.driver.BlobStoreBuilder;
2525
import com.salesforce.multicloudj.blob.driver.ByteArray;
2626
import com.salesforce.multicloudj.blob.driver.CopyRequest;
2727
import com.salesforce.multicloudj.blob.driver.CopyResponse;
28-
import com.salesforce.multicloudj.blob.driver.DownloadRequest;
29-
import com.salesforce.multicloudj.blob.driver.DownloadResponse;
3028
import com.salesforce.multicloudj.blob.driver.DirectoryDownloadRequest;
3129
import com.salesforce.multicloudj.blob.driver.DirectoryDownloadResponse;
3230
import com.salesforce.multicloudj.blob.driver.DirectoryUploadRequest;
3331
import com.salesforce.multicloudj.blob.driver.DirectoryUploadResponse;
34-
import com.salesforce.multicloudj.blob.driver.ListBlobsPageRequest;
35-
import com.salesforce.multicloudj.blob.driver.ListBlobsPageResponse;
32+
import com.salesforce.multicloudj.blob.driver.DownloadRequest;
33+
import com.salesforce.multicloudj.blob.driver.DownloadResponse;
3634
import com.salesforce.multicloudj.blob.driver.FailedBlobDownload;
3735
import com.salesforce.multicloudj.blob.driver.FailedBlobUpload;
36+
import com.salesforce.multicloudj.blob.driver.ListBlobsPageRequest;
37+
import com.salesforce.multicloudj.blob.driver.ListBlobsPageResponse;
3838
import com.salesforce.multicloudj.blob.driver.ListBlobsRequest;
3939
import com.salesforce.multicloudj.blob.driver.MultipartPart;
4040
import com.salesforce.multicloudj.blob.driver.MultipartUpload;
@@ -48,10 +48,10 @@
4848
import com.salesforce.multicloudj.common.exceptions.ResourceNotFoundException;
4949
import com.salesforce.multicloudj.common.exceptions.SubstrateSdkException;
5050
import com.salesforce.multicloudj.common.exceptions.UnknownException;
51-
import com.salesforce.multicloudj.common.exceptions.UnSupportedOperationException;
5251
import com.salesforce.multicloudj.common.gcp.CommonErrorCodeMapping;
5352
import com.salesforce.multicloudj.common.gcp.GcpConstants;
5453
import com.salesforce.multicloudj.common.gcp.GcpCredentialsProvider;
54+
import com.salesforce.multicloudj.common.provider.Provider;
5555
import lombok.Getter;
5656
import org.apache.http.HttpHost;
5757
import org.apache.http.client.config.RequestConfig;
@@ -65,22 +65,22 @@
6565
import java.io.IOException;
6666
import java.io.InputStream;
6767
import java.io.OutputStream;
68+
import java.lang.reflect.Method;
6869
import java.net.URL;
6970
import java.nio.channels.Channels;
7071
import java.nio.file.Files;
7172
import java.nio.file.Path;
73+
import java.nio.file.Paths;
7274
import java.util.ArrayList;
7375
import java.util.Collection;
76+
import java.util.Collections;
7477
import java.util.HashMap;
7578
import java.util.Iterator;
7679
import java.util.List;
7780
import java.util.Map;
78-
import java.util.Collections;
79-
import java.util.HashMap;
8081
import java.util.UUID;
8182
import java.util.concurrent.TimeUnit;
8283
import java.util.stream.Collectors;
83-
import java.nio.file.Paths;
8484

8585
/**
8686
* GCP implementation of BlobStore
@@ -89,10 +89,9 @@
8989
@AutoService(AbstractBlobStore.class)
9090
public class GcpBlobStore extends AbstractBlobStore<GcpBlobStore> {
9191

92-
private static final String TAG_PREFIX = "gcp-tag-";
93-
9492
private final Storage storage;
9593
private final GcpTransformer transformer;
94+
private static final String TAG_PREFIX = "gcp-tag-";
9695

9796
public GcpBlobStore() {
9897
this(new Builder(), null);
@@ -240,7 +239,7 @@ protected void doDelete(Collection<BlobIdentifier> objects) {
240239
List<BlobId> blobIds = objects.stream()
241240
.map(obj -> transformer.toBlobId(bucket, obj.getKey(), obj.getVersionId()))
242241
.collect(Collectors.toList());
243-
242+
244243
storage.delete(blobIds);
245244
}
246245

@@ -304,9 +303,9 @@ protected ListBlobsPageResponse doListPage(ListBlobsPageRequest request) {
304303
List<BlobInfo> blobs = new ArrayList<>();
305304
for (Blob blob : page.getValues()) {
306305
blobs.add(BlobInfo.builder()
307-
.withKey(blob.getName())
308-
.withObjectSize(blob.getSize())
309-
.build());
306+
.withKey(blob.getName())
307+
.withObjectSize(blob.getSize())
308+
.build());
310309
}
311310

312311
return new ListBlobsPageResponse(
@@ -442,40 +441,30 @@ protected URL doGeneratePresignedUrl(PresignedUrlRequest request) {
442441
httpMethod = HttpMethod.GET;
443442
break;
444443
}
445-
List<Storage.SignUrlOption> options = new ArrayList<>();
446-
options.add(Storage.SignUrlOption.httpMethod(httpMethod));
447-
options.add(Storage.SignUrlOption.withV4Signature());
448-
449-
// Add KMS encryption header if specified
450-
if (request.getKmsKeyId() != null && !request.getKmsKeyId().isEmpty()) {
451-
Map<String, String> extHeaders = new HashMap<>();
452-
extHeaders.put("x-goog-encryption-kms-key-name", request.getKmsKeyId());
453-
options.add(Storage.SignUrlOption.withExtHeaders(extHeaders));
454-
}
455-
456444
return storage.signUrl(blobInfo,
457445
request.getDuration().toMillis(),
458446
TimeUnit.MILLISECONDS,
459-
options.toArray(new Storage.SignUrlOption[0]));
447+
Storage.SignUrlOption.httpMethod(httpMethod),
448+
Storage.SignUrlOption.withV4Signature());
460449
}
461450

462451
@Override
463452
protected boolean doDoesObjectExist(String key, String versionId) {
464453
return storage.get(transformer.toBlobId(key, versionId)) != null;
465454
}
466455

467-
/**
456+
/**
468457
* Maximum number of objects that can be deleted in a single batch operation.
469458
* GCP supports up to 1000 objects per batch delete.
470459
*/
471460
private static final int MAX_OBJECTS_PER_BATCH_DELETE = 1000;
472461

462+
@Override
473463
protected DirectoryUploadResponse doUploadDirectory(DirectoryUploadRequest directoryUploadRequest) {
474464
try {
475465
Path sourceDir = Paths.get(directoryUploadRequest.getLocalSourceDirectory());
476466
List<Path> filePaths = transformer.toFilePaths(directoryUploadRequest);
477467
List<FailedBlobUpload> failedUploads = new ArrayList<>();
478-
479468
// Create directory marker object if prefix is specified
480469
if (directoryUploadRequest.getPrefix() != null && !directoryUploadRequest.getPrefix().isEmpty()) {
481470
try {
@@ -515,6 +504,7 @@ protected DirectoryUploadResponse doUploadDirectory(DirectoryUploadRequest direc
515504
}
516505
}
517506

507+
@Override
518508
protected DirectoryDownloadResponse doDownloadDirectory(DirectoryDownloadRequest req) {
519509
try {
520510
Path targetDir = Paths.get(req.getLocalDestinationDirectory());
@@ -569,6 +559,7 @@ protected DirectoryDownloadResponse doDownloadDirectory(DirectoryDownloadRequest
569559
}
570560
}
571561

562+
@Override
572563
protected void doDeleteDirectory(String prefix) {
573564
try {
574565
// List all blobs with the given prefix and delete them in batches
@@ -644,6 +635,47 @@ public Builder withTransformerSupplier(GcpTransformerSupplier transformerSupplie
644635
return this;
645636
}
646637

638+
/**
639+
* Copies all configuration from another BlobStoreBuilder using reflection.
640+
* This automatically handles all fields without needing manual updates when new configs are added.
641+
* @param source The source builder to copy from
642+
* @return An instance of self
643+
*/
644+
public Builder copyFrom(BlobStoreBuilder<?> source) {
645+
try {
646+
// Find all "with*" methods in this builder
647+
Method[] methods = BlobStoreBuilder.class.getDeclaredMethods();
648+
649+
for (Method method : methods) {
650+
String methodName = method.getName();
651+
652+
// Look for "with*" setter methods
653+
if (methodName.startsWith("with") && method.getParameterCount() == 1) {
654+
// Extract property name (e.g., "withBucket" -> "Bucket")
655+
String propertyName = methodName.substring(4);
656+
657+
// Try to find corresponding getter (e.g., "getBucket")
658+
String getterName = "get" + propertyName;
659+
660+
try {
661+
Method getter = BlobStoreBuilder.class.getMethod(getterName);
662+
Object value = getter.invoke(source);
663+
664+
// Only copy non-null values
665+
if (value != null) {
666+
method.invoke(this, value);
667+
}
668+
} catch (NoSuchMethodException e) {
669+
// Getter doesn't exist, skip this property
670+
}
671+
}
672+
}
673+
} catch (Exception e) {
674+
throw new RuntimeException("Failed to copy builder configuration", e);
675+
}
676+
return this;
677+
}
678+
647679
/**
648680
* Helper function for generating the Storage client
649681
*/

‎blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpTransformer.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,13 @@
55
import com.google.cloud.storage.BlobId;
66
import com.google.cloud.storage.BlobInfo;
77
import com.google.cloud.storage.Storage;
8+
import com.google.cloud.storage.StorageClass;
89
import com.google.common.collect.ImmutableMap;
910
import com.salesforce.multicloudj.blob.driver.BlobIdentifier;
1011
import com.salesforce.multicloudj.blob.driver.BlobMetadata;
1112
import com.salesforce.multicloudj.blob.driver.CopyRequest;
12-
import com.salesforce.multicloudj.blob.driver.DirectoryUploadRequest;
1313
import com.salesforce.multicloudj.blob.driver.CopyResponse;
14+
import com.salesforce.multicloudj.blob.driver.DirectoryUploadRequest;
1415
import com.salesforce.multicloudj.blob.driver.DownloadRequest;
1516
import com.salesforce.multicloudj.blob.driver.DownloadResponse;
1617
import com.salesforce.multicloudj.blob.driver.ListBlobsPageRequest;
@@ -20,25 +21,23 @@
2021
import com.salesforce.multicloudj.blob.driver.UploadPartResponse;
2122
import com.salesforce.multicloudj.blob.driver.UploadRequest;
2223
import com.salesforce.multicloudj.blob.driver.UploadResponse;
23-
import com.google.cloud.storage.StorageClass;
24+
import com.salesforce.multicloudj.common.exceptions.InvalidArgumentException;
2425
import com.salesforce.multicloudj.common.exceptions.UnSupportedOperationException;
2526
import com.salesforce.multicloudj.common.util.HexUtil;
2627
import lombok.Getter;
2728
import org.apache.commons.lang3.tuple.ImmutablePair;
2829
import org.apache.commons.lang3.tuple.Pair;
29-
import com.salesforce.multicloudj.common.exceptions.InvalidArgumentException;
30-
31-
import java.io.InputStream;
32-
import java.util.Collections;
33-
import java.util.List;
34-
import java.util.Map;
35-
import java.util.stream.Collectors;
3630

3731
import java.io.IOException;
32+
import java.io.InputStream;
3833
import java.nio.file.Files;
3934
import java.nio.file.Path;
4035
import java.nio.file.Paths;
4136
import java.util.ArrayList;
37+
import java.util.Collections;
38+
import java.util.List;
39+
import java.util.Map;
40+
import java.util.stream.Collectors;
4241
import java.util.stream.Stream;
4342

4443
@Getter

‎blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/async/GcpAsyncBlobStore.java‎

Lines changed: 6 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -10,26 +10,7 @@
1010
import com.salesforce.multicloudj.common.gcp.GcpConstants;
1111
import lombok.Getter;
1212

13-
import com.google.cloud.storage.Blob;
14-
import com.google.cloud.storage.BlobId;
15-
import com.google.cloud.storage.BlobInfo;
16-
import com.salesforce.multicloudj.blob.driver.DirectoryDownloadRequest;
17-
import com.salesforce.multicloudj.blob.driver.DirectoryDownloadResponse;
18-
import com.salesforce.multicloudj.blob.driver.DirectoryUploadRequest;
19-
import com.salesforce.multicloudj.blob.driver.DirectoryUploadResponse;
20-
import com.salesforce.multicloudj.blob.driver.FailedBlobDownload;
21-
import com.salesforce.multicloudj.blob.driver.FailedBlobUpload;
22-
23-
import java.nio.file.Files;
24-
import java.nio.file.Path;
25-
import java.nio.file.Paths;
26-
import java.util.ArrayList;
27-
import java.util.List;
28-
import java.util.concurrent.CompletableFuture;
2913
import java.util.concurrent.ExecutorService;
30-
import java.util.concurrent.Executors;
31-
import java.util.concurrent.TimeUnit;
32-
import java.util.stream.Collectors;
3314

3415
/**
3516
* GCP implementation of AsyncBlobStore
@@ -55,8 +36,8 @@ public GcpAsyncBlobStore(AbstractBlobStore<?> blobStore, ExecutorService executo
5536
this.transformerSupplier = transformerSupplier;
5637
}
5738

58-
public static GcpAsyncBlobStore.Builder builder() {
59-
return new GcpAsyncBlobStore.Builder();
39+
public static Builder builder() {
40+
return new Builder();
6041
}
6142

6243
@Getter
@@ -87,25 +68,15 @@ public Builder withTransformerSupplier(GcpTransformerSupplier transformerSupplie
8768

8869
@Override
8970
public GcpAsyncBlobStore build() {
90-
GcpBlobStore blobStore = getGcpBlobStore();
71+
GcpBlobStore blobStore = gcpBlobStore;
9172
if(blobStore == null) {
9273
blobStore = new GcpBlobStore.Builder()
93-
.withStorage(getStorage())
94-
.withTransformerSupplier(getTransformerSupplier())
95-
.withBucket(getBucket())
96-
.withCredentialsOverrider(getCredentialsOverrider())
97-
.withEndpoint(getEndpoint())
98-
.withIdleConnectionTimeout(getIdleConnectionTimeout())
99-
.withMaxConnections(getMaxConnections())
100-
.withProperties(getProperties())
101-
.withProxyEndpoint(getProxyEndpoint())
102-
.withRegion(getRegion())
103-
.withSocketTimeout(getSocketTimeout())
104-
.withValidator(getValidator())
74+
.copyFrom(this)
75+
.withStorage(storage)
76+
.withTransformerSupplier(transformerSupplier)
10577
.build();
10678
}
10779
return new GcpAsyncBlobStore(blobStore, getExecutorService(), storage, transformerSupplier);
10880
}
10981
}
110-
11182
}

‎blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStoreIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public AbstractBlobStore<?> createBlobStore(boolean useValidBucket, boolean useV
4444
// Live recording path – rely on real ADC
4545
try {
4646
Credentials credentials = GoogleCredentials.getApplicationDefault();
47-
return createBlobStore(bucketNameToUse, credentials);
47+
return createBlobStore(bucketNameToUse, credentials);
4848
} catch (IOException e) {
4949
// Fallback to NoCredentials if unable to load application default credentials
5050
return createBlobStore(bucketNameToUse, NoCredentials.getInstance());

0 commit comments

Comments
 (0)