Skip to content

Commit f080175

Browse files
roseyang62roseyang62
authored
Merge branch 'main' into investigate-pubsub-conformance-test
2 parents 199d364 + a22c4b0 commit f080175

File tree

46 files changed

+275
-42
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+275
-42
lines changed

‎blob/blob-ali/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<parent>
1010
<groupId>com.salesforce.multicloudj</groupId>
1111
<artifactId>blob</artifactId>
12-
<version>0.2.26</version>
12+
<version>0.2.27-SNAPSHOT</version>
1313
<relativePath>../pom.xml</relativePath>
1414
</parent>
1515

‎blob/blob-aws/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<parent>
1010
<groupId>com.salesforce.multicloudj</groupId>
1111
<artifactId>blob</artifactId>
12-
<version>0.2.26</version>
12+
<version>0.2.27-SNAPSHOT</version>
1313
<relativePath>../pom.xml</relativePath>
1414
</parent>
1515

‎blob/blob-client/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<parent>
1010
<groupId>com.salesforce.multicloudj</groupId>
1111
<artifactId>blob</artifactId>
12-
<version>0.2.26</version>
12+
<version>0.2.27-SNAPSHOT</version>
1313
<relativePath>../pom.xml</relativePath>
1414
</parent>
1515

‎blob/blob-gcp/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<parent>
1010
<groupId>com.salesforce.multicloudj</groupId>
1111
<artifactId>blob</artifactId>
12-
<version>0.2.26</version>
12+
<version>0.2.27-SNAPSHOT</version>
1313
<relativePath>../pom.xml</relativePath>
1414
</parent>
1515

‎blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpBlobClient.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,11 @@ private static Storage buildStorage(Builder builder) {
115115
.build());
116116
}
117117

118+
if (builder.getRetryConfig() != null) {
119+
GcpTransformer transformer = new GcpTransformer(null);
120+
storageBuilder.setRetrySettings(transformer.toGcpRetrySettings(builder.getRetryConfig()));
121+
}
122+
118123
return storageBuilder.build().getService();
119124
}
120125

‎blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStore.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -993,6 +993,11 @@ private static Storage buildStorage(Builder builder) {
993993
storageOptionsBuilder.setCredentials(credentials);
994994
}
995995

996+
if (builder.getRetryConfig() != null) {
997+
GcpTransformer transformer = builder.transformerSupplier.get(builder.getBucket());
998+
storageOptionsBuilder.setRetrySettings(transformer.toGcpRetrySettings(builder.getRetryConfig()));
999+
}
1000+
9961001
return storageOptionsBuilder.build().getService();
9971002
}
9981003

@@ -1014,6 +1019,11 @@ private static MultipartUploadClient buildMultipartUploadClient(Builder builder)
10141019
storageOptionsBuilder.setCredentials(credentials);
10151020
}
10161021

1022+
if (builder.getRetryConfig() != null) {
1023+
GcpTransformer transformer = builder.transformerSupplier.get(builder.getBucket());
1024+
storageOptionsBuilder.setRetrySettings(transformer.toGcpRetrySettings(builder.getRetryConfig()));
1025+
}
1026+
10171027
return MultipartUploadClient.create(MultipartUploadSettings.of(storageOptionsBuilder.build()));
10181028
}
10191029

‎blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpTransformer.java‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.salesforce.multicloudj.blob.gcp;
22

3+
import com.google.api.gax.retrying.RetrySettings;
34
import com.google.cloud.storage.Blob;
45
import com.google.cloud.storage.BlobId;
56
import com.google.cloud.storage.BlobInfo;
@@ -25,16 +26,17 @@
2526
import com.salesforce.multicloudj.blob.driver.ObjectLockInfo;
2627
import com.salesforce.multicloudj.blob.driver.RetentionMode;
2728
import com.salesforce.multicloudj.common.exceptions.InvalidArgumentException;
29+
import com.salesforce.multicloudj.common.retries.RetryConfig;
2830
import com.salesforce.multicloudj.common.util.HexUtil;
2931
import lombok.Getter;
3032
import org.apache.commons.lang3.tuple.ImmutablePair;
3133
import org.apache.commons.lang3.tuple.Pair;
32-
3334
import java.io.IOException;
3435
import java.io.InputStream;
3536
import java.nio.file.Files;
3637
import java.nio.file.Path;
3738
import java.nio.file.Paths;
39+
import java.time.Duration;
3840
import java.time.Instant;
3941
import java.time.OffsetDateTime;
4042
import java.time.ZoneOffset;
@@ -439,6 +441,73 @@ public List<List<com.salesforce.multicloudj.blob.driver.BlobInfo>> partitionList
439441
return partitionedList;
440442
}
441443

444+
/**
445+
* Converts MultiCloudJ RetryConfig to GCP RetrySettings
446+
*
447+
* @param retryConfig The retry configuration to convert
448+
* @return GCP RetrySettings
449+
* @throws InvalidArgumentException if retryConfig is null or has invalid values
450+
*/
451+
public RetrySettings toGcpRetrySettings(RetryConfig retryConfig) {
452+
if (retryConfig == null) {
453+
throw new InvalidArgumentException("RetryConfig cannot be null");
454+
}
455+
if (retryConfig.getMaxAttempts() != null && retryConfig.getMaxAttempts() <= 0) {
456+
throw new InvalidArgumentException("RetryConfig.maxAttempts must be greater than 0, got: " + retryConfig.getMaxAttempts());
457+
}
458+
459+
RetrySettings.Builder settingsBuilder = RetrySettings.newBuilder();
460+
461+
// Only set maxAttempts if provided, otherwise use GCP SDK default
462+
if (retryConfig.getMaxAttempts() != null) {
463+
settingsBuilder.setMaxAttempts(retryConfig.getMaxAttempts());
464+
}
465+
466+
// If mode is not set, use GCP SDK's default backoff strategy
467+
468+
// Configure backoff strategy based on mode
469+
if (retryConfig.getMode() == RetryConfig.Mode.EXPONENTIAL) {
470+
if (retryConfig.getInitialDelayMillis() <= 0) {
471+
throw new InvalidArgumentException("RetryConfig.initialDelayMillis must be greater than 0 for EXPONENTIAL mode, got: " + retryConfig.getInitialDelayMillis());
472+
}
473+
if (retryConfig.getMaxDelayMillis() <= 0) {
474+
throw new InvalidArgumentException("RetryConfig.maxDelayMillis must be greater than 0 for EXPONENTIAL mode, got: " + retryConfig.getMaxDelayMillis());
475+
}
476+
settingsBuilder.setInitialRetryDelayDuration(Duration.ofMillis(retryConfig.getInitialDelayMillis()))
477+
.setRetryDelayMultiplier(retryConfig.getMultiplier())
478+
.setMaxRetryDelayDuration(Duration.ofMillis(retryConfig.getMaxDelayMillis()));
479+
} else if (retryConfig.getMode() == RetryConfig.Mode.FIXED) {
480+
if (retryConfig.getFixedDelayMillis() <= 0) {
481+
throw new InvalidArgumentException("RetryConfig.fixedDelayMillis must be greater than 0 for FIXED mode, got: " + retryConfig.getFixedDelayMillis());
482+
}
483+
// FIXED mode is simulated by setting multiplier to 1.0 and both delays to the same value
484+
settingsBuilder.setInitialRetryDelayDuration(Duration.ofMillis(retryConfig.getFixedDelayMillis()))
485+
.setRetryDelayMultiplier(1.0)
486+
.setMaxRetryDelayDuration(Duration.ofMillis(retryConfig.getFixedDelayMillis()));
487+
}
488+
489+
// Set total timeout if provided
490+
if (retryConfig.getTotalTimeout() != null) {
491+
if (retryConfig.getTotalTimeout() <= 0) {
492+
throw new InvalidArgumentException("RetryConfig.totalTimeout must be greater than 0, got: " + retryConfig.getTotalTimeout());
493+
}
494+
settingsBuilder.setTotalTimeoutDuration(Duration.ofMillis(retryConfig.getTotalTimeout()));
495+
}
496+
497+
// Set attempt timeout if provided
498+
if (retryConfig.getAttemptTimeout() != null) {
499+
if (retryConfig.getAttemptTimeout() <= 0) {
500+
throw new InvalidArgumentException("RetryConfig.attemptTimeout must be greater than 0, got: " + retryConfig.getAttemptTimeout());
501+
}
502+
Duration attemptTimeout = Duration.ofMillis(retryConfig.getAttemptTimeout());
503+
settingsBuilder.setInitialRpcTimeoutDuration(attemptTimeout)
504+
.setRpcTimeoutMultiplier(1.0)
505+
.setMaxRpcTimeoutDuration(attemptTimeout);
506+
}
507+
508+
return settingsBuilder.build();
509+
}
510+
442511
/**
443512
* Converts a list of BlobInfo objects to BlobIdentifier objects for deletion.
444513
*

‎blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpBlobClientTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.salesforce.multicloudj.sts.model.CredentialsOverrider;
1313
import com.salesforce.multicloudj.sts.model.CredentialsType;
1414
import com.salesforce.multicloudj.sts.model.StsCredentials;
15+
import com.salesforce.multicloudj.common.retries.RetryConfig;
1516
import org.junit.jupiter.api.BeforeEach;
1617
import org.junit.jupiter.api.Test;
1718

@@ -212,4 +213,24 @@ void testBuildStorageWithAllConfigurations() {
212213
assertNotNull(client);
213214
assertEquals("gcp", client.getProviderId());
214215
}
216+
217+
@Test
218+
void testBuildStorageWithRetryConfig() {
219+
// Build with retry configuration
220+
RetryConfig retryConfig = RetryConfig.builder()
221+
.mode(RetryConfig.Mode.EXPONENTIAL)
222+
.maxAttempts(3)
223+
.initialDelayMillis(100L)
224+
.maxDelayMillis(1000L)
225+
.multiplier(2.0)
226+
.build();
227+
228+
GcpBlobClient.Builder builder = new GcpBlobClient.Builder();
229+
builder.withRetryConfig(retryConfig);
230+
GcpBlobClient client = builder.build();
231+
232+
// Verify the client was created successfully
233+
assertNotNull(client);
234+
assertEquals("gcp", client.getProviderId());
235+
}
215236
}

‎blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpTransformerTest.java‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.salesforce.multicloudj.blob.gcp;
22

3+
import com.google.api.gax.retrying.RetrySettings;
34
import com.google.cloud.storage.Blob;
45
import com.google.cloud.storage.BlobId;
56
import com.google.cloud.storage.BlobInfo;
@@ -20,6 +21,7 @@
2021
import com.salesforce.multicloudj.blob.driver.ObjectLockConfiguration;
2122
import com.salesforce.multicloudj.blob.driver.RetentionMode;
2223
import com.salesforce.multicloudj.common.exceptions.InvalidArgumentException;
24+
import com.salesforce.multicloudj.common.retries.RetryConfig;
2325
import org.apache.commons.lang3.tuple.Pair;
2426
import org.junit.jupiter.api.BeforeEach;
2527
import org.junit.jupiter.api.Test;
@@ -1795,6 +1797,132 @@ public void testToBlobMetadata_WithNullUpdateTime() {
17951797
assertNull(blobMetadata.getLastModified());
17961798
}
17971799

1800+
@Test
1801+
public void testToGcpRetrySettings_Exponential() {
1802+
// Given
1803+
RetryConfig retryConfig = RetryConfig.builder()
1804+
.mode(RetryConfig.Mode.EXPONENTIAL)
1805+
.maxAttempts(3)
1806+
.initialDelayMillis(100L)
1807+
.multiplier(2.0)
1808+
.maxDelayMillis(5000L)
1809+
.totalTimeout(30000L)
1810+
.build();
1811+
1812+
// When
1813+
RetrySettings settings = transformer.toGcpRetrySettings(retryConfig);
1814+
1815+
// Then
1816+
assertEquals(3, settings.getMaxAttempts());
1817+
assertEquals(Duration.ofMillis(100), settings.getInitialRetryDelayDuration());
1818+
assertEquals(2.0, settings.getRetryDelayMultiplier());
1819+
assertEquals(Duration.ofMillis(5000), settings.getMaxRetryDelayDuration());
1820+
assertEquals(Duration.ofMillis(30000), settings.getTotalTimeoutDuration());
1821+
}
1822+
1823+
@Test
1824+
public void testToGcpRetrySettings_Fixed() {
1825+
// Given
1826+
RetryConfig retryConfig = RetryConfig.builder()
1827+
.mode(RetryConfig.Mode.FIXED)
1828+
.maxAttempts(5)
1829+
.fixedDelayMillis(1000L)
1830+
.build();
1831+
1832+
// When
1833+
RetrySettings settings = transformer.toGcpRetrySettings(retryConfig);
1834+
1835+
// Then
1836+
assertEquals(5, settings.getMaxAttempts());
1837+
assertEquals(Duration.ofMillis(1000), settings.getInitialRetryDelayDuration());
1838+
assertEquals(1.0, settings.getRetryDelayMultiplier());
1839+
assertEquals(Duration.ofMillis(1000), settings.getMaxRetryDelayDuration());
1840+
}
1841+
1842+
@Test
1843+
public void testToGcpRetrySettings_NullConfig() {
1844+
assertThrows(InvalidArgumentException.class, () -> {
1845+
transformer.toGcpRetrySettings(null);
1846+
});
1847+
}
1848+
1849+
@Test
1850+
public void testToGcpRetrySettings_InvalidMaxAttempts() {
1851+
RetryConfig retryConfig = RetryConfig.builder()
1852+
.maxAttempts(0)
1853+
.build();
1854+
assertThrows(InvalidArgumentException.class, () -> {
1855+
transformer.toGcpRetrySettings(retryConfig);
1856+
});
1857+
}
1858+
1859+
@Test
1860+
public void testToGcpRetrySettings_InvalidExponentialDelays() {
1861+
RetryConfig retryConfig1 = RetryConfig.builder()
1862+
.mode(RetryConfig.Mode.EXPONENTIAL)
1863+
.initialDelayMillis(0)
1864+
.build();
1865+
assertThrows(InvalidArgumentException.class, () -> {
1866+
transformer.toGcpRetrySettings(retryConfig1);
1867+
});
1868+
1869+
RetryConfig retryConfig2 = RetryConfig.builder()
1870+
.mode(RetryConfig.Mode.EXPONENTIAL)
1871+
.initialDelayMillis(100)
1872+
.maxDelayMillis(0)
1873+
.build();
1874+
assertThrows(InvalidArgumentException.class, () -> {
1875+
transformer.toGcpRetrySettings(retryConfig2);
1876+
});
1877+
}
1878+
1879+
@Test
1880+
public void testToGcpRetrySettings_InvalidFixedDelay() {
1881+
RetryConfig retryConfig = RetryConfig.builder()
1882+
.mode(RetryConfig.Mode.FIXED)
1883+
.fixedDelayMillis(0)
1884+
.build();
1885+
assertThrows(InvalidArgumentException.class, () -> {
1886+
transformer.toGcpRetrySettings(retryConfig);
1887+
});
1888+
}
1889+
1890+
@Test
1891+
public void testToGcpRetrySettings_InvalidTotalTimeout() {
1892+
RetryConfig retryConfig = RetryConfig.builder()
1893+
.totalTimeout(0L)
1894+
.build();
1895+
assertThrows(InvalidArgumentException.class, () -> {
1896+
transformer.toGcpRetrySettings(retryConfig);
1897+
});
1898+
}
1899+
1900+
@Test
1901+
public void testToGcpRetrySettings_AttemptTimeout() {
1902+
// Given
1903+
RetryConfig retryConfig = RetryConfig.builder()
1904+
.attemptTimeout(2000L)
1905+
.build();
1906+
1907+
// When
1908+
RetrySettings settings = transformer.toGcpRetrySettings(retryConfig);
1909+
1910+
// Then
1911+
assertEquals(Duration.ofMillis(2000), settings.getInitialRpcTimeoutDuration());
1912+
assertEquals(1.0, settings.getRpcTimeoutMultiplier());
1913+
assertEquals(Duration.ofMillis(2000), settings.getMaxRpcTimeoutDuration());
1914+
}
1915+
1916+
@Test
1917+
public void testToGcpRetrySettings_InvalidAttemptTimeout() {
1918+
RetryConfig retryConfig = RetryConfig.builder()
1919+
.attemptTimeout(0L)
1920+
.build();
1921+
assertThrows(InvalidArgumentException.class, () -> {
1922+
transformer.toGcpRetrySettings(retryConfig);
1923+
});
1924+
}
1925+
17981926
@Test
17991927
public void testToGenerationId_WithInvalidFormat() {
18001928
// Given

‎blob/blob-inmemory/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<parent>
1010
<groupId>com.salesforce.multicloudj</groupId>
1111
<artifactId>blob</artifactId>
12-
<version>0.2.26</version>
12+
<version>0.2.27-SNAPSHOT</version>
1313
<relativePath>../pom.xml</relativePath>
1414
</parent>
1515

0 commit comments

Comments
 (0)