diff --git a/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsBlobStore.java b/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsBlobStore.java index 780753d1d..ea2def677 100644 --- a/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsBlobStore.java +++ b/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsBlobStore.java @@ -235,7 +235,9 @@ protected DownloadResponse doDownload(DownloadRequest downloadRequest, ByteArray @Override protected DownloadResponse doDownload(DownloadRequest downloadRequest, File file) { GetObjectRequest request = transformer.toRequest(downloadRequest); - GetObjectResponse response = s3Client.getObject(request, ResponseTransformer.toFile(file)); + Path destinationPath = createDownloadDestinationPath(downloadRequest, file.toPath()); + GetObjectResponse response = + s3Client.getObject(request, ResponseTransformer.toFile(destinationPath)); return transformer.toDownloadResponse(downloadRequest, response); } @@ -249,7 +251,9 @@ protected DownloadResponse doDownload(DownloadRequest downloadRequest, File file @Override protected DownloadResponse doDownload(DownloadRequest downloadRequest, Path path) { GetObjectRequest request = transformer.toRequest(downloadRequest); - GetObjectResponse response = s3Client.getObject(request, ResponseTransformer.toFile(path)); + Path destinationPath = createDownloadDestinationPath(downloadRequest, path); + GetObjectResponse response = + s3Client.getObject(request, ResponseTransformer.toFile(destinationPath)); return transformer.toDownloadResponse(downloadRequest, response); } diff --git a/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java b/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java index 4319db5ef..e71ca6b12 100644 --- a/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java +++ b/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java @@ -35,6 +35,7 @@ import com.salesforce.multicloudj.common.retries.RetryConfig; import com.salesforce.multicloudj.common.util.HexUtil; import java.io.InputStream; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; @@ -100,6 +101,7 @@ import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryDownload; import software.amazon.awssdk.transfer.s3.model.CompletedDirectoryUpload; import software.amazon.awssdk.transfer.s3.model.DownloadDirectoryRequest; +import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; import software.amazon.awssdk.transfer.s3.model.UploadDirectoryRequest; public class AwsTransformer { @@ -293,6 +295,14 @@ public GetObjectRequest toRequest(DownloadRequest request) { return builder.build(); } + /** Builds a {@link DownloadFileRequest} for use with {@code S3TransferManager.downloadFile}. */ + public DownloadFileRequest toRequest(DownloadRequest request, Path destinationPath) { + return DownloadFileRequest.builder() + .getObjectRequest(toRequest(request)) + .destination(destinationPath) + .build(); + } + /** * Reading the first 500 bytes - createRangeString(0, 500) - "bytes=0-500" Reading a middle 500 * bytes - createRangeString(123, 623) - "bytes=123-623" Reading the last 500 bytes - diff --git a/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStore.java b/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStore.java index 845ea5922..794c0b9bd 100644 --- a/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStore.java +++ b/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStore.java @@ -171,9 +171,7 @@ protected CompletableFuture doDownload( @Override protected CompletableFuture doDownload(DownloadRequest request, File file) { - return client - .getObject(transformer.toRequest(request), AsyncResponseTransformer.toFile(file)) - .thenApply(response -> transformer.toDownloadResponse(request, response)); + return doDownload(request, file.toPath()); } /** @@ -185,8 +183,16 @@ protected CompletableFuture doDownload(DownloadRequest request */ @Override protected CompletableFuture doDownload(DownloadRequest request, Path path) { + Path destinationPath = createDownloadDestinationPath(request, path); + if (request.isParallelDownload()) { + return transferManager + .downloadFile(transformer.toRequest(request, destinationPath)) + .completionFuture() + .thenApply( + completed -> transformer.toDownloadResponse(request, completed.response())); + } return client - .getObject(transformer.toRequest(request), path) + .getObject(transformer.toRequest(request), destinationPath) .thenApply(response -> transformer.toDownloadResponse(request, response)); } diff --git a/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsBlobStoreTest.java b/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsBlobStoreTest.java index fb6474e45..6bf04d977 100644 --- a/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsBlobStoreTest.java +++ b/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsBlobStoreTest.java @@ -183,27 +183,25 @@ void setup() { .withSessionCredentials(sessionCreds) .build(); - aws = - new AwsBlobStore.Builder() - .withTransformerSupplier(transformerSupplier) - .withCredentialsOverrider(credsOverrider) - .withBucket("bucket-1") - .withRegion("us-east-2") - .withEndpoint(URI.create("https://blob.endpoint.com")) - .withProxyEndpoint(URI.create("https://proxy.endpoint.com:443")) - .withSocketTimeout(Duration.ofMinutes(1)) - .withIdleConnectionTimeout(Duration.ofMinutes(5)) - .withMaxConnections(100) - .build(); + AwsBlobStore.Builder builder1 = new AwsBlobStore.Builder(); + builder1.withTransformerSupplier(transformerSupplier); + builder1.withCredentialsOverrider(credsOverrider); + builder1.withBucket("bucket-1"); + builder1.withRegion("us-east-2"); + builder1.withEndpoint(URI.create("https://blob.endpoint.com")); + builder1.withProxyEndpoint(URI.create("https://proxy.endpoint.com:443")); + builder1.withSocketTimeout(Duration.ofMinutes(1)); + builder1.withIdleConnectionTimeout(Duration.ofMinutes(5)); + builder1.withMaxConnections(100); + aws = builder1.build(); credsOverrider = new CredentialsOverrider.Builder(CredentialsType.ASSUME_ROLE).withRole("some-role").build(); - aws = - new AwsBlobStore.Builder() - .withTransformerSupplier(transformerSupplier) - .withCredentialsOverrider(credsOverrider) - .withBucket("bucket-1") - .withRegion("us-east-2") - .build(); + AwsBlobStore.Builder builder2 = new AwsBlobStore.Builder(); + builder2.withTransformerSupplier(transformerSupplier); + builder2.withCredentialsOverrider(credsOverrider); + builder2.withBucket("bucket-1"); + builder2.withRegion("us-east-2"); + aws = builder2.build(); } @AfterEach @@ -557,6 +555,33 @@ void testDoDownloadPath() { } } + @Test + void testDoDownloadPath_WithCreateParentPath() throws IOException { + Instant now = Instant.now(); + setupMockGetObjectResponse(now, false); + + Path rootPath = Path.of("tempCreateParentRoot"); + try { + Files.createDirectories(rootPath); + DownloadRequest request = + DownloadRequest.builder() + .withKey("prefix-a/prefix-b/object-1") + .withVersionId("version-1") + .withRange(10L, 110L) + .withCreateParentPath(true) + .build(); + DownloadResponse response = aws.doDownload(request, rootPath); + assertEquals("prefix-a/prefix-b/object-1", response.getKey()); + // Verify the intermediate parent directories were created. + assertTrue(Files.exists(rootPath.resolve("prefix-a/prefix-b"))); + } finally { + Files.deleteIfExists(rootPath.resolve("prefix-a/prefix-b/object-1")); + Files.deleteIfExists(rootPath.resolve("prefix-a/prefix-b")); + Files.deleteIfExists(rootPath.resolve("prefix-a")); + Files.deleteIfExists(rootPath); + } + } + @Test void testDoDownloadInputStream() { Instant now = Instant.now(); diff --git a/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java b/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java index 35401c1e6..e0338d5f8 100644 --- a/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java +++ b/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java @@ -637,8 +637,7 @@ void testDoDownloadFile() throws ExecutionException, InterruptedException, IOExc aws.doDownload(generateTestDownloadRequest(), path.toFile()).get(); ArgumentCaptor getObjectRequestCaptor = ArgumentCaptor.forClass(GetObjectRequest.class); - verify(mockS3Client, times(1)) - .getObject(getObjectRequestCaptor.capture(), any(AsyncResponseTransformer.class)); + verify(mockS3Client, times(1)).getObject(getObjectRequestCaptor.capture(), any(Path.class)); verifyDownloadTestResults(response, getObjectRequestCaptor, now); } finally { try { diff --git a/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-delete-0.json b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-delete-0.json new file mode 100644 index 000000000..bd94bd4e7 --- /dev/null +++ b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-delete-0.json @@ -0,0 +1,22 @@ +{ + "id" : "4746989d-5b8c-4c51-886c-fbbe79d07646", + "name" : "AwsBlobStoreIT_testDownload_createParentPath-DELETE-0", + "request" : { + "url" : "/chameleon-jcloud/conformance-tests/download_create_parent/nested/object_unversioned", + "method" : "DELETE" + }, + "response" : { + "status" : 204, + "headers" : { + "Server" : "AmazonS3", + "x-amz-request-id" : "VXEX05CQD0KHCT36", + "x-amz-id-2" : "Mlz6UpTuS4EgJLGjOZ8OsyTVCNY1PSNKLoxzFm2Y0O9Tc48yPGdpJexIpNvjm9srWklzqMM8TpWiJRnqpTivEK7Ib/1D/8jh", + "Date" : "Wed, 08 Apr 2026 18:25:31 GMT" + } + }, + "uuid" : "4746989d-5b8c-4c51-886c-fbbe79d07646", + "persistent" : true, + "scenarioName" : "scenario-1-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "scenario-1-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 611 +} \ No newline at end of file diff --git a/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-delete-3.json b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-delete-3.json new file mode 100644 index 000000000..ef415698d --- /dev/null +++ b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-delete-3.json @@ -0,0 +1,23 @@ +{ + "id" : "3e9969dd-f7e1-4d98-ba06-c5445b2dc705", + "name" : "AwsBlobStoreIT_testDownload_createParentPath-DELETE-3", + "request" : { + "url" : "/chameleon-jcloud/conformance-tests/download_create_parent/nested/object_unversioned", + "method" : "DELETE" + }, + "response" : { + "status" : 204, + "headers" : { + "Server" : "AmazonS3", + "x-amz-request-id" : "HCB3TTEM4VF0J5XR", + "x-amz-id-2" : "9HPuhg2cLJkvpT/Uo0HBOsNKySysE0E+7vBeJ27+axf6KHuJ5ndWDLENRcZhU+xddVmn413e3h5ziU08clcwQRcRvaQbWCgA", + "Date" : "Wed, 08 Apr 2026 18:25:30 GMT" + } + }, + "uuid" : "3e9969dd-f7e1-4d98-ba06-c5445b2dc705", + "persistent" : true, + "scenarioName" : "scenario-1-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-1-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 614 +} \ No newline at end of file diff --git a/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-get-1.json b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-get-1.json new file mode 100644 index 000000000..3ce8b1487 --- /dev/null +++ b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-get-1.json @@ -0,0 +1,29 @@ +{ + "id" : "5f1f134b-b3ee-4151-bd9d-d12f182b65c5", + "name" : "AwsBlobStoreIT_testDownload_createParentPath-GET-1", + "request" : { + "url" : "/chameleon-jcloud/conformance-tests/download_create_parent/nested/object_unversioned", + "method" : "GET" + }, + "response" : { + "status" : 200, + "base64Body" : "VGhpcyBpcyB0ZXN0IGRhdGFwG71L3h7gTjsFlvY5lOuu", + "headers" : { + "Accept-Ranges" : "bytes", + "Server" : "AmazonS3", + "ETag" : "\"701bbd4bde1ee04e3b0596f63994ebae\"", + "Last-Modified" : "Wed, 08 Apr 2026 18:25:30 GMT", + "x-amz-request-id" : "VXESB9G89Y9KD5MC", + "x-amz-server-side-encryption" : "AES256", + "x-amz-id-2" : "gTtbjpHXcgEIaRPKuv6iFtF5og9wMy0hK1xWi1MDaJ7c7DJ5gt7qzNtyeovpT0cL9n3jA6XKCsDlXf7zvajREmsWo+9Qdms0", + "x-amz-transfer-encoding" : "append-md5", + "Date" : "Wed, 08 Apr 2026 18:25:31 GMT", + "Content-Type" : "application/octet-stream" + } + }, + "uuid" : "5f1f134b-b3ee-4151-bd9d-d12f182b65c5", + "persistent" : true, + "scenarioName" : "scenario-2-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "scenario-2-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 612 +} \ No newline at end of file diff --git a/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-get-4.json b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-get-4.json new file mode 100644 index 000000000..eb07fb615 --- /dev/null +++ b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-get-4.json @@ -0,0 +1,30 @@ +{ + "id" : "dddd72ba-994f-42ce-a419-8b41493695e4", + "name" : "AwsBlobStoreIT_testDownload_createParentPath-GET-4", + "request" : { + "url" : "/chameleon-jcloud/conformance-tests/download_create_parent/nested/object_unversioned", + "method" : "GET" + }, + "response" : { + "status" : 200, + "base64Body" : "VGhpcyBpcyB0ZXN0IGRhdGFwG71L3h7gTjsFlvY5lOuu", + "headers" : { + "Accept-Ranges" : "bytes", + "Server" : "AmazonS3", + "ETag" : "\"701bbd4bde1ee04e3b0596f63994ebae\"", + "Last-Modified" : "Wed, 08 Apr 2026 18:25:29 GMT", + "x-amz-request-id" : "MESE4HAQ08SHPET3", + "x-amz-server-side-encryption" : "AES256", + "x-amz-id-2" : "wxbH9+r/MD+ScA1mJn0JJrymn8k0DJkG7E7aTVAH8AcHBaipm7TBtq4PDO82SsToSTglTZX6FWHTr/Rdgd0CMiOQzIsN0TwG", + "x-amz-transfer-encoding" : "append-md5", + "Date" : "Wed, 08 Apr 2026 18:25:29 GMT", + "Content-Type" : "application/octet-stream" + } + }, + "uuid" : "dddd72ba-994f-42ce-a419-8b41493695e4", + "persistent" : true, + "scenarioName" : "scenario-2-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-2-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 615 +} \ No newline at end of file diff --git a/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-put-2.json b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-put-2.json new file mode 100644 index 000000000..eba20bcfe --- /dev/null +++ b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-put-2.json @@ -0,0 +1,29 @@ +{ + "id" : "1fd4aac4-a94e-481d-92fd-4803ecf9b8e9", + "name" : "AwsBlobStoreIT_testDownload_createParentPath-PUT-2", + "request" : { + "url" : "/chameleon-jcloud/conformance-tests/download_create_parent/nested/object_unversioned", + "method" : "PUT", + "bodyPatterns" : [ { + "binaryEqualTo" : "VGhpcyBpcyB0ZXN0IGRhdGE=" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "Server" : "AmazonS3", + "ETag" : "\"701bbd4bde1ee04e3b0596f63994ebae\"", + "x-amz-checksum-crc64nvme" : "4G16TxxAsWw=", + "x-amz-checksum-type" : "FULL_OBJECT", + "x-amz-request-id" : "HCBB47AWN97E3P3X", + "x-amz-server-side-encryption" : "AES256", + "x-amz-id-2" : "8WD8Kbc/RSYeUT/99bgNloFHSY1UwOV9DwWJl4ALpHoMUlv7qVqqUNkzLuViEvo0Jq+7vl+Qk7n8FK/eW0mPcMSJjt6MpJAg", + "Date" : "Wed, 08 Apr 2026 18:25:30 GMT" + } + }, + "uuid" : "1fd4aac4-a94e-481d-92fd-4803ecf9b8e9", + "persistent" : true, + "scenarioName" : "scenario-3-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "scenario-3-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 613 +} \ No newline at end of file diff --git a/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-put-5.json b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-put-5.json new file mode 100644 index 000000000..95290d8c0 --- /dev/null +++ b/blob/blob-aws/src/test/resources/mappings/awsblobstoreit_testdownload_createparentpath-put-5.json @@ -0,0 +1,30 @@ +{ + "id" : "e303f571-78e5-4d9d-8e99-ba7ad93cfa67", + "name" : "AwsBlobStoreIT_testDownload_createParentPath-PUT-5", + "request" : { + "url" : "/chameleon-jcloud/conformance-tests/download_create_parent/nested/object_unversioned", + "method" : "PUT", + "bodyPatterns" : [ { + "binaryEqualTo" : "VGhpcyBpcyB0ZXN0IGRhdGE=" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "Server" : "AmazonS3", + "ETag" : "\"701bbd4bde1ee04e3b0596f63994ebae\"", + "x-amz-checksum-crc64nvme" : "4G16TxxAsWw=", + "x-amz-checksum-type" : "FULL_OBJECT", + "x-amz-request-id" : "MES8QY0BV3RMMP8C", + "x-amz-server-side-encryption" : "AES256", + "x-amz-id-2" : "s2JXrMu6YbChtQZ1wQnJAgwo2n8lGjVgRUzDcgUEVnMZ+sLJdlhoO93bh2/jK1gPxvrMVokyX5zSmnbnGOMgP9rnAFw7eq6A", + "Date" : "Wed, 08 Apr 2026 18:25:29 GMT" + } + }, + "uuid" : "e303f571-78e5-4d9d-8e99-ba7ad93cfa67", + "persistent" : true, + "scenarioName" : "scenario-3-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-3-chameleon-jcloud-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 616 +} \ No newline at end of file diff --git a/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/async/driver/AbstractAsyncBlobStore.java b/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/async/driver/AbstractAsyncBlobStore.java index 20d260763..9fd8a5349 100644 --- a/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/async/driver/AbstractAsyncBlobStore.java +++ b/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/async/driver/AbstractAsyncBlobStore.java @@ -24,11 +24,14 @@ import com.salesforce.multicloudj.blob.driver.UploadPartResponse; import com.salesforce.multicloudj.blob.driver.UploadRequest; import com.salesforce.multicloudj.blob.driver.UploadResponse; +import com.salesforce.multicloudj.common.exceptions.SubstrateSdkException; import com.salesforce.multicloudj.sts.model.CredentialsOverrider; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.List; @@ -328,4 +331,25 @@ protected abstract CompletableFuture doUploadDirectory( DirectoryUploadRequest directoryUploadRequest); protected abstract CompletableFuture doDeleteDirectory(String prefix); + + /** + * Resolves the local download destination; when {@link DownloadRequest#isCreateParentPath()} is + * true, appends the object key and creates any missing parent directories. Subclasses may + * override to change the exception type thrown on directory-creation failure. + */ + protected Path createDownloadDestinationPath(DownloadRequest request, Path destination) { + if (!request.isCreateParentPath()) { + return destination; + } + Path resolved = destination.resolve(request.getKey()).normalize(); + Path parent = resolved.getParent(); + if (parent != null) { + try { + Files.createDirectories(parent); + } catch (IOException e) { + throw new SubstrateSdkException("Failed to create destination directories", e); + } + } + return resolved; + } } diff --git a/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/AbstractBlobStore.java b/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/AbstractBlobStore.java index 8260ba112..42f93f0b0 100644 --- a/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/AbstractBlobStore.java +++ b/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/AbstractBlobStore.java @@ -1,11 +1,14 @@ package com.salesforce.multicloudj.blob.driver; +import com.salesforce.multicloudj.common.exceptions.SubstrateSdkException; import com.salesforce.multicloudj.common.provider.Provider; import com.salesforce.multicloudj.sts.model.CredentialsOverrider; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; import java.util.Iterator; @@ -321,6 +324,27 @@ protected void doDeleteDirectory(String prefix) { "Directory delete is not supported by this substrate implementation"); } + /** + * Resolves the local download destination; when {@link DownloadRequest#isCreateParentPath()} is + * true, appends the object key and creates any missing parent directories. Subclasses may + * override to change the exception type thrown on directory-creation failure. + */ + protected Path createDownloadDestinationPath(DownloadRequest request, Path destination) { + if (!request.isCreateParentPath()) { + return destination; + } + Path resolved = destination.resolve(request.getKey()).normalize(); + Path parent = resolved.getParent(); + if (parent != null) { + try { + Files.createDirectories(parent); + } catch (IOException e) { + throw new SubstrateSdkException("Failed to create destination directories", e); + } + } + return resolved; + } + public abstract static class Builder> extends BlobStoreBuilder implements Provider.Builder { diff --git a/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DownloadRequest.java b/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DownloadRequest.java index a1eedac9f..a7d6d7bad 100644 --- a/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DownloadRequest.java +++ b/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DownloadRequest.java @@ -11,6 +11,8 @@ public class DownloadRequest { private final Long start; private final Long end; private final String kmsKeyId; + private final boolean parallelDownload; + private final boolean createParentPath; private DownloadRequest(Builder builder) { this.key = builder.key; @@ -18,6 +20,8 @@ private DownloadRequest(Builder builder) { this.start = builder.start; this.end = builder.end; this.kmsKeyId = builder.kmsKeyId; + this.parallelDownload = builder.parallelDownload; + this.createParentPath = builder.createParentPath; } public static Builder builder() { @@ -30,6 +34,8 @@ public static class Builder { private Long start; private Long end; private String kmsKeyId; + private boolean parallelDownload; + private boolean createParentPath; /** Specifies the key of the Blob to download. */ public Builder withKey(String key) { @@ -90,6 +96,28 @@ public Builder withKmsKeyId(String kmsKeyId) { return this; } + /** + * (Optional) Enables provider-specific parallel download optimization when supported for + * file-based destinations ({@code Path} / {@code File}). Ignored for {@code OutputStream} and + * related streaming-style downloads so content is not fully materialized to disk first. + * Defaults to false. + */ + public Builder withParallelDownload(boolean parallelDownload) { + this.parallelDownload = parallelDownload; + return this; + } + + /** + * (Optional) If true, the destination is treated as a root directory: the object key's parent + * path structure is preserved beneath it, and any missing parent directories are created on + * the local filesystem before the download is written. Only applies to file-based destinations + * ({@code File} / {@code Path}). Defaults to false. + */ + public Builder withCreateParentPath(boolean createParentPath) { + this.createParentPath = createParentPath; + return this; + } + public DownloadRequest build() { return new DownloadRequest(this); } diff --git a/blob/blob-client/src/test/java/com/salesforce/multicloudj/blob/client/AbstractBlobStoreIT.java b/blob/blob-client/src/test/java/com/salesforce/multicloudj/blob/client/AbstractBlobStoreIT.java index 6b993adaa..d185fa4f4 100644 --- a/blob/blob-client/src/test/java/com/salesforce/multicloudj/blob/client/AbstractBlobStoreIT.java +++ b/blob/blob-client/src/test/java/com/salesforce/multicloudj/blob/client/AbstractBlobStoreIT.java @@ -50,6 +50,7 @@ import java.util.Arrays; import java.util.Base64; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -546,6 +547,37 @@ public void testDownload_happy() throws IOException { false); } + /** + * {@link DownloadRequest.Builder#withCreateParentPath(boolean)}: object key contains slashes and + * content is written under a destination directory preserving that layout. + */ + @Test + public void testDownload_createParentPath() throws IOException { + String key = "conformance-tests/download_create_parent/nested/object_unversioned"; + runDownloadTest( + "create parent path file download", + key, + key, + false, + DownloadType.File, + true, + true, + false, + false, + true); + runDownloadTest( + "create parent path path download", + key, + key, + false, + DownloadType.Path, + true, + true, + false, + false, + true); + } + @Test public void testVersionedDownload_happy() throws IOException { runVersionedDownloadTests( @@ -678,6 +710,55 @@ private void runDownloadTest( boolean useCorrectVersionId, boolean wantError) throws IOException { + runDownloadTest( + testName, + uploadKey, + downloadKey, + useVersionedBucket, + downloadType, + downloadUsingVersionId, + useCorrectVersionId, + wantError, + false, + false); + } + + private void runDownloadTest( + String testName, + String uploadKey, + String downloadKey, + boolean useVersionedBucket, + DownloadType downloadType, + boolean downloadUsingVersionId, + boolean useCorrectVersionId, + boolean wantError, + boolean parallelDownload) + throws IOException { + runDownloadTest( + testName, + uploadKey, + downloadKey, + useVersionedBucket, + downloadType, + downloadUsingVersionId, + useCorrectVersionId, + wantError, + parallelDownload, + false); + } + + private void runDownloadTest( + String testName, + String uploadKey, + String downloadKey, + boolean useVersionedBucket, + DownloadType downloadType, + boolean downloadUsingVersionId, + boolean useCorrectVersionId, + boolean wantError, + boolean parallelDownload, + boolean createParentPath) + throws IOException { // Test data String blobData = "This is test data"; byte[] blobBytes = blobData.getBytes(StandardCharsets.UTF_8); @@ -712,11 +793,14 @@ private void runDownloadTest( requestBuilder.withVersionId( useCorrectVersionId ? uploadResponse.getVersionId() : "fakeVersionId"); } + requestBuilder.withParallelDownload(parallelDownload); + requestBuilder.withCreateParentPath(createParentPath); DownloadRequest request = requestBuilder.build(); DownloadResponse response; byte[] content; try { - Pair result = readContent(bucketClient, request, downloadType); + Pair result = + readContent(bucketClient, request, downloadType, createParentPath); response = result.getLeft(); content = result.getRight(); Assertions.assertEquals( @@ -855,6 +939,15 @@ private void runRangedReadDownloadTest( private Pair readContent( BucketClient bucketClient, DownloadRequest request, DownloadType downloadType) throws IOException { + return readContent(bucketClient, request, downloadType, false); + } + + private Pair readContent( + BucketClient bucketClient, + DownloadRequest request, + DownloadType downloadType, + boolean createParentPath) + throws IOException { byte[] content = null; DownloadResponse response = null; switch (downloadType) { @@ -878,17 +971,39 @@ private Pair readContent( content = byteArray.getBytes(); break; case File: - Path path = Files.createTempFile("tempFile", ".txt"); - File file = path.toFile(); - file.delete(); - response = bucketClient.download(request, file); - content = Files.readAllBytes(path); + if (createParentPath) { + Path rootDir = Files.createTempDirectory("mcbj-create-parent-"); + try { + response = bucketClient.download(request, rootDir.toFile()); + Path dataPath = rootDir.resolve(request.getKey()).normalize(); + content = Files.readAllBytes(dataPath); + } finally { + deleteRecursivelyQuietly(rootDir); + } + } else { + Path path = Files.createTempFile("tempFile", ".txt"); + File file = path.toFile(); + file.delete(); + response = bucketClient.download(request, file); + content = Files.readAllBytes(path); + } break; case Path: - Path path2 = Files.createTempFile("tempPath", ".txt"); - path2.toFile().delete(); - response = bucketClient.download(request, path2); - content = Files.readAllBytes(path2); + if (createParentPath) { + Path rootDir = Files.createTempDirectory("mcbj-create-parent-"); + try { + response = bucketClient.download(request, rootDir); + Path dataPath = rootDir.resolve(request.getKey()).normalize(); + content = Files.readAllBytes(dataPath); + } finally { + deleteRecursivelyQuietly(rootDir); + } + } else { + Path path2 = Files.createTempFile("tempPath", ".txt"); + path2.toFile().delete(); + response = bucketClient.download(request, path2); + content = Files.readAllBytes(path2); + } break; default: throw new IllegalArgumentException("Unsupported download type: " + downloadType); @@ -896,6 +1011,26 @@ private Pair readContent( return new ImmutablePair<>(response, content); } + private static void deleteRecursivelyQuietly(Path root) { + if (root == null || !Files.exists(root)) { + return; + } + try (var paths = Files.walk(root)) { + paths.sorted(Comparator.reverseOrder()).forEach(AbstractBlobStoreIT::deletePathQuietly); + } catch (IOException e) { + LoggerFactory.getLogger(AbstractBlobStoreIT.class) + .debug("Failed to walk temp directory for cleanup: {}", root, e); + } + } + + private static void deletePathQuietly(Path p) { + try { + Files.deleteIfExists(p); + } catch (IOException e) { + LoggerFactory.getLogger(AbstractBlobStoreIT.class).debug("Failed to delete path: {}", p, e); + } + } + // Note: This tests delete for non-versioned buckets @Test public void testDelete() throws IOException { diff --git a/blob/blob-client/src/test/java/com/salesforce/multicloudj/blob/driver/DownloadRequestTest.java b/blob/blob-client/src/test/java/com/salesforce/multicloudj/blob/driver/DownloadRequestTest.java index 9470b72ee..ec8f8b45e 100644 --- a/blob/blob-client/src/test/java/com/salesforce/multicloudj/blob/driver/DownloadRequestTest.java +++ b/blob/blob-client/src/test/java/com/salesforce/multicloudj/blob/driver/DownloadRequestTest.java @@ -16,6 +16,8 @@ void testBuilder_WithAllFields() { Long start = 0L; Long end = 100L; String kmsKeyId = "arn:aws:kms:us-east-1:123456789012:key/test-key-id"; + boolean parallelDownload = true; + boolean createParentPath = true; // When DownloadRequest request = @@ -24,6 +26,8 @@ void testBuilder_WithAllFields() { .withVersionId(versionId) .withRange(start, end) .withKmsKeyId(kmsKeyId) + .withParallelDownload(parallelDownload) + .withCreateParentPath(createParentPath) .build(); // Then @@ -32,6 +36,8 @@ void testBuilder_WithAllFields() { assertEquals(start, request.getStart()); assertEquals(end, request.getEnd()); assertEquals(kmsKeyId, request.getKmsKeyId()); + assertEquals(parallelDownload, request.isParallelDownload()); + assertEquals(createParentPath, request.isCreateParentPath()); } @Test @@ -72,6 +78,8 @@ void testBuilder_MinimalFields() { // Then assertNotNull(request); assertEquals(key, request.getKey()); + assertEquals(false, request.isParallelDownload()); + assertEquals(false, request.isCreateParentPath()); } @Test diff --git a/blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStore.java b/blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStore.java index c1398b767..054993a22 100644 --- a/blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStore.java +++ b/blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpBlobStore.java @@ -11,6 +11,7 @@ import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.BlobInfo.Retention; import com.google.cloud.storage.Bucket; import com.google.cloud.storage.HttpMethod; @@ -32,6 +33,12 @@ import com.google.cloud.storage.multipartupload.model.ListPartsResponse; import com.google.cloud.storage.multipartupload.model.UploadPartRequest; import com.google.cloud.storage.multipartupload.model.UploadPartResponse; +import com.google.cloud.storage.transfermanager.DownloadJob; +import com.google.cloud.storage.transfermanager.DownloadResult; +import com.google.cloud.storage.transfermanager.ParallelDownloadConfig; +import com.google.cloud.storage.transfermanager.TransferManager; +import com.google.cloud.storage.transfermanager.TransferManagerConfig; +import com.google.cloud.storage.transfermanager.TransferStatus; import com.google.common.collect.Iterators; import com.google.common.io.ByteStreams; import com.salesforce.multicloudj.blob.driver.AbstractBlobStore; @@ -87,6 +94,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -108,9 +116,14 @@ @AutoService(AbstractBlobStore.class) public class GcpBlobStore extends AbstractBlobStore { + private static final int PARALLEL_LARGE_FILE_DOWNLOAD_MAX_WORKERS = 8; + + private static final String OBJECT_KEY_DIRECTORY_PREFIX_REGEX = "^.*/"; + private final Storage storage; private final MultipartUploadClient multipartUploadClient; private final GcpTransformer transformer; + private final TransferManager transferManager; private static final String TAG_PREFIX = "gcp-tag-"; private static final String RESPONSE_CONTENT_DISPOSITION = "response-content-disposition"; @@ -122,6 +135,7 @@ public GcpBlobStore(Builder builder, Storage storage, MultipartUploadClient mpuC super(builder); this.storage = storage; this.multipartUploadClient = mpuClient; + this.transferManager = builder.getTransferManager(); this.transformer = builder.transformerSupplier.get(bucket); } @@ -189,6 +203,8 @@ protected UploadResponse doUpload(UploadRequest uploadRequest, Path path) { protected DownloadResponse doDownload( DownloadRequest downloadRequest, OutputStream outputStream) { BlobId blobId = transformer.toBlobId(downloadRequest); + // Parallel download uses Transfer Manager / file paths only; OutputStream downloads always use + // ReadChannel streaming (parallelDownload is ignored for this overload). try (ReadChannel reader = storage.reader(blobId); var channel = Channels.newInputStream(reader)) { @@ -223,13 +239,8 @@ protected DownloadResponse doDownload(DownloadRequest downloadRequest, File file return doDownload(downloadRequest, file.toPath()); } - /** - * Performs Blob download and returns an InputStream - * - * @param downloadRequest Wrapper object containing download data - * @return Returns a DownloadResponse object that contains metadata about the blob and an - * InputStream for reading the content - */ + // parallelDownload not supported: TransferManager writes to disk, + // cannot produce an InputStream directly. @Override protected DownloadResponse doDownload(DownloadRequest downloadRequest) { BlobId blobId = transformer.toBlobId(downloadRequest); @@ -261,13 +272,170 @@ protected DownloadResponse doDownload(DownloadRequest downloadRequest) { */ @Override protected DownloadResponse doDownload(DownloadRequest downloadRequest, Path path) { - try (OutputStream outputStream = Files.newOutputStream(path)) { + Path destinationPath = createDownloadDestinationPath(downloadRequest, path); + // GCP TransferManager only supports full-file downloads; + // fall back to ReadChannel for range requests. + if (downloadRequest.isParallelDownload() + && downloadRequest.getStart() == null + && downloadRequest.getEnd() == null) { + return doParallelDownload(downloadRequest, destinationPath); + } + try (OutputStream outputStream = Files.newOutputStream(destinationPath)) { return doDownload(downloadRequest, outputStream); } catch (IOException e) { throw new SubstrateSdkException("Request failed while saving content to path", e); } } + /** + * Parallel download using the GCS transfer manager when available (divide-and-conquer / sliced + * Range GETs for large objects); otherwise {@link Blob#downloadTo(Path)}. Matches the shape of + * {@code AwsBlobStore#doParallelDownload(GetObjectRequest, Path)}: request + resolved file path + * only. + */ + private DownloadResponse doParallelDownload(DownloadRequest downloadRequest, Path destination) { + BlobId blobId = transformer.toBlobId(downloadRequest); + Blob blob = getRequiredBlob(blobId); + ParallelTmPaths tmPaths = computeParallelTmPaths(downloadRequest, destination); + if (transferManager == null || tmPaths == null) { + return downloadBlobToPath(blob, destination); + } + executeTransferManagerDownload(blobId, downloadRequest.getKey(), tmPaths); + return transformer.toDownloadResponse(blob); + } + + private void executeTransferManagerDownload( + BlobId blobId, String objectKey, ParallelTmPaths tmPaths) { + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build(); + ParallelDownloadConfig parallelDownloadConfig = + ParallelDownloadConfig.newBuilder() + .setBucketName(getBucket()) + .setDownloadDirectory(tmPaths.downloadDirectory) + .setStripPrefix(tmPaths.stripPrefix) + .build(); + DownloadJob job = + transferManager.downloadBlobs( + Collections.singletonList(blobInfo), parallelDownloadConfig); + DownloadResult result = job.getDownloadResults().get(0); + if (result.getStatus() != TransferStatus.SUCCESS) { + Exception failure = result.getException(); + throw new SubstrateSdkException( + "Parallel download failed", + failure != null ? failure : new IllegalStateException(result.getStatus().name())); + } + Path expected = tmPaths.expectedOutputPath(objectKey); + Path actual = result.getOutputDestination().normalize(); + if (!actual.equals(expected.normalize())) { + throw new SubstrateSdkException( + "Parallel download wrote unexpected path (expected " + + expected + + ", got " + + actual + + ")"); + } + } + + private DownloadResponse downloadBlobToPath(Blob blob, Path destinationPath) { + blob.downloadTo(destinationPath); + return transformer.toDownloadResponse(blob); + } + + /** + * Derives {@link ParallelDownloadConfig} directory and strip-prefix so the transfer manager + * destination matches {@code destinationPath}, when that layout is expressible (otherwise {@code + * null} and we fall back to {@link Blob#downloadTo(Path)}). + * + *

Unlike AWS S3's transfer manager, which takes an explicit per-download file {@link Path} + * ({@code DownloadFileRequest#destination}), GCS {@link TransferManager} only places files + * under a {@linkplain ParallelDownloadConfig#getDownloadDirectory() download directory} using + * the object key (optionally {@linkplain ParallelDownloadConfig#getStripPrefix() + * strip-prefixed}). We need this mapping so the same resolved {@code destinationPath} as + * {@link #createDownloadDestinationPath} is honored when parallel download is enabled. + */ + private static ParallelTmPaths computeParallelTmPaths( + DownloadRequest request, Path destinationPath) { + String key = request.getKey(); + Path normalizedDest = destinationPath.normalize(); + ParallelTmPaths paths; + if (request.isCreateParentPath()) { + Path downloadRoot = inferDownloadRootFromResolvedKeyPath(destinationPath, key); + if (downloadRoot == null) { + return null; + } + paths = new ParallelTmPaths(downloadRoot, ""); + } else { + Path parent = destinationPath.getParent(); + Path downloadDir = parent != null ? parent.normalize() : Paths.get("."); + Path name = destinationPath.getFileName(); + if (name == null) { + return null; + } + String destFileName = name.toString(); + if (key.indexOf('/') < 0) { + if (!key.equals(destFileName)) { + return null; + } + paths = new ParallelTmPaths(downloadDir, ""); + } else { + String suffix = key.replaceFirst(OBJECT_KEY_DIRECTORY_PREFIX_REGEX, ""); + if (!suffix.equals(destFileName)) { + return null; + } + paths = new ParallelTmPaths(downloadDir, OBJECT_KEY_DIRECTORY_PREFIX_REGEX); + } + } + if (!paths.expectedOutputPath(key).equals(normalizedDest)) { + return null; + } + return paths; + } + + /** + * For {@code createParentPath}, {@code destinationPath} is {@code root.resolve(key)}; recover + * {@code root} by walking parents and matching object key segments (same layout as {@link + * #createDownloadDestinationPath}). + */ + private static Path inferDownloadRootFromResolvedKeyPath( + Path destinationPath, String objectKey) { + List segments = + Arrays.stream(objectKey.split("/")) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + if (segments.isEmpty()) { + return null; + } + Path current = destinationPath.normalize(); + for (int i = segments.size() - 1; i >= 0; i--) { + Path fileName = current.getFileName(); + if (fileName == null || !fileName.toString().equals(segments.get(i))) { + return null; + } + current = current.getParent(); + if (current == null && i > 0) { + return null; + } + } + return current; + } + + private static final class ParallelTmPaths { + private final Path downloadDirectory; + private final String stripPrefix; + + private ParallelTmPaths(Path downloadDirectory, String stripPrefix) { + this.downloadDirectory = downloadDirectory; + this.stripPrefix = stripPrefix; + } + + private Path expectedOutputPath(String objectKey) { + String relative = + stripPrefix.isEmpty() + ? objectKey + : objectKey.replaceFirst(stripPrefix, ""); + return downloadDirectory.resolve(relative).normalize(); + } + } + @Override protected void doDelete(String key, String versionId) { validateBucketExists(); @@ -941,6 +1109,9 @@ public Class getException(Throwable t) { @Override public void close() { try { + if (transferManager != null) { + transferManager.close(); + } if (storage != null) { storage.close(); } @@ -954,6 +1125,7 @@ public static class Builder extends AbstractBlobStore.Builder mockedStatic = Mockito.mockStatic(ByteStreams.class)) { + DownloadRequest downloadRequest = + DownloadRequest.builder().withKey(TEST_KEY).withParallelDownload(true).build(); + DownloadResponse expectedResponse = DownloadResponse.builder().key(TEST_KEY).build(); + + when(mockTransformer.toBlobId(downloadRequest)).thenReturn(mockBlobId); + when(mockStorage.reader(mockBlobId)).thenReturn(mockReadChannel); + when(mockStorage.get(mockBlobId)).thenReturn(mockBlob); + when(mockTransformer.toDownloadResponse(mockBlob)).thenReturn(expectedResponse); + doReturn(new ImmutablePair<>(null, null)) + .when(mockTransformer) + .computeRange(any(), any(), anyLong()); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + DownloadResponse response = gcpBlobStore.doDownload(downloadRequest, outputStream); + + assertEquals(expectedResponse, response); + verify(mockStorage).reader(mockBlobId); + verify(mockBlob, never()).downloadTo(any(Path.class)); + } + } + @Test void testDoDownload_BlobNotFound() { try (MockedStatic mockedStatic = Mockito.mockStatic(ByteStreams.class)) { @@ -577,6 +601,89 @@ void testDoDownload_WithPath() { } } + @Test + void testDoDownload_WithPathAndCreateParentPath() { + try (MockedStatic ignored = Mockito.mockStatic(ByteStreams.class)) { + // Given + Path destinationRoot = tempDir.resolve("download-root"); + DownloadRequest downloadRequest = + DownloadRequest.builder() + .withKey("prefix-a/prefix-b/test.txt") + .withCreateParentPath(true) + .build(); + + DownloadResponse expectedResponse = + DownloadResponse.builder().key("prefix-a/prefix-b/test.txt").build(); + + when(mockTransformer.toBlobId(downloadRequest)).thenReturn(mockBlobId); + when(mockStorage.reader(mockBlobId)).thenReturn(mockReadChannel); + when(mockStorage.get(mockBlobId)).thenReturn(mockBlob); + when(mockTransformer.toDownloadResponse(mockBlob)).thenReturn(expectedResponse); + doReturn(new ImmutablePair<>(null, null)) + .when(mockTransformer) + .computeRange(any(), any(), anyLong()); + + // When + DownloadResponse response = gcpBlobStore.doDownload(downloadRequest, destinationRoot); + + // Then + assertEquals(expectedResponse, response); + assertTrue(Files.exists(destinationRoot.resolve("prefix-a/prefix-b"))); + } + } + + @Test + void testDoDownload_WithPathAndParallelDownload() { + // Given + Path testFile = tempDir.resolve("download.txt"); + DownloadRequest downloadRequest = + DownloadRequest.builder().withKey(TEST_KEY).withParallelDownload(true).build(); + DownloadResponse expectedResponse = DownloadResponse.builder().key(TEST_KEY).build(); + + when(mockTransformer.toBlobId(downloadRequest)).thenReturn(mockBlobId); + when(mockStorage.get(mockBlobId)).thenReturn(mockBlob); + when(mockTransformer.toDownloadResponse(mockBlob)).thenReturn(expectedResponse); + + // When + DownloadResponse response = gcpBlobStore.doDownload(downloadRequest, testFile); + + // Then + assertEquals(expectedResponse, response); + verify(mockBlob).downloadTo(testFile); + verify(mockStorage, never()).reader(mockBlobId); + } + + @Test + void testDoDownload_WithPathAndParallelDownloadWithRangeFallsBackToStream() { + try (MockedStatic ignored = Mockito.mockStatic(ByteStreams.class)) { + // Given + Path testFile = tempDir.resolve("download.txt"); + DownloadRequest downloadRequest = + DownloadRequest.builder() + .withKey(TEST_KEY) + .withParallelDownload(true) + .withRange(10L, 20L) + .build(); + DownloadResponse expectedResponse = DownloadResponse.builder().key(TEST_KEY).build(); + + when(mockTransformer.toBlobId(downloadRequest)).thenReturn(mockBlobId); + when(mockStorage.reader(mockBlobId)).thenReturn(mockReadChannel); + when(mockStorage.get(mockBlobId)).thenReturn(mockBlob); + when(mockTransformer.toDownloadResponse(mockBlob)).thenReturn(expectedResponse); + doReturn(new ImmutablePair<>(10L, 20L)) + .when(mockTransformer) + .computeRange(any(), any(), anyLong()); + + // When + DownloadResponse response = gcpBlobStore.doDownload(downloadRequest, testFile); + + // Then + assertEquals(expectedResponse, response); + verify(mockStorage).reader(mockBlobId); + verify(mockBlob, never()).downloadTo(any(Path.class)); + } + } + @Test void testDoDownload_WithPath_ThrowsException() { try (MockedStatic mockedStatic = Mockito.mockStatic(ByteStreams.class)) { diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-delete-0.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-delete-0.json new file mode 100644 index 000000000..5f31ae049 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-delete-0.json @@ -0,0 +1,27 @@ +{ + "id" : "ef574685-0d98-49b0-93bf-de8d62d1bde0", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-DELETE-0", + "request" : { + "url" : "/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned", + "method" : "DELETE" + }, + "response" : { + "status" : 204, + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "no-cache, no-store, max-age=0, must-revalidate", + "X-GUploader-UploadID" : "AMNfjG1iOU3EL_1jEGg61GP28ldUgXdt-_up267ywVVFwLo9J3Rf1VZDn-iFPBFM9TeDmCRcJAXmy9E", + "Vary" : [ "Origin", "X-Origin" ], + "Pragma" : "no-cache", + "Expires" : "Mon, 01 Jan 1990 00:00:00 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:51 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "ef574685-0d98-49b0-93bf-de8d62d1bde0", + "persistent" : true, + "scenarioName" : "scenario-1-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "scenario-1-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 980 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-delete-7.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-delete-7.json new file mode 100644 index 000000000..3ad329539 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-delete-7.json @@ -0,0 +1,28 @@ +{ + "id" : "26f04f3c-de07-4210-af19-159746423b9a", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-DELETE-7", + "request" : { + "url" : "/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned", + "method" : "DELETE" + }, + "response" : { + "status" : 204, + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "no-cache, no-store, max-age=0, must-revalidate", + "X-GUploader-UploadID" : "AMNfjG1ONhJJv-IoBytlep65Zz5vzn502bZfH15GTIUYXfJJfdJzVrkFx4IfVg20t8u1AJ5PYahAUjQ", + "Vary" : [ "Origin", "X-Origin" ], + "Pragma" : "no-cache", + "Expires" : "Mon, 01 Jan 1990 00:00:00 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:49 GMT", + "Content-Type" : "application/json" + } + }, + "uuid" : "26f04f3c-de07-4210-af19-159746423b9a", + "persistent" : true, + "scenarioName" : "scenario-1-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-1-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 987 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-1.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-1.json new file mode 100644 index 000000000..db8c185be --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-1.json @@ -0,0 +1,27 @@ +{ + "id" : "cf10fdad-29f5-4ae1-ab0e-063355a6486a", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-GET-1", + "request" : { + "url" : "/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o?maxResults=1&projection=full", + "method" : "GET" + }, + "response" : { + "status" : 200, + "body" : "{\n \"kind\": \"storage#objects\",\n \"nextPageToken\": \"CiRjb25mb3JtYW5jZS10ZXN0cy9ibG9iLWZvci1tZXRhZGF0YTM=\",\n \"items\": [\n {\n \"kind\": \"storage#object\",\n \"id\": \"substrate-sdk-gcp-poc1-test-bucket/conformance-tests/blob-for-metadata3/1775494921891632\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fblob-for-metadata3\",\n \"mediaLink\": \"https://storage.googleapis.com/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fblob-for-metadata3?generation=1775494921891632&alt=media\",\n \"name\": \"conformance-tests/blob-for-metadata3\",\n \"bucket\": \"substrate-sdk-gcp-poc1-test-bucket\",\n \"generation\": \"1775494921891632\",\n \"metageneration\": \"1\",\n \"contentType\": \"application/octet-stream\",\n \"storageClass\": \"STANDARD\",\n \"size\": \"43\",\n \"md5Hash\": \"cJNvqBbhpGaIyrO3jHSozA==\",\n \"crc32c\": \"7+lREg==\",\n \"etag\": \"CLDe+oDa2ZMDEAE=\",\n \"timeCreated\": \"2026-04-06T17:02:01.992Z\",\n \"updated\": \"2026-04-06T17:02:01.992Z\",\n \"timeStorageClassUpdated\": \"2026-04-06T17:02:01.992Z\",\n \"timeFinalized\": \"2026-04-06T17:02:01.992Z\",\n \"metadata\": {\n \"def\": \"bar\",\n \"abc\": \"foo\"\n }\n }\n ]\n}\n", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "private, max-age=0, must-revalidate, no-transform", + "X-GUploader-UploadID" : "AMNfjG1qoLxbbnDp692ADJDTdKKn6jfrsgZ0RDNzWKmQS-jz2Llb7qYp36O2P7VMNDLDwkGz5cMELCk", + "Vary" : [ "Origin", "X-Origin" ], + "Expires" : "Wed, 08 Apr 2026 18:24:50 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:50 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "cf10fdad-29f5-4ae1-ab0e-063355a6486a", + "persistent" : true, + "scenarioName" : "scenario-2-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o", + "requiredScenarioState" : "scenario-2-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-2", + "insertionIndex" : 981 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-10.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-10.json new file mode 100644 index 000000000..a1b4a0c23 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-10.json @@ -0,0 +1,26 @@ +{ + "id" : "1fd893eb-b740-4d35-bb6e-ad806d5bea1a", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-GET-10", + "request" : { + "url" : "/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?generation=1775672688513518&projection=full", + "method" : "GET" + }, + "response" : { + "status" : 200, + "body" : "{\n \"kind\": \"storage#object\",\n \"id\": \"substrate-sdk-gcp-poc1-test-bucket/conformance-tests/download_create_parent/nested/object_unversioned/1775672688513518\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned\",\n \"mediaLink\": \"https://storage.googleapis.com/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?generation=1775672688513518&alt=media\",\n \"name\": \"conformance-tests/download_create_parent/nested/object_unversioned\",\n \"bucket\": \"substrate-sdk-gcp-poc1-test-bucket\",\n \"generation\": \"1775672688513518\",\n \"metageneration\": \"1\",\n \"contentType\": \"application/octet-stream\",\n \"storageClass\": \"STANDARD\",\n \"size\": \"17\",\n \"md5Hash\": \"cBu9S94e4E47BZb2OZTrrg==\",\n \"crc32c\": \"E30mnQ==\",\n \"etag\": \"CO6j2J7w3pMDEAE=\",\n \"timeCreated\": \"2026-04-08T18:24:48.585Z\",\n \"updated\": \"2026-04-08T18:24:48.585Z\",\n \"timeStorageClassUpdated\": \"2026-04-08T18:24:48.585Z\",\n \"timeFinalized\": \"2026-04-08T18:24:48.585Z\"\n}\n", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "private, max-age=0, must-revalidate, no-transform", + "ETag" : "CO6j2J7w3pMDEAE=", + "X-GUploader-UploadID" : "AMNfjG1KvOoggxjm_pBjLS6n0Qlj5HtV3DXXAjForE9kTa1khsLQNVGTZldkWDyMgyMDXpeZEWqdbjg", + "Vary" : [ "Origin", "X-Origin" ], + "Expires" : "Wed, 08 Apr 2026 18:24:48 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:48 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "1fd893eb-b740-4d35-bb6e-ad806d5bea1a", + "persistent" : true, + "insertionIndex" : 990 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-11.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-11.json new file mode 100644 index 000000000..81dc05be5 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-11.json @@ -0,0 +1,29 @@ +{ + "id" : "0b920178-f519-45f7-8d13-5b011dbbadee", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-GET-11", + "request" : { + "url" : "/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?projection=full", + "method" : "GET" + }, + "response" : { + "status" : 200, + "body" : "{\n \"kind\": \"storage#object\",\n \"id\": \"substrate-sdk-gcp-poc1-test-bucket/conformance-tests/download_create_parent/nested/object_unversioned/1775672688513518\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned\",\n \"mediaLink\": \"https://storage.googleapis.com/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?generation=1775672688513518&alt=media\",\n \"name\": \"conformance-tests/download_create_parent/nested/object_unversioned\",\n \"bucket\": \"substrate-sdk-gcp-poc1-test-bucket\",\n \"generation\": \"1775672688513518\",\n \"metageneration\": \"1\",\n \"contentType\": \"application/octet-stream\",\n \"storageClass\": \"STANDARD\",\n \"size\": \"17\",\n \"md5Hash\": \"cBu9S94e4E47BZb2OZTrrg==\",\n \"crc32c\": \"E30mnQ==\",\n \"etag\": \"CO6j2J7w3pMDEAE=\",\n \"timeCreated\": \"2026-04-08T18:24:48.585Z\",\n \"updated\": \"2026-04-08T18:24:48.585Z\",\n \"timeStorageClassUpdated\": \"2026-04-08T18:24:48.585Z\",\n \"timeFinalized\": \"2026-04-08T18:24:48.585Z\"\n}\n", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "private, max-age=0, must-revalidate, no-transform", + "ETag" : "CO6j2J7w3pMDEAE=", + "X-GUploader-UploadID" : "AMNfjG2AhRzdqtlcISrAjxwK12XLFnuXi0Jaa89bjS4wkcqQOG_lw5Yko7cam3YRYDkepog8HxRpNyc", + "Vary" : [ "Origin", "X-Origin" ], + "Expires" : "Wed, 08 Apr 2026 18:24:48 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:48 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "0b920178-f519-45f7-8d13-5b011dbbadee", + "persistent" : true, + "scenarioName" : "scenario-3-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-3-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 991 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-2.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-2.json new file mode 100644 index 000000000..e9a6af11c --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-2.json @@ -0,0 +1,35 @@ +{ + "id" : "90320411-b2b4-43ba-b186-c535478898b1", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-GET-2", + "request" : { + "url" : "/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?alt=media&generation=1775672690246375", + "method" : "GET" + }, + "response" : { + "status" : 200, + "base64Body" : "VGhpcyBpcyB0ZXN0IGRhdGE=", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "X-Goog-Generation" : "1775672690246375", + "Pragma" : "no-cache", + "Last-Modified" : "Wed, 08 Apr 2026 18:24:50 GMT", + "X-Goog-Metageneration" : "1", + "Date" : "Wed, 08 Apr 2026 18:24:50 GMT", + "X-Goog-Stored-Content-Encoding" : "identity", + "X-Goog-Hash" : "crc32c=E30mnQ==,md5=cBu9S94e4E47BZb2OZTrrg==", + "Cache-Control" : "no-cache, no-store, max-age=0, must-revalidate", + "ETag" : "COeFwp/w3pMDEAE=", + "Content-Disposition" : "attachment", + "X-Goog-Storage-Class" : "STANDARD", + "X-GUploader-UploadID" : "AMNfjG3yM_eEU1ORtLE8YJyZMMNPUr1x2TG_3o3YB7c1Jjw3yjN_Mewzl60c4hlNHf9IXK-gUcgu9bY", + "Vary" : [ "Origin", "X-Origin" ], + "Expires" : "Mon, 01 Jan 1990 00:00:00 GMT", + "X-Goog-Stored-Content-Length" : "17", + "Content-Type" : "application/octet-stream" + } + }, + "uuid" : "90320411-b2b4-43ba-b186-c535478898b1", + "persistent" : true, + "insertionIndex" : 982 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-3.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-3.json new file mode 100644 index 000000000..1f779e239 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-3.json @@ -0,0 +1,26 @@ +{ + "id" : "1a1f61f9-086e-4ce2-8ee4-a2df5ddc529c", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-GET-3", + "request" : { + "url" : "/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?generation=1775672690246375&projection=full", + "method" : "GET" + }, + "response" : { + "status" : 200, + "body" : "{\n \"kind\": \"storage#object\",\n \"id\": \"substrate-sdk-gcp-poc1-test-bucket/conformance-tests/download_create_parent/nested/object_unversioned/1775672690246375\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned\",\n \"mediaLink\": \"https://storage.googleapis.com/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?generation=1775672690246375&alt=media\",\n \"name\": \"conformance-tests/download_create_parent/nested/object_unversioned\",\n \"bucket\": \"substrate-sdk-gcp-poc1-test-bucket\",\n \"generation\": \"1775672690246375\",\n \"metageneration\": \"1\",\n \"contentType\": \"application/octet-stream\",\n \"storageClass\": \"STANDARD\",\n \"size\": \"17\",\n \"md5Hash\": \"cBu9S94e4E47BZb2OZTrrg==\",\n \"crc32c\": \"E30mnQ==\",\n \"etag\": \"COeFwp/w3pMDEAE=\",\n \"timeCreated\": \"2026-04-08T18:24:50.314Z\",\n \"updated\": \"2026-04-08T18:24:50.314Z\",\n \"timeStorageClassUpdated\": \"2026-04-08T18:24:50.314Z\",\n \"timeFinalized\": \"2026-04-08T18:24:50.314Z\"\n}\n", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "private, max-age=0, must-revalidate, no-transform", + "ETag" : "COeFwp/w3pMDEAE=", + "X-GUploader-UploadID" : "AMNfjG0z8g81qI1MDP_kM46b3j6ZnedeYcBCsyVWIDf9Kmv3z4vSQOZHfbX-bAg9nxTgnKU", + "Vary" : [ "Origin", "X-Origin" ], + "Expires" : "Wed, 08 Apr 2026 18:24:50 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:50 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "1a1f61f9-086e-4ce2-8ee4-a2df5ddc529c", + "persistent" : true, + "insertionIndex" : 983 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-4.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-4.json new file mode 100644 index 000000000..5662b9b39 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-4.json @@ -0,0 +1,28 @@ +{ + "id" : "a8e011f4-fff0-4e5a-ad90-185187fe6020", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-GET-4", + "request" : { + "url" : "/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?projection=full", + "method" : "GET" + }, + "response" : { + "status" : 200, + "body" : "{\n \"kind\": \"storage#object\",\n \"id\": \"substrate-sdk-gcp-poc1-test-bucket/conformance-tests/download_create_parent/nested/object_unversioned/1775672690246375\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned\",\n \"mediaLink\": \"https://storage.googleapis.com/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?generation=1775672690246375&alt=media\",\n \"name\": \"conformance-tests/download_create_parent/nested/object_unversioned\",\n \"bucket\": \"substrate-sdk-gcp-poc1-test-bucket\",\n \"generation\": \"1775672690246375\",\n \"metageneration\": \"1\",\n \"contentType\": \"application/octet-stream\",\n \"storageClass\": \"STANDARD\",\n \"size\": \"17\",\n \"md5Hash\": \"cBu9S94e4E47BZb2OZTrrg==\",\n \"crc32c\": \"E30mnQ==\",\n \"etag\": \"COeFwp/w3pMDEAE=\",\n \"timeCreated\": \"2026-04-08T18:24:50.314Z\",\n \"updated\": \"2026-04-08T18:24:50.314Z\",\n \"timeStorageClassUpdated\": \"2026-04-08T18:24:50.314Z\",\n \"timeFinalized\": \"2026-04-08T18:24:50.314Z\"\n}\n", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "private, max-age=0, must-revalidate, no-transform", + "ETag" : "COeFwp/w3pMDEAE=", + "X-GUploader-UploadID" : "AMNfjG1btKVo_kcAP_UfrrBoXgRGJ3xkkxM3y7mhs8SJZJC3i88WNmn2d50Yj25iVI3jc4NnP34s8k8", + "Vary" : [ "Origin", "X-Origin" ], + "Expires" : "Wed, 08 Apr 2026 18:24:50 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:50 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "a8e011f4-fff0-4e5a-ad90-185187fe6020", + "persistent" : true, + "scenarioName" : "scenario-3-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-conformance-tests-download_create_parent-nested-object_unversioned", + "requiredScenarioState" : "scenario-3-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-conformance-tests-download_create_parent-nested-object_unversioned-2", + "insertionIndex" : 984 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-8.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-8.json new file mode 100644 index 000000000..b7b7cee78 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-8.json @@ -0,0 +1,28 @@ +{ + "id" : "ce914a7d-bb3c-4e08-b4ea-62f3027745ac", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-GET-8", + "request" : { + "url" : "/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o?maxResults=1&projection=full", + "method" : "GET" + }, + "response" : { + "status" : 200, + "body" : "{\n \"kind\": \"storage#objects\",\n \"nextPageToken\": \"CiRjb25mb3JtYW5jZS10ZXN0cy9ibG9iLWZvci1tZXRhZGF0YTM=\",\n \"items\": [\n {\n \"kind\": \"storage#object\",\n \"id\": \"substrate-sdk-gcp-poc1-test-bucket/conformance-tests/blob-for-metadata3/1775494921891632\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fblob-for-metadata3\",\n \"mediaLink\": \"https://storage.googleapis.com/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fblob-for-metadata3?generation=1775494921891632&alt=media\",\n \"name\": \"conformance-tests/blob-for-metadata3\",\n \"bucket\": \"substrate-sdk-gcp-poc1-test-bucket\",\n \"generation\": \"1775494921891632\",\n \"metageneration\": \"1\",\n \"contentType\": \"application/octet-stream\",\n \"storageClass\": \"STANDARD\",\n \"size\": \"43\",\n \"md5Hash\": \"cJNvqBbhpGaIyrO3jHSozA==\",\n \"crc32c\": \"7+lREg==\",\n \"etag\": \"CLDe+oDa2ZMDEAE=\",\n \"timeCreated\": \"2026-04-06T17:02:01.992Z\",\n \"updated\": \"2026-04-06T17:02:01.992Z\",\n \"timeStorageClassUpdated\": \"2026-04-06T17:02:01.992Z\",\n \"timeFinalized\": \"2026-04-06T17:02:01.992Z\",\n \"metadata\": {\n \"def\": \"bar\",\n \"abc\": \"foo\"\n }\n }\n ]\n}\n", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "private, max-age=0, must-revalidate, no-transform", + "X-GUploader-UploadID" : "AMNfjG1prIDxxUtK5Yh2H677hvEtZz6UrTvp1VFL570sl8_XYs4NMb612jqTmrN8coValq4XRoI1gqY", + "Vary" : [ "Origin", "X-Origin" ], + "Expires" : "Wed, 08 Apr 2026 18:24:49 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:49 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "ce914a7d-bb3c-4e08-b4ea-62f3027745ac", + "persistent" : true, + "scenarioName" : "scenario-2-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-2-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-2", + "insertionIndex" : 988 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-9.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-9.json new file mode 100644 index 000000000..a47f81297 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-get-9.json @@ -0,0 +1,35 @@ +{ + "id" : "ae153843-178c-426b-9a7b-1bf0de1c367d", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-GET-9", + "request" : { + "url" : "/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?alt=media&generation=1775672688513518", + "method" : "GET" + }, + "response" : { + "status" : 200, + "base64Body" : "VGhpcyBpcyB0ZXN0IGRhdGE=", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "X-Goog-Generation" : "1775672688513518", + "Pragma" : "no-cache", + "Last-Modified" : "Wed, 08 Apr 2026 18:24:48 GMT", + "X-Goog-Metageneration" : "1", + "Date" : "Wed, 08 Apr 2026 18:24:49 GMT", + "X-Goog-Stored-Content-Encoding" : "identity", + "X-Goog-Hash" : "crc32c=E30mnQ==,md5=cBu9S94e4E47BZb2OZTrrg==", + "Cache-Control" : "no-cache, no-store, max-age=0, must-revalidate", + "ETag" : "CO6j2J7w3pMDEAE=", + "Content-Disposition" : "attachment", + "X-Goog-Storage-Class" : "STANDARD", + "X-GUploader-UploadID" : "AMNfjG1ba9gnSeCB6ge1w28xlPu8ipHdAlFYYCLHnu1uYGmVS5ghoXKFpmpJX2jX94aeMA1WXzc4RXA", + "Vary" : [ "Origin", "X-Origin" ], + "Expires" : "Mon, 01 Jan 1990 00:00:00 GMT", + "X-Goog-Stored-Content-Length" : "17", + "Content-Type" : "application/octet-stream" + } + }, + "uuid" : "ae153843-178c-426b-9a7b-1bf0de1c367d", + "persistent" : true, + "insertionIndex" : 989 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-post-13.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-post-13.json new file mode 100644 index 000000000..e56af95a0 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-post-13.json @@ -0,0 +1,34 @@ +{ + "id" : "2cc59112-d6d2-4696-a38b-85b0ad5ae30a", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-POST-13", + "request" : { + "url" : "/upload/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o?name=conformance-tests/download_create_parent/nested/object_unversioned&uploadType=resumable", + "method" : "POST", + "bodyPatterns" : [ { + "equalToJson" : "{\"bucket\":\"substrate-sdk-gcp-poc1-test-bucket\",\"metadata\":{},\"name\":\"conformance-tests/download_create_parent/nested/object_unversioned\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "no-cache, no-store, max-age=0, must-revalidate", + "X-GUploader-UploadID" : "AMNfjG1QFACXXZvWg7EPGdA9Jx7dgNQ4PnT3dXu96rfQnztoh6LfNsnwhBimjRp04vl8GQdyv3VaqtptuN7j_csW-KhD0Q-ESTGZ8qlRfF55WMs", + "Vary" : [ "Origin", "X-Origin" ], + "Pragma" : "no-cache", + "Expires" : "Mon, 01 Jan 1990 00:00:00 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:48 GMT", + "Location" : "https://storage.googleapis.com/upload/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o?name=conformance-tests/download_create_parent/nested/object_unversioned&uploadType=resumable&upload_id=AMNfjG1QFACXXZvWg7EPGdA9Jx7dgNQ4PnT3dXu96rfQnztoh6LfNsnwhBimjRp04vl8GQdyv3VaqtptuN7j_csW-KhD0Q-ESTGZ8qlRfF55WMs", + "Content-Type" : "text/plain; charset=utf-8" + } + }, + "uuid" : "2cc59112-d6d2-4696-a38b-85b0ad5ae30a", + "persistent" : true, + "scenarioName" : "scenario-4-upload-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o", + "requiredScenarioState" : "Started", + "newScenarioState" : "scenario-4-upload-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-2", + "insertionIndex" : 993 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-post-6.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-post-6.json new file mode 100644 index 000000000..94c745ac2 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-post-6.json @@ -0,0 +1,33 @@ +{ + "id" : "9e7c2c2a-689e-4d29-92c8-549bfc29f34e", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-POST-6", + "request" : { + "url" : "/upload/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o?name=conformance-tests/download_create_parent/nested/object_unversioned&uploadType=resumable", + "method" : "POST", + "bodyPatterns" : [ { + "equalToJson" : "{\"bucket\":\"substrate-sdk-gcp-poc1-test-bucket\",\"metadata\":{},\"name\":\"conformance-tests/download_create_parent/nested/object_unversioned\"}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "no-cache, no-store, max-age=0, must-revalidate", + "X-GUploader-UploadID" : "AMNfjG3jP4uk5W_pcr5Eb94aB6Zkw9j4Z3LPVJpi8GIL1aG0J0rc2gL-XqPvSCboDgth9zN29p2jRh7sEgxBysbNLvgg507GABKji2EPP1qtsA", + "Vary" : [ "Origin", "X-Origin" ], + "Pragma" : "no-cache", + "Expires" : "Mon, 01 Jan 1990 00:00:00 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:50 GMT", + "Location" : "https://storage.googleapis.com/upload/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o?name=conformance-tests/download_create_parent/nested/object_unversioned&uploadType=resumable&upload_id=AMNfjG3jP4uk5W_pcr5Eb94aB6Zkw9j4Z3LPVJpi8GIL1aG0J0rc2gL-XqPvSCboDgth9zN29p2jRh7sEgxBysbNLvgg507GABKji2EPP1qtsA", + "Content-Type" : "text/plain; charset=utf-8" + } + }, + "uuid" : "9e7c2c2a-689e-4d29-92c8-549bfc29f34e", + "persistent" : true, + "scenarioName" : "scenario-4-upload-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o", + "requiredScenarioState" : "scenario-4-upload-storage-v1-b-substrate-sdk-gcp-poc1-test-bucket-o-2", + "insertionIndex" : 986 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-put-12.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-put-12.json new file mode 100644 index 000000000..3b2cdd2ea --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-put-12.json @@ -0,0 +1,31 @@ +{ + "id" : "e06f4bdd-c00c-4b7c-8b18-bf1dc074e6b4", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-PUT-12", + "request" : { + "url" : "/upload/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o?name=conformance-tests/download_create_parent/nested/object_unversioned&uploadType=resumable&upload_id=AMNfjG1QFACXXZvWg7EPGdA9Jx7dgNQ4PnT3dXu96rfQnztoh6LfNsnwhBimjRp04vl8GQdyv3VaqtptuN7j_csW-KhD0Q-ESTGZ8qlRfF55WMs", + "method" : "PUT", + "bodyPatterns" : [ { + "equalTo" : "This is test data", + "caseInsensitive" : false + } ] + }, + "response" : { + "status" : 200, + "body" : "{\n \"kind\": \"storage#object\",\n \"id\": \"substrate-sdk-gcp-poc1-test-bucket/conformance-tests/download_create_parent/nested/object_unversioned/1775672688513518\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned\",\n \"mediaLink\": \"https://storage.googleapis.com/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?generation=1775672688513518&alt=media\",\n \"name\": \"conformance-tests/download_create_parent/nested/object_unversioned\",\n \"bucket\": \"substrate-sdk-gcp-poc1-test-bucket\",\n \"generation\": \"1775672688513518\",\n \"metageneration\": \"1\",\n \"contentType\": \"application/octet-stream\",\n \"storageClass\": \"STANDARD\",\n \"size\": \"17\",\n \"md5Hash\": \"cBu9S94e4E47BZb2OZTrrg==\",\n \"crc32c\": \"E30mnQ==\",\n \"etag\": \"CO6j2J7w3pMDEAE=\",\n \"timeCreated\": \"2026-04-08T18:24:48.585Z\",\n \"updated\": \"2026-04-08T18:24:48.585Z\",\n \"timeStorageClassUpdated\": \"2026-04-08T18:24:48.585Z\",\n \"timeFinalized\": \"2026-04-08T18:24:48.585Z\"\n}\n", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "no-cache, no-store, max-age=0, must-revalidate", + "ETag" : "CO6j2J7w3pMDEAE=", + "X-GUploader-UploadID" : "AMNfjG1QFACXXZvWg7EPGdA9Jx7dgNQ4PnT3dXu96rfQnztoh6LfNsnwhBimjRp04vl8GQdyv3VaqtptuN7j_csW-KhD0Q-ESTGZ8qlRfF55WMs", + "Vary" : [ "Origin", "X-Origin" ], + "Pragma" : "no-cache", + "Expires" : "Mon, 01 Jan 1990 00:00:00 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:48 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "e06f4bdd-c00c-4b7c-8b18-bf1dc074e6b4", + "persistent" : true, + "insertionIndex" : 992 +} \ No newline at end of file diff --git a/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-put-5.json b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-put-5.json new file mode 100644 index 000000000..c7e5f3288 --- /dev/null +++ b/blob/blob-gcp/src/test/resources/mappings/gcpblobstoreit_testdownload_createparentpath-put-5.json @@ -0,0 +1,31 @@ +{ + "id" : "9c552366-c370-4cd7-bb5f-d85bdda11943", + "name" : "GcpBlobStoreIT_testDownload_createParentPath-PUT-5", + "request" : { + "url" : "/upload/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o?name=conformance-tests/download_create_parent/nested/object_unversioned&uploadType=resumable&upload_id=AMNfjG3jP4uk5W_pcr5Eb94aB6Zkw9j4Z3LPVJpi8GIL1aG0J0rc2gL-XqPvSCboDgth9zN29p2jRh7sEgxBysbNLvgg507GABKji2EPP1qtsA", + "method" : "PUT", + "bodyPatterns" : [ { + "equalTo" : "This is test data", + "caseInsensitive" : false + } ] + }, + "response" : { + "status" : 200, + "body" : "{\n \"kind\": \"storage#object\",\n \"id\": \"substrate-sdk-gcp-poc1-test-bucket/conformance-tests/download_create_parent/nested/object_unversioned/1775672690246375\",\n \"selfLink\": \"https://www.googleapis.com/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned\",\n \"mediaLink\": \"https://storage.googleapis.com/download/storage/v1/b/substrate-sdk-gcp-poc1-test-bucket/o/conformance-tests%2Fdownload_create_parent%2Fnested%2Fobject_unversioned?generation=1775672690246375&alt=media\",\n \"name\": \"conformance-tests/download_create_parent/nested/object_unversioned\",\n \"bucket\": \"substrate-sdk-gcp-poc1-test-bucket\",\n \"generation\": \"1775672690246375\",\n \"metageneration\": \"1\",\n \"contentType\": \"application/octet-stream\",\n \"storageClass\": \"STANDARD\",\n \"size\": \"17\",\n \"md5Hash\": \"cBu9S94e4E47BZb2OZTrrg==\",\n \"crc32c\": \"E30mnQ==\",\n \"etag\": \"COeFwp/w3pMDEAE=\",\n \"timeCreated\": \"2026-04-08T18:24:50.314Z\",\n \"updated\": \"2026-04-08T18:24:50.314Z\",\n \"timeStorageClassUpdated\": \"2026-04-08T18:24:50.314Z\",\n \"timeFinalized\": \"2026-04-08T18:24:50.314Z\"\n}\n", + "headers" : { + "Alt-Svc" : "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "Server" : "UploadServer", + "Cache-Control" : "no-cache, no-store, max-age=0, must-revalidate", + "ETag" : "COeFwp/w3pMDEAE=", + "X-GUploader-UploadID" : "AMNfjG3jP4uk5W_pcr5Eb94aB6Zkw9j4Z3LPVJpi8GIL1aG0J0rc2gL-XqPvSCboDgth9zN29p2jRh7sEgxBysbNLvgg507GABKji2EPP1qtsA", + "Vary" : [ "Origin", "X-Origin" ], + "Pragma" : "no-cache", + "Expires" : "Mon, 01 Jan 1990 00:00:00 GMT", + "Date" : "Wed, 08 Apr 2026 18:24:50 GMT", + "Content-Type" : "application/json; charset=UTF-8" + } + }, + "uuid" : "9c552366-c370-4cd7-bb5f-d85bdda11943", + "persistent" : true, + "insertionIndex" : 985 +} \ No newline at end of file diff --git a/blob/blob-inmemory/src/main/java/com/salesforce/multicloudj/blob/inmemory/InMemoryBlobStore.java b/blob/blob-inmemory/src/main/java/com/salesforce/multicloudj/blob/inmemory/InMemoryBlobStore.java index 6622c4eae..be52f364a 100644 --- a/blob/blob-inmemory/src/main/java/com/salesforce/multicloudj/blob/inmemory/InMemoryBlobStore.java +++ b/blob/blob-inmemory/src/main/java/com/salesforce/multicloudj/blob/inmemory/InMemoryBlobStore.java @@ -33,6 +33,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.MalformedURLException; @@ -274,13 +275,31 @@ protected DownloadResponse doDownload(DownloadRequest downloadRequest, Path path try { byte[] data = extractRange(blob.getData(), downloadRequest.getStart(), downloadRequest.getEnd()); - Files.write(path, data); + Path destinationPath = createDownloadDestinationPath(downloadRequest, path); + Files.write(destinationPath, data); return buildDownloadResponse(downloadRequest.getKey(), blob, data.length); } catch (Exception e) { throw new UnknownException("Failed to download blob to path", e); } } + @Override + protected Path createDownloadDestinationPath(DownloadRequest request, Path destination) { + if (!request.isCreateParentPath()) { + return destination; + } + Path resolved = destination.resolve(request.getKey()).normalize(); + Path parent = resolved.getParent(); + if (parent != null) { + try { + Files.createDirectories(parent); + } catch (IOException e) { + throw new UnknownException("Failed to create destination directories", e); + } + } + return resolved; + } + @Override protected DownloadResponse doDownload(DownloadRequest downloadRequest) { validateBucketExists();