blob: onboard SymbolicLinks in AWS and GCP

blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java

@@ -585,6 +585,7 @@ public UploadDirectoryRequest toUploadDirectoryRequest(DirectoryUploadRequest re
             .bucket(getBucket())
             .source(Paths.get(request.getLocalSourceDirectory()))
             .maxDepth(request.isIncludeSubFolders() ? Integer.MAX_VALUE : 1)
+            .followSymbolicLinks(request.isFollowSymbolicLinks())
             .s3Prefix(request.getPrefix());
 
     // Merge tags into the existing PutObjectRequest per file; putObjectRequest(Consumer) would

blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsTransformerTest.java

@@ -41,6 +41,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.Test;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
@@ -719,6 +720,29 @@ void testToUploadDirectoryRequest() {
     assertTrue(request.maxDepth().isPresent());
   }
 
+  @Test
+  void testToUploadDirectoryRequest_FollowSymbolicLinks() {
+    DirectoryUploadRequest directoryUploadRequest =
+        DirectoryUploadRequest.builder()
+            .localSourceDirectory("/home/documents")
+            .prefix("/files")
+            .includeSubFolders(true)
+            .followSymbolicLinks(true)
+            .build();
+    UploadDirectoryRequest request = transformer.toUploadDirectoryRequest(directoryUploadRequest);
+    assertEquals(Optional.of(true), request.followSymbolicLinks());
+
+    directoryUploadRequest =
+        DirectoryUploadRequest.builder()
+            .localSourceDirectory("/home/documents")
+            .prefix("/files")
+            .includeSubFolders(true)
+            .followSymbolicLinks(false)
+            .build();
+    request = transformer.toUploadDirectoryRequest(directoryUploadRequest);
+    assertEquals(Optional.of(false), request.followSymbolicLinks());
+  }
+
   @Test
   void testToUploadDirectoryRequest_WithTags() {
     // Given

blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java

@@ -1309,128 +1309,107 @@ void doDownloadDirectory() throws ExecutionException, InterruptedException {
   }
 
   @Test
-  void doUploadDirectory() throws ExecutionException, InterruptedException, IOException {
-    // Create a temporary directory with test files
-    Path tempDir = Files.createTempDirectory("test-upload-dir");
-    try {
-      // Create test files
-      Path file1 = tempDir.resolve("file1.txt");
-      Path file2 = tempDir.resolve("subdir").resolve("file2.txt");
-      Files.createDirectories(file2.getParent());
-      Files.write(file1, "content1".getBytes());
-      Files.write(file2, "content2".getBytes());
-
-      // Mock transfer manager directory upload
-      DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class);
-      CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class);
-      doReturn(mockDirectoryUpload)
-          .when(mockS3TransferManager)
-          .uploadDirectory(any(UploadDirectoryRequest.class));
-      doReturn(CompletableFuture.completedFuture(mockCompletedUpload))
-          .when(mockDirectoryUpload)
-          .completionFuture();
-      doReturn(List.of()).when(mockCompletedUpload).failedTransfers();
-
-      DirectoryUploadRequest uploadRequest =
-          DirectoryUploadRequest.builder()
-              .localSourceDirectory(tempDir.toString())
-              .prefix("files/")
-              .includeSubFolders(true)
-              .build();
-
-      // Perform the request
-      DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
-
-      // Verify the results
-      assertNotNull(response);
-      assertTrue(response.getFailedTransfers().isEmpty());
-
-      // Verify transfer manager uploadDirectory was called with correct request
-      ArgumentCaptor<UploadDirectoryRequest> requestCaptor =
-          ArgumentCaptor.forClass(UploadDirectoryRequest.class);
-      verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
-      UploadDirectoryRequest capturedRequest = requestCaptor.getValue();
-      assertEquals(BUCKET, capturedRequest.bucket());
-      assertEquals(tempDir, capturedRequest.source());
-      assertEquals("files/", capturedRequest.s3Prefix().orElse(null));
-    } finally {
-      // Clean up
-      Files.walk(tempDir)
-          .sorted((a, b) -> b.compareTo(a))
-          .forEach(
-              path -> {
-                try {
-                  Files.deleteIfExists(path);
-                } catch (IOException e) {
-                  // Ignore cleanup errors
-                }
-              });
-    }
+  void doUploadDirectory() throws ExecutionException, InterruptedException {
+    DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class);
+    CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class);
+    doReturn(mockDirectoryUpload)
+        .when(mockS3TransferManager)
+        .uploadDirectory(any(UploadDirectoryRequest.class));
+    doReturn(CompletableFuture.completedFuture(mockCompletedUpload))
+        .when(mockDirectoryUpload)
+        .completionFuture();
+    doReturn(List.of()).when(mockCompletedUpload).failedTransfers();
+
+    DirectoryUploadRequest uploadRequest =
+        DirectoryUploadRequest.builder()
+            .localSourceDirectory("/tmp/test-upload-dir")
+            .prefix("files/")
+            .includeSubFolders(true)
+            .build();
+
+    DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
+
+    assertNotNull(response);
+    assertTrue(response.getFailedTransfers().isEmpty());
+
+    ArgumentCaptor<UploadDirectoryRequest> requestCaptor =
+        ArgumentCaptor.forClass(UploadDirectoryRequest.class);
+    verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
+    UploadDirectoryRequest capturedRequest = requestCaptor.getValue();
+    assertEquals(BUCKET, capturedRequest.bucket());
+    assertEquals(Paths.get("/tmp/test-upload-dir"), capturedRequest.source());
+    assertEquals("files/", capturedRequest.s3Prefix().orElse(null));
   }
 
   @Test
-  void doUploadDirectory_WithTags() throws ExecutionException, InterruptedException, IOException {
-    // Create a temporary directory with test files
-    Path tempDir = Files.createTempDirectory("test-upload-dir-tags");
-    try {
-      // Create test files
-      Path file1 = tempDir.resolve("file1.txt");
-      Path file2 = tempDir.resolve("subdir").resolve("file2.txt");
-      Files.createDirectories(file2.getParent());
-      Files.write(file1, "content1".getBytes());
-      Files.write(file2, "content2".getBytes());
-
-      Map<String, String> tags = Map.of("tag1", "value1", "tag2", "value2");
-
-      // Mock transfer manager directory upload
-      DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class);
-      CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class);
-      doReturn(mockDirectoryUpload)
-          .when(mockS3TransferManager)
-          .uploadDirectory(any(UploadDirectoryRequest.class));
-      doReturn(CompletableFuture.completedFuture(mockCompletedUpload))
-          .when(mockDirectoryUpload)
-          .completionFuture();
-      doReturn(List.of()).when(mockCompletedUpload).failedTransfers();
-
-      DirectoryUploadRequest uploadRequest =
-          DirectoryUploadRequest.builder()
-              .localSourceDirectory(tempDir.toString())
-              .prefix("files/")
-              .includeSubFolders(true)
-              .tags(tags)
-              .build();
-
-      // Perform the request
-      DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
-
-      // Verify the results
-      assertNotNull(response);
-      assertTrue(response.getFailedTransfers().isEmpty());
-
-      // Verify transfer manager uploadDirectory was called with correct request (including tags via
-      // transformer)
-      ArgumentCaptor<UploadDirectoryRequest> requestCaptor =
-          ArgumentCaptor.forClass(UploadDirectoryRequest.class);
-      verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
-      UploadDirectoryRequest capturedRequest = requestCaptor.getValue();
-      assertEquals(BUCKET, capturedRequest.bucket());
-      assertEquals(tempDir, capturedRequest.source());
-      assertEquals("files/", capturedRequest.s3Prefix().orElse(null));
-      assertNotNull(capturedRequest.uploadFileRequestTransformer());
-    } finally {
-      // Clean up
-      Files.walk(tempDir)
-          .sorted((a, b) -> b.compareTo(a))
-          .forEach(
-              path -> {
-                try {
-                  Files.deleteIfExists(path);
-                } catch (IOException e) {
-                  // Ignore cleanup errors
-                }
-              });
-    }
+  void doUploadDirectory_WithTags() throws ExecutionException, InterruptedException {
+    Map<String, String> tags = Map.of("tag1", "value1", "tag2", "value2");
+
+    DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class);
+    CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class);
+    doReturn(mockDirectoryUpload)
+        .when(mockS3TransferManager)
+        .uploadDirectory(any(UploadDirectoryRequest.class));
+    doReturn(CompletableFuture.completedFuture(mockCompletedUpload))
+        .when(mockDirectoryUpload)
+        .completionFuture();
+    doReturn(List.of()).when(mockCompletedUpload).failedTransfers();
+
+    DirectoryUploadRequest uploadRequest =
+        DirectoryUploadRequest.builder()
+            .localSourceDirectory("/tmp/test-upload-dir-tags")
+            .prefix("files/")
+            .includeSubFolders(true)
+            .tags(tags)
+            .build();
+
+    DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
+
+    assertNotNull(response);
+    assertTrue(response.getFailedTransfers().isEmpty());
+
+    ArgumentCaptor<UploadDirectoryRequest> requestCaptor =
+        ArgumentCaptor.forClass(UploadDirectoryRequest.class);
+    verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
+    UploadDirectoryRequest capturedRequest = requestCaptor.getValue();
+    assertEquals(BUCKET, capturedRequest.bucket());
+    assertEquals(Paths.get("/tmp/test-upload-dir-tags"), capturedRequest.source());
+    assertEquals("files/", capturedRequest.s3Prefix().orElse(null));
+    assertNotNull(capturedRequest.uploadFileRequestTransformer());
+  }
+
+  @Test
+  void doUploadDirectory_FollowSymbolicLinks()
+      throws ExecutionException, InterruptedException {
+    DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class);
+    CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class);
+    doReturn(mockDirectoryUpload)
+        .when(mockS3TransferManager)
+        .uploadDirectory(any(UploadDirectoryRequest.class));
+    doReturn(CompletableFuture.completedFuture(mockCompletedUpload))
+        .when(mockDirectoryUpload)
+        .completionFuture();
+    doReturn(List.of()).when(mockCompletedUpload).failedTransfers();
+
+    DirectoryUploadRequest uploadRequest =
+        DirectoryUploadRequest.builder()
+            .localSourceDirectory("/tmp/test-upload-dir-symlink")
+            .prefix("files/")
+            .includeSubFolders(true)
+            .followSymbolicLinks(true)
+            .build();
+
+    DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
+
+    assertNotNull(response);
+    assertTrue(response.getFailedTransfers().isEmpty());
+
+    ArgumentCaptor<UploadDirectoryRequest> requestCaptor =
+        ArgumentCaptor.forClass(UploadDirectoryRequest.class);
+    verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
+    UploadDirectoryRequest capturedRequest = requestCaptor.getValue();
+    assertEquals(BUCKET, capturedRequest.bucket());
+    assertTrue(capturedRequest.followSymbolicLinks().orElse(false));
   }
 
   @Test

blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DirectoryUploadRequest.java

@@ -12,6 +12,12 @@ public class DirectoryUploadRequest {
   private final String prefix;
   private final boolean includeSubFolders;
 
+  /**
+   * When true, symbolic links encountered during directory traversal will be followed, uploading
+   * the files they point to. Defaults to false.
+   */
+  private final boolean followSymbolicLinks;
+
   /**
    * (Optional parameter) The map of tagName to tagValue to be associated with all blobs in the
    * directory

blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpTransformer.java

@@ -30,6 +30,7 @@
 import com.salesforce.multicloudj.common.util.HexUtil;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.FileVisitOption;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -389,7 +390,10 @@ public List<Path> toFilePaths(DirectoryUploadRequest request) {
     Path sourceDir = Paths.get(request.getLocalSourceDirectory());
     List<Path> filePaths = new ArrayList<>();
 
-    try (Stream<Path> paths = Files.walk(sourceDir)) {
+    try (Stream<Path> paths =
+        request.isFollowSymbolicLinks()
+            ? Files.walk(sourceDir, Integer.MAX_VALUE, FileVisitOption.FOLLOW_LINKS)
+            : Files.walk(sourceDir)) {
       filePaths =
           paths
               .filter(Files::isRegularFile)