blob: onboard SymbolicLinks in AWS and GCP

blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java

@@ -569,6 +569,7 @@ public UploadDirectoryRequest toUploadDirectoryRequest(DirectoryUploadRequest re
             .bucket(getBucket())
             .source(Paths.get(request.getLocalSourceDirectory()))
             .maxDepth(request.isIncludeSubFolders() ? Integer.MAX_VALUE : 1)
+            .followSymbolicLinks(request.isFollowSymbolicLinks())
             .s3Prefix(request.getPrefix());
 
     // Merge tags into the existing PutObjectRequest per file; putObjectRequest(Consumer) would

blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsTransformerTest.java

@@ -41,6 +41,7 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.Test;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
@@ -682,6 +683,29 @@ void testToUploadDirectoryRequest() {
     assertTrue(request.maxDepth().isPresent());
   }
 
+  @Test
+  void testToUploadDirectoryRequest_FollowSymbolicLinks() {
+    DirectoryUploadRequest directoryUploadRequest =
+        DirectoryUploadRequest.builder()
+            .localSourceDirectory("/home/documents")
+            .prefix("/files")
+            .includeSubFolders(true)
+            .followSymbolicLinks(true)
+            .build();
+    UploadDirectoryRequest request = transformer.toUploadDirectoryRequest(directoryUploadRequest);
+    assertEquals(Optional.of(true), request.followSymbolicLinks());
+
+    directoryUploadRequest =
+        DirectoryUploadRequest.builder()
+            .localSourceDirectory("/home/documents")
+            .prefix("/files")
+            .includeSubFolders(true)
+            .followSymbolicLinks(false)
+            .build();
+    request = transformer.toUploadDirectoryRequest(directoryUploadRequest);
+    assertEquals(Optional.of(false), request.followSymbolicLinks());
+  }
+
   @Test
   void testToUploadDirectoryRequest_WithTags() {
     // Given

blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java

@@ -1433,6 +1433,57 @@ void doUploadDirectory_WithTags() throws ExecutionException, InterruptedExceptio
     }
   }
 
+  @Test
+  void doUploadDirectory_FollowSymbolicLinks()
+      throws ExecutionException, InterruptedException, IOException {
+    Path tempDir = Files.createTempDirectory("test-upload-dir-symlink");
+    try {
+      Path file1 = tempDir.resolve("file1.txt");
+      Files.write(file1, "content1".getBytes());
+
+      DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class);
+      CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class);
+      doReturn(mockDirectoryUpload)
+          .when(mockS3TransferManager)
+          .uploadDirectory(any(UploadDirectoryRequest.class));
+      doReturn(CompletableFuture.completedFuture(mockCompletedUpload))
+          .when(mockDirectoryUpload)
+          .completionFuture();
+      doReturn(List.of()).when(mockCompletedUpload).failedTransfers();
+
+      DirectoryUploadRequest uploadRequest =
+          DirectoryUploadRequest.builder()
+              .localSourceDirectory(tempDir.toString())
+              .prefix("files/")
+              .includeSubFolders(true)
+              .followSymbolicLinks(true)
+              .build();
+
+      DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get();
+
+      assertNotNull(response);
+      assertTrue(response.getFailedTransfers().isEmpty());
+
+      ArgumentCaptor<UploadDirectoryRequest> requestCaptor =
+          ArgumentCaptor.forClass(UploadDirectoryRequest.class);
+      verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture());
+      UploadDirectoryRequest capturedRequest = requestCaptor.getValue();
+      assertEquals(BUCKET, capturedRequest.bucket());
+      assertTrue(capturedRequest.followSymbolicLinks().orElse(false));
+    } finally {
+      Files.walk(tempDir)
+          .sorted((a, b) -> b.compareTo(a))
+          .forEach(
+              path -> {
+                try {
+                  Files.deleteIfExists(path);
+                } catch (IOException e) {
+                  // Ignore cleanup errors
+                }
+              });
+    }
+  }
+
   @Test
   void doDeleteDirectory() throws ExecutionException, InterruptedException {
 

blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DirectoryUploadRequest.java

@@ -12,6 +12,12 @@ public class DirectoryUploadRequest {
   private final String prefix;
   private final boolean includeSubFolders;
 
+  /**
+   * When true, symbolic links encountered during directory traversal will be followed, uploading
+   * the files they point to. Defaults to false.
+   */
+  private final boolean followSymbolicLinks;
+
   /**
    * (Optional parameter) The map of tagName to tagValue to be associated with all blobs in the
    * directory

blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpTransformer.java

@@ -30,6 +30,7 @@
 import com.salesforce.multicloudj.common.util.HexUtil;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.FileVisitOption;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -389,7 +390,10 @@ public List<Path> toFilePaths(DirectoryUploadRequest request) {
     Path sourceDir = Paths.get(request.getLocalSourceDirectory());
     List<Path> filePaths = new ArrayList<>();
 
-    try (Stream<Path> paths = Files.walk(sourceDir)) {
+    try (Stream<Path> paths =
+        request.isFollowSymbolicLinks()
+            ? Files.walk(sourceDir, Integer.MAX_VALUE, FileVisitOption.FOLLOW_LINKS)
+            : Files.walk(sourceDir)) {
       filePaths =
           paths
               .filter(Files::isRegularFile)

blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpTransformerTest.java

@@ -965,6 +965,145 @@ public void testToFilePaths_UploadRequest_NoSubFolders() throws IOException {
     }
   }
 
+  @Test
+  public void testToFilePaths_FollowSymbolicLinks() throws IOException {
+    Path tempDir = Files.createTempDirectory("test-upload-symlink");
+    Path realDir = Files.createTempDirectory("test-upload-target");
+    Path realFile = realDir.resolve("linked-file.txt");
+    Files.write(realFile, "linked-content".getBytes());
+    Path file1 = tempDir.resolve("file1.txt");
+    Files.write(file1, "content1".getBytes());
+    Path symlink = tempDir.resolve("link-dir");
+    Files.createSymbolicLink(symlink, realDir);
+
+    try {
+      DirectoryUploadRequest requestNoFollow =
+          DirectoryUploadRequest.builder()
+              .localSourceDirectory(tempDir.toString())
+              .prefix("uploads/")
+              .includeSubFolders(true)
+              .followSymbolicLinks(false)
+              .build();
+      List<Path> noFollowPaths = transformer.toFilePaths(requestNoFollow);
+      assertEquals(1, noFollowPaths.size());
+      assertTrue(noFollowPaths.contains(file1));
+
+      DirectoryUploadRequest requestFollow =
+          DirectoryUploadRequest.builder()
+              .localSourceDirectory(tempDir.toString())
+              .prefix("uploads/")
+              .includeSubFolders(true)
+              .followSymbolicLinks(true)
+              .build();
+      List<Path> followPaths = transformer.toFilePaths(requestFollow);
+      assertEquals(2, followPaths.size());
+      assertTrue(followPaths.contains(file1));
+    } finally {
+      Files.walk(realDir)
+          .sorted(Comparator.reverseOrder())
+          .map(Path::toFile)
+          .forEach(File::delete);
+      Files.walk(tempDir)
+          .sorted(Comparator.reverseOrder())
+          .map(Path::toFile)
+          .forEach(File::delete);
+    }
+  }
+
+  @Test
+  public void testToFilePaths_CircularSymbolicLinks() throws IOException {
+    Path tempDir = Files.createTempDirectory("test-circular-symlink");
+    try {
+      Files.write(tempDir.resolve("file.txt"), "content".getBytes());
+      Path dirA = Files.createDirectory(tempDir.resolve("dirA"));
+      Path dirB = Files.createDirectory(tempDir.resolve("dirB"));
+      Files.createSymbolicLink(dirA.resolve("link-to-b"), dirB);
+      Files.createSymbolicLink(dirB.resolve("link-to-a"), dirA);
+
+      DirectoryUploadRequest request =
+          DirectoryUploadRequest.builder()
+              .localSourceDirectory(tempDir.toString())
+              .prefix("uploads/")
+              .includeSubFolders(true)
+              .followSymbolicLinks(true)
+              .build();
+
+      assertThrows(RuntimeException.class, () -> transformer.toFilePaths(request));
+    } finally {
+      Files.deleteIfExists(tempDir.resolve("dirA/link-to-b"));
+      Files.deleteIfExists(tempDir.resolve("dirB/link-to-a"));
+      Files.walk(tempDir)
+          .sorted(Comparator.reverseOrder())
+          .map(Path::toFile)
+          .forEach(File::delete);
+    }
+  }
+
+  @Test
+  public void testToFilePaths_BrokenSymbolicLinks() throws IOException {
+    Path tempDir = Files.createTempDirectory("test-broken-symlink");
+    try {
+      Path file1 = tempDir.resolve("file1.txt");
+      Files.write(file1, "content".getBytes());
+      Files.createSymbolicLink(tempDir.resolve("broken-link"), Paths.get("/non/existent/path"));
+
+      DirectoryUploadRequest request =
+          DirectoryUploadRequest.builder()
+              .localSourceDirectory(tempDir.toString())
+              .prefix("uploads/")
+              .includeSubFolders(true)
+              .followSymbolicLinks(true)
+              .build();
+
+      List<Path> paths = transformer.toFilePaths(request);
+      assertEquals(1, paths.size());
+      assertTrue(paths.contains(file1));
+    } finally {
+      Files.walk(tempDir)
+          .sorted(Comparator.reverseOrder())
+          .map(Path::toFile)
+          .forEach(File::delete);
+    }
+  }
+
+  @Test
+  public void testToFilePaths_NestedSymbolicLinks() throws IOException {
+    Path tempDir = Files.createTempDirectory("test-nested-symlink");
+    Path externalDir1 = Files.createTempDirectory("test-nested-target1");
+    Path externalDir2 = Files.createTempDirectory("test-nested-target2");
+    try {
+      Files.write(tempDir.resolve("root.txt"), "root".getBytes());
+      Files.write(externalDir2.resolve("deep.txt"), "deep".getBytes());
+      Files.createSymbolicLink(externalDir1.resolve("link-to-dir2"), externalDir2);
+      Files.createSymbolicLink(tempDir.resolve("link-to-dir1"), externalDir1);
+
+      DirectoryUploadRequest request =
+          DirectoryUploadRequest.builder()
+              .localSourceDirectory(tempDir.toString())
+              .prefix("uploads/")
+              .includeSubFolders(true)
+              .followSymbolicLinks(true)
+              .build();
+
+      List<Path> paths = transformer.toFilePaths(request);
+      assertEquals(2, paths.size());
+      assertTrue(paths.contains(tempDir.resolve("root.txt")));
+    } finally {
+      Files.walk(externalDir2)
+          .sorted(Comparator.reverseOrder())
+          .map(Path::toFile)
+          .forEach(File::delete);
+      Files.walk(externalDir1)
+          .sorted(Comparator.reverseOrder())
+          .map(Path::toFile)
+          .forEach(File::delete);
+      Files.walk(tempDir)
+          .sorted(Comparator.reverseOrder())
+          .map(Path::toFile)
+          .forEach(File::delete);
+    }
+  }
+
   @Test
   public void testToBlobKey() {
     Path sourceDir = Paths.get("/source");

examples/src/main/java/com/salesforce/multicloudj/blob/Main.java

@@ -348,6 +348,7 @@ public static void uploadDirectory() {
             .localSourceDirectory("/tmp/test-directory") // Change this to your test directory
             .prefix("uploads/")
             .includeSubFolders(true)
+            .followSymbolicLinks(false) // Set to true to follow symbolic links during upload
             .build();
 
     System.out.println("DirectoryUploadRequest created: " + request);
@@ -385,6 +386,49 @@ public static void uploadDirectory() {
     }
   }
 
+  /**
+   * Uploads a directory to blob storage with symbolic links enabled. When followSymbolicLinks is
+   * true, any symlinks in the directory will be followed and the target files/directories will be
+   * uploaded.
+   */
+  public static void uploadDirectoryWithSymbolicLinks() {
+    AsyncBucketClient asyncClient = getAsyncBucketClient(getProvider());
+
+    DirectoryUploadRequest request =
+        DirectoryUploadRequest.builder()
+            .localSourceDirectory("/tmp/test-directory")
+            .prefix("uploads-with-symlinks/")
+            .includeSubFolders(true)
+            .followSymbolicLinks(true)
+            .build();
+
+    try {
+      CompletableFuture<DirectoryUploadResponse> future = asyncClient.uploadDirectory(request);
+      DirectoryUploadResponse response = future.get();
+
+      getLogger().info("Directory upload with symbolic links completed");
+      getLogger().info("Failed transfers: {}", response.getFailedTransfers().size());
+
+      if (!response.getFailedTransfers().isEmpty()) {
+        response
+            .getFailedTransfers()
+            .forEach(
+                failure -> {
+                  getLogger()
+                      .error(
+                          "Failed: {} - {}",
+                          failure.getSource(),
+                          failure.getException().getMessage());
+                });
+      } else {
+        getLogger().info("All files (including symlinked) uploaded successfully!");
+      }
+    } catch (Exception e) {
+      getLogger()
+          .error("Directory upload with symbolic links failed: {}", e.getMessage(), e);
+    }
+  }
+
   /** Downloads a directory from blob storage */
   public static void downloadDirectory() {
     // Get the AsyncBucketClient instance