diff --git a/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java b/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java index 91354724a..6817a3bd5 100644 --- a/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java +++ b/blob/blob-aws/src/main/java/com/salesforce/multicloudj/blob/aws/AwsTransformer.java @@ -585,6 +585,7 @@ public UploadDirectoryRequest toUploadDirectoryRequest(DirectoryUploadRequest re .bucket(getBucket()) .source(Paths.get(request.getLocalSourceDirectory())) .maxDepth(request.isIncludeSubFolders() ? Integer.MAX_VALUE : 1) + .followSymbolicLinks(request.isFollowSymbolicLinks()) .s3Prefix(request.getPrefix()); // Merge tags into the existing PutObjectRequest per file; putObjectRequest(Consumer) would diff --git a/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsTransformerTest.java b/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsTransformerTest.java index eaa707990..e05af7499 100644 --- a/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsTransformerTest.java +++ b/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/AwsTransformerTest.java @@ -41,6 +41,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.async.AsyncRequestBody; @@ -719,6 +720,29 @@ void testToUploadDirectoryRequest() { assertTrue(request.maxDepth().isPresent()); } + @Test + void testToUploadDirectoryRequest_FollowSymbolicLinks() { + DirectoryUploadRequest directoryUploadRequest = + DirectoryUploadRequest.builder() + .localSourceDirectory("/home/documents") + .prefix("/files") + .includeSubFolders(true) + .followSymbolicLinks(true) + .build(); + UploadDirectoryRequest request = transformer.toUploadDirectoryRequest(directoryUploadRequest); + assertEquals(Optional.of(true), request.followSymbolicLinks()); + + directoryUploadRequest = + DirectoryUploadRequest.builder() + .localSourceDirectory("/home/documents") + .prefix("/files") + .includeSubFolders(true) + .followSymbolicLinks(false) + .build(); + request = transformer.toUploadDirectoryRequest(directoryUploadRequest); + assertEquals(Optional.of(false), request.followSymbolicLinks()); + } + @Test void testToUploadDirectoryRequest_WithTags() { // Given diff --git a/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java b/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java index 92fd08f58..322d2a005 100644 --- a/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java +++ b/blob/blob-aws/src/test/java/com/salesforce/multicloudj/blob/aws/async/AwsAsyncBlobStoreTest.java @@ -1309,128 +1309,107 @@ void doDownloadDirectory() throws ExecutionException, InterruptedException { } @Test - void doUploadDirectory() throws ExecutionException, InterruptedException, IOException { - // Create a temporary directory with test files - Path tempDir = Files.createTempDirectory("test-upload-dir"); - try { - // Create test files - Path file1 = tempDir.resolve("file1.txt"); - Path file2 = tempDir.resolve("subdir").resolve("file2.txt"); - Files.createDirectories(file2.getParent()); - Files.write(file1, "content1".getBytes()); - Files.write(file2, "content2".getBytes()); - - // Mock transfer manager directory upload - DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class); - CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class); - doReturn(mockDirectoryUpload) - .when(mockS3TransferManager) - .uploadDirectory(any(UploadDirectoryRequest.class)); - doReturn(CompletableFuture.completedFuture(mockCompletedUpload)) - .when(mockDirectoryUpload) - .completionFuture(); - doReturn(List.of()).when(mockCompletedUpload).failedTransfers(); - - DirectoryUploadRequest uploadRequest = - DirectoryUploadRequest.builder() - .localSourceDirectory(tempDir.toString()) - .prefix("files/") - .includeSubFolders(true) - .build(); - - // Perform the request - DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get(); - - // Verify the results - assertNotNull(response); - assertTrue(response.getFailedTransfers().isEmpty()); - - // Verify transfer manager uploadDirectory was called with correct request - ArgumentCaptor requestCaptor = - ArgumentCaptor.forClass(UploadDirectoryRequest.class); - verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture()); - UploadDirectoryRequest capturedRequest = requestCaptor.getValue(); - assertEquals(BUCKET, capturedRequest.bucket()); - assertEquals(tempDir, capturedRequest.source()); - assertEquals("files/", capturedRequest.s3Prefix().orElse(null)); - } finally { - // Clean up - Files.walk(tempDir) - .sorted((a, b) -> b.compareTo(a)) - .forEach( - path -> { - try { - Files.deleteIfExists(path); - } catch (IOException e) { - // Ignore cleanup errors - } - }); - } + void doUploadDirectory() throws ExecutionException, InterruptedException { + DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class); + CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class); + doReturn(mockDirectoryUpload) + .when(mockS3TransferManager) + .uploadDirectory(any(UploadDirectoryRequest.class)); + doReturn(CompletableFuture.completedFuture(mockCompletedUpload)) + .when(mockDirectoryUpload) + .completionFuture(); + doReturn(List.of()).when(mockCompletedUpload).failedTransfers(); + + DirectoryUploadRequest uploadRequest = + DirectoryUploadRequest.builder() + .localSourceDirectory("/tmp/test-upload-dir") + .prefix("files/") + .includeSubFolders(true) + .build(); + + DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get(); + + assertNotNull(response); + assertTrue(response.getFailedTransfers().isEmpty()); + + ArgumentCaptor requestCaptor = + ArgumentCaptor.forClass(UploadDirectoryRequest.class); + verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture()); + UploadDirectoryRequest capturedRequest = requestCaptor.getValue(); + assertEquals(BUCKET, capturedRequest.bucket()); + assertEquals(Paths.get("/tmp/test-upload-dir"), capturedRequest.source()); + assertEquals("files/", capturedRequest.s3Prefix().orElse(null)); } @Test - void doUploadDirectory_WithTags() throws ExecutionException, InterruptedException, IOException { - // Create a temporary directory with test files - Path tempDir = Files.createTempDirectory("test-upload-dir-tags"); - try { - // Create test files - Path file1 = tempDir.resolve("file1.txt"); - Path file2 = tempDir.resolve("subdir").resolve("file2.txt"); - Files.createDirectories(file2.getParent()); - Files.write(file1, "content1".getBytes()); - Files.write(file2, "content2".getBytes()); - - Map tags = Map.of("tag1", "value1", "tag2", "value2"); - - // Mock transfer manager directory upload - DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class); - CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class); - doReturn(mockDirectoryUpload) - .when(mockS3TransferManager) - .uploadDirectory(any(UploadDirectoryRequest.class)); - doReturn(CompletableFuture.completedFuture(mockCompletedUpload)) - .when(mockDirectoryUpload) - .completionFuture(); - doReturn(List.of()).when(mockCompletedUpload).failedTransfers(); - - DirectoryUploadRequest uploadRequest = - DirectoryUploadRequest.builder() - .localSourceDirectory(tempDir.toString()) - .prefix("files/") - .includeSubFolders(true) - .tags(tags) - .build(); - - // Perform the request - DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get(); - - // Verify the results - assertNotNull(response); - assertTrue(response.getFailedTransfers().isEmpty()); - - // Verify transfer manager uploadDirectory was called with correct request (including tags via - // transformer) - ArgumentCaptor requestCaptor = - ArgumentCaptor.forClass(UploadDirectoryRequest.class); - verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture()); - UploadDirectoryRequest capturedRequest = requestCaptor.getValue(); - assertEquals(BUCKET, capturedRequest.bucket()); - assertEquals(tempDir, capturedRequest.source()); - assertEquals("files/", capturedRequest.s3Prefix().orElse(null)); - assertNotNull(capturedRequest.uploadFileRequestTransformer()); - } finally { - // Clean up - Files.walk(tempDir) - .sorted((a, b) -> b.compareTo(a)) - .forEach( - path -> { - try { - Files.deleteIfExists(path); - } catch (IOException e) { - // Ignore cleanup errors - } - }); - } + void doUploadDirectory_WithTags() throws ExecutionException, InterruptedException { + Map tags = Map.of("tag1", "value1", "tag2", "value2"); + + DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class); + CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class); + doReturn(mockDirectoryUpload) + .when(mockS3TransferManager) + .uploadDirectory(any(UploadDirectoryRequest.class)); + doReturn(CompletableFuture.completedFuture(mockCompletedUpload)) + .when(mockDirectoryUpload) + .completionFuture(); + doReturn(List.of()).when(mockCompletedUpload).failedTransfers(); + + DirectoryUploadRequest uploadRequest = + DirectoryUploadRequest.builder() + .localSourceDirectory("/tmp/test-upload-dir-tags") + .prefix("files/") + .includeSubFolders(true) + .tags(tags) + .build(); + + DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get(); + + assertNotNull(response); + assertTrue(response.getFailedTransfers().isEmpty()); + + ArgumentCaptor requestCaptor = + ArgumentCaptor.forClass(UploadDirectoryRequest.class); + verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture()); + UploadDirectoryRequest capturedRequest = requestCaptor.getValue(); + assertEquals(BUCKET, capturedRequest.bucket()); + assertEquals(Paths.get("/tmp/test-upload-dir-tags"), capturedRequest.source()); + assertEquals("files/", capturedRequest.s3Prefix().orElse(null)); + assertNotNull(capturedRequest.uploadFileRequestTransformer()); + } + + @Test + void doUploadDirectory_FollowSymbolicLinks() + throws ExecutionException, InterruptedException { + DirectoryUpload mockDirectoryUpload = mock(DirectoryUpload.class); + CompletedDirectoryUpload mockCompletedUpload = mock(CompletedDirectoryUpload.class); + doReturn(mockDirectoryUpload) + .when(mockS3TransferManager) + .uploadDirectory(any(UploadDirectoryRequest.class)); + doReturn(CompletableFuture.completedFuture(mockCompletedUpload)) + .when(mockDirectoryUpload) + .completionFuture(); + doReturn(List.of()).when(mockCompletedUpload).failedTransfers(); + + DirectoryUploadRequest uploadRequest = + DirectoryUploadRequest.builder() + .localSourceDirectory("/tmp/test-upload-dir-symlink") + .prefix("files/") + .includeSubFolders(true) + .followSymbolicLinks(true) + .build(); + + DirectoryUploadResponse response = aws.doUploadDirectory(uploadRequest).get(); + + assertNotNull(response); + assertTrue(response.getFailedTransfers().isEmpty()); + + ArgumentCaptor requestCaptor = + ArgumentCaptor.forClass(UploadDirectoryRequest.class); + verify(mockS3TransferManager, times(1)).uploadDirectory(requestCaptor.capture()); + UploadDirectoryRequest capturedRequest = requestCaptor.getValue(); + assertEquals(BUCKET, capturedRequest.bucket()); + assertTrue(capturedRequest.followSymbolicLinks().orElse(false)); } @Test diff --git a/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DirectoryUploadRequest.java b/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DirectoryUploadRequest.java index 2e496634f..819341c23 100644 --- a/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DirectoryUploadRequest.java +++ b/blob/blob-client/src/main/java/com/salesforce/multicloudj/blob/driver/DirectoryUploadRequest.java @@ -12,6 +12,12 @@ public class DirectoryUploadRequest { private final String prefix; private final boolean includeSubFolders; + /** + * When true, symbolic links encountered during directory traversal will be followed, uploading + * the files they point to. Defaults to false. + */ + private final boolean followSymbolicLinks; + /** * (Optional parameter) The map of tagName to tagValue to be associated with all blobs in the * directory diff --git a/blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpTransformer.java b/blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpTransformer.java index a01f19b43..995d86439 100644 --- a/blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpTransformer.java +++ b/blob/blob-gcp/src/main/java/com/salesforce/multicloudj/blob/gcp/GcpTransformer.java @@ -30,6 +30,7 @@ import com.salesforce.multicloudj.common.util.HexUtil; import java.io.IOException; import java.io.InputStream; +import java.nio.file.FileVisitOption; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -389,7 +390,10 @@ public List toFilePaths(DirectoryUploadRequest request) { Path sourceDir = Paths.get(request.getLocalSourceDirectory()); List filePaths = new ArrayList<>(); - try (Stream paths = Files.walk(sourceDir)) { + try (Stream paths = + request.isFollowSymbolicLinks() + ? Files.walk(sourceDir, Integer.MAX_VALUE, FileVisitOption.FOLLOW_LINKS) + : Files.walk(sourceDir)) { filePaths = paths .filter(Files::isRegularFile) diff --git a/blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpTransformerTest.java b/blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpTransformerTest.java index c951e171f..3f6d16e14 100644 --- a/blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpTransformerTest.java +++ b/blob/blob-gcp/src/test/java/com/salesforce/multicloudj/blob/gcp/GcpTransformerTest.java @@ -965,6 +965,145 @@ public void testToFilePaths_UploadRequest_NoSubFolders() throws IOException { } } + @Test + public void testToFilePaths_FollowSymbolicLinks() throws IOException { + Path tempDir = Files.createTempDirectory("test-upload-symlink"); + Path realDir = Files.createTempDirectory("test-upload-target"); + Path realFile = realDir.resolve("linked-file.txt"); + Files.write(realFile, "linked-content".getBytes()); + Path file1 = tempDir.resolve("file1.txt"); + Files.write(file1, "content1".getBytes()); + Path symlink = tempDir.resolve("link-dir"); + Files.createSymbolicLink(symlink, realDir); + + try { + DirectoryUploadRequest requestNoFollow = + DirectoryUploadRequest.builder() + .localSourceDirectory(tempDir.toString()) + .prefix("uploads/") + .includeSubFolders(true) + .followSymbolicLinks(false) + .build(); + List noFollowPaths = transformer.toFilePaths(requestNoFollow); + assertEquals(1, noFollowPaths.size()); + assertTrue(noFollowPaths.contains(file1)); + + DirectoryUploadRequest requestFollow = + DirectoryUploadRequest.builder() + .localSourceDirectory(tempDir.toString()) + .prefix("uploads/") + .includeSubFolders(true) + .followSymbolicLinks(true) + .build(); + List followPaths = transformer.toFilePaths(requestFollow); + assertEquals(2, followPaths.size()); + assertTrue(followPaths.contains(file1)); + } finally { + Files.walk(realDir) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + + @Test + public void testToFilePaths_CircularSymbolicLinks() throws IOException { + Path tempDir = Files.createTempDirectory("test-circular-symlink"); + try { + Files.write(tempDir.resolve("file.txt"), "content".getBytes()); + Path dirA = Files.createDirectory(tempDir.resolve("dirA")); + Path dirB = Files.createDirectory(tempDir.resolve("dirB")); + Files.createSymbolicLink(dirA.resolve("link-to-b"), dirB); + Files.createSymbolicLink(dirB.resolve("link-to-a"), dirA); + + DirectoryUploadRequest request = + DirectoryUploadRequest.builder() + .localSourceDirectory(tempDir.toString()) + .prefix("uploads/") + .includeSubFolders(true) + .followSymbolicLinks(true) + .build(); + + assertThrows(RuntimeException.class, () -> transformer.toFilePaths(request)); + } finally { + Files.deleteIfExists(tempDir.resolve("dirA/link-to-b")); + Files.deleteIfExists(tempDir.resolve("dirB/link-to-a")); + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + + @Test + public void testToFilePaths_BrokenSymbolicLinks() throws IOException { + Path tempDir = Files.createTempDirectory("test-broken-symlink"); + try { + Path file1 = tempDir.resolve("file1.txt"); + Files.write(file1, "content".getBytes()); + Files.createSymbolicLink(tempDir.resolve("broken-link"), Paths.get("/non/existent/path")); + + DirectoryUploadRequest request = + DirectoryUploadRequest.builder() + .localSourceDirectory(tempDir.toString()) + .prefix("uploads/") + .includeSubFolders(true) + .followSymbolicLinks(true) + .build(); + + List paths = transformer.toFilePaths(request); + assertEquals(1, paths.size()); + assertTrue(paths.contains(file1)); + } finally { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + + @Test + public void testToFilePaths_NestedSymbolicLinks() throws IOException { + Path tempDir = Files.createTempDirectory("test-nested-symlink"); + Path externalDir1 = Files.createTempDirectory("test-nested-target1"); + Path externalDir2 = Files.createTempDirectory("test-nested-target2"); + try { + Files.write(tempDir.resolve("root.txt"), "root".getBytes()); + Files.write(externalDir2.resolve("deep.txt"), "deep".getBytes()); + Files.createSymbolicLink(externalDir1.resolve("link-to-dir2"), externalDir2); + Files.createSymbolicLink(tempDir.resolve("link-to-dir1"), externalDir1); + + DirectoryUploadRequest request = + DirectoryUploadRequest.builder() + .localSourceDirectory(tempDir.toString()) + .prefix("uploads/") + .includeSubFolders(true) + .followSymbolicLinks(true) + .build(); + + List paths = transformer.toFilePaths(request); + assertEquals(2, paths.size()); + assertTrue(paths.contains(tempDir.resolve("root.txt"))); + } finally { + Files.walk(externalDir2) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + Files.walk(externalDir1) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + @Test public void testToBlobKey() { Path sourceDir = Paths.get("/source"); diff --git a/examples/src/main/java/com/salesforce/multicloudj/blob/Main.java b/examples/src/main/java/com/salesforce/multicloudj/blob/Main.java index 3fdfe11b1..6b0130fad 100644 --- a/examples/src/main/java/com/salesforce/multicloudj/blob/Main.java +++ b/examples/src/main/java/com/salesforce/multicloudj/blob/Main.java @@ -348,6 +348,7 @@ public static void uploadDirectory() { .localSourceDirectory("/tmp/test-directory") // Change this to your test directory .prefix("uploads/") .includeSubFolders(true) + .followSymbolicLinks(false) // Set to true to follow symbolic links during upload .build(); System.out.println("DirectoryUploadRequest created: " + request); @@ -385,6 +386,49 @@ public static void uploadDirectory() { } } + /** + * Uploads a directory to blob storage with symbolic links enabled. When followSymbolicLinks is + * true, any symlinks in the directory will be followed and the target files/directories will be + * uploaded. + */ + public static void uploadDirectoryWithSymbolicLinks() { + AsyncBucketClient asyncClient = getAsyncBucketClient(getProvider()); + + DirectoryUploadRequest request = + DirectoryUploadRequest.builder() + .localSourceDirectory("/tmp/test-directory") + .prefix("uploads-with-symlinks/") + .includeSubFolders(true) + .followSymbolicLinks(true) + .build(); + + try { + CompletableFuture future = asyncClient.uploadDirectory(request); + DirectoryUploadResponse response = future.get(); + + getLogger().info("Directory upload with symbolic links completed"); + getLogger().info("Failed transfers: {}", response.getFailedTransfers().size()); + + if (!response.getFailedTransfers().isEmpty()) { + response + .getFailedTransfers() + .forEach( + failure -> { + getLogger() + .error( + "Failed: {} - {}", + failure.getSource(), + failure.getException().getMessage()); + }); + } else { + getLogger().info("All files (including symlinked) uploaded successfully!"); + } + } catch (Exception e) { + getLogger() + .error("Directory upload with symbolic links failed: {}", e.getMessage(), e); + } + } + /** Downloads a directory from blob storage */ public static void downloadDirectory() { // Get the AsyncBucketClient instance