Merge pull request #65 from latchbio/rahuldesai1/remove-delete-on-overwrite

remove delete on write for latch files

Author: rahuldesai1

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Latch Nextflow Changelog
 
+## 2.4.1
+
+## Fixed
+* Remove deletion of latch files when overwriting
+
 ## 2.4.0
 
 ## Updated

LATCH_VERSION

@@ -1 +1 @@
-v2.3.6
+v2.4.1

modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy

@@ -16,6 +16,11 @@
 
 package nextflow.processor
 
+import nextflow.file.CopyOptions
+
+import java.nio.file.StandardCopyOption
+import java.nio.file.StandardOpenOption
+
 import static nextflow.util.CacheHelper.*
 
 import java.nio.file.FileAlreadyExistsException
@@ -425,10 +430,16 @@ class PublishDir {
 
             if( !sameRealPath && shouldOverwrite(source, destination) ) {
                 log.warn "Overwriting file at ${destination.toUriString()}"
-                FileHelper.deletePath(destination)
-                processFileImpl(source, destination)
+
+                if (destination.getFileSystem().provider().getScheme().equals("latch")) {
+                    processFileImpl(source, destination, true)
+                } else {
+                    FileHelper.deletePath(destination)
+                    processFileImpl(source, destination)
+                }
+
             } else {
-                log.debug "Skipping file. File already exists at ${destination.toUriString()}"
+                log.debug "Skipping upload. File already exists at ${destination.toUriString()}"
             }
         }
 
@@ -498,7 +509,7 @@ class PublishDir {
         return sourceHash != targetHash
     }
 
-    protected void processFileImpl( Path source, Path destination ) {
+    protected void processFileImpl( Path source, Path destination, boolean overwrite = false ) {
         log.trace "publishing file: $source -[$mode]-> $destination"
 
         if( !mode || mode == Mode.SYMLINK ) {
@@ -512,10 +523,18 @@ class PublishDir {
             FilesEx.mklink(source, [hard:true], destination)
         }
         else if( mode == Mode.MOVE ) {
-            FileHelper.movePath(source, destination)
+            if (overwrite) {
+                FileHelper.movePath(source, destination, StandardCopyOption.REPLACE_EXISTING)
+            } else {
+                FileHelper.movePath(source, destination)
+            }
         }
         else if( mode == Mode.COPY ) {
-            FileHelper.copyPath(source, destination)
+            if (overwrite) {
+                FileHelper.copyPath(source, destination, StandardCopyOption.REPLACE_EXISTING)
+            } else {
+                FileHelper.copyPath(source, destination)
+            }
         }
         else if( mode == Mode.COPY_NO_FOLLOW ) {
             FileHelper.copyPath(source, destination, LinkOption.NOFOLLOW_LINKS)

modules/nf-commons/src/main/nextflow/file/CopyMoveHelper.java

@@ -174,7 +174,10 @@ static void copyToForeignTarget(Path source, Path target, CopyOption... options)
 
         // delete target if it exists and REPLACE_EXISTING is specified
         if (opts.replaceExisting()) {
-            FileHelper.deletePath(target);
+            // rahul: latch paths can be overridden and do not require explicit delete
+            if (!target.getFileSystem().provider().getScheme().equals("latch")) {
+                FileHelper.deletePath(target);
+            }
         } else if (Files.exists(target)) {
             throw new FileAlreadyExistsException(target.toString());
         }