fix: add storeDir output to TES request

Author: JaeAeich

plugins/nf-ga4gh/src/main/nextflow/ga4gh/tes/executor/TesTaskHandler.groovy

@@ -218,8 +218,10 @@ class TesTaskHandler extends TaskHandler {
             final type = param.type == 'dir'
                 ? 'DIRECTORY'
                 : 'FILE'
-            for( final pattern : param.getFilePatterns(task.context, task.workDir) )
-                body.addOutputsItem(outItem(pattern, type))
+            for( final pattern : param.getFilePatterns(task.context, task.workDir) ) {
+                def storeDir = task.config.get('storeDir')?.toString()
+                body.addOutputsItem(outItem(pattern, type, storeDir))
+            }
         }
 
         body.setName(task.getName())
@@ -272,27 +274,32 @@ class TesTaskHandler extends TaskHandler {
         return result
     }
 
-    private TesOutput outItem( String fileName, String type = null ) {
+    private TesOutput outItem(String fileName, String type = null, String storeDir = null) {
         final result = new TesOutput()
-        if( fileName.contains('*') || fileName.contains('?') ) {
-            result.path = "$WORK_DIR/$fileName"
+
+        result.path = "$WORK_DIR/$fileName"
+
+        if (fileName.contains('*') || fileName.contains('?')) {
             result.pathPrefix = WORK_DIR
-            result.url = task.workDir.toUriString()
         }
-        else {
-            result.path = "$WORK_DIR/$fileName"
-            result.url = task.workDir.resolve(fileName).toUriString()
+
+        if (storeDir) {
+            result.url = storeDir.endsWith('/') ? "${storeDir}${fileName}" : "${storeDir}/${fileName}"
+        } else {
+            result.url = fileName.contains('*') || fileName.contains('?')
+                ? task.workDir.toUriString()
+                : task.workDir.resolve(fileName).toUriString()
+
+            // fix local path for TES Azure only when using task.workDir
+            result.url = fixTesAzureLocalPath(result.url)
         }
-        if( type != null )
-            result.type = type
 
-        // fix local path for TES Azure
-        result.url = fixTesAzureLocalPath(result.url)
+        if (type != null)
+            result.type = type
 
         log.trace "[TES] Adding OUTPUT file: $result"
         return result
     }
-
     /**
      * Fix local paths when using TES Azure, which requires the
      * following AZ path:

plugins/nf-ga4gh/src/test/nextflow/ga4gh/tes/executor/TesTaskHandlerTest.groovy

@@ -20,6 +20,7 @@ import nextflow.ga4gh.tes.client.api.TaskServiceApi
 import nextflow.ga4gh.tes.client.model.TesCreateTaskResponse
 import nextflow.ga4gh.tes.client.model.TesTask
 import nextflow.processor.TaskStatus
+import nextflow.script.params.FileOutParam
 
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -89,4 +90,78 @@ class TesTaskHandlerTest extends Specification {
         handler.requestId == '12345'
     }
 
+    def 'should add outputs to both workDir and storeDir when storeDir is configured' () {
+
+        given:
+        def executor = Mock(TesExecutor)
+        def storeDirUri = "s3://some/store/dir"
+        def workDir = Paths.get("/work/dir")
+        def fileOutParam = Mock(FileOutParam)
+        fileOutParam.type >> 'file'
+        fileOutParam.getFilePatterns(_, _) >> ['output.txt', 'result.log']
+
+        def task = Mock(TaskRun) {
+            getInputFilesMap() >> [:]
+            getOutputFilesNames() >> []
+            getOutputsByType(FileOutParam) >> [(fileOutParam): null]
+            getWorkDir() >> workDir
+        }
+        task.getName() >> 'tes-task'
+        def config = Mock(TaskConfig)
+        config.get('storeDir') >> storeDirUri
+        task.getConfig() >> config
+        task.getContainer() >> 'foo'
+        def handler = new TesTaskHandler(task, executor)
+
+        when:
+        def tesTask = handler.newTesTask()
+
+        then:
+        def outputs = tesTask.getOutputs()
+
+        def workDirOutputs = outputs.findAll { it.url.startsWith(workDir.toUriString()) }
+
+        def storeDirOutputs = outputs.findAll { it.url.startsWith(storeDirUri) }
+
+        storeDirOutputs.size() == 2
+        storeDirOutputs.any { it.url.endsWith('output.txt') }
+        storeDirOutputs.any { it.url.endsWith('result.log') }
+
+        workDirOutputs.size() >= 2
+    }
+
+    def 'should not add storeDir outputs when storeDir is not configured' () {
+
+        given:
+        def executor = Mock(TesExecutor)
+        def workDir = Paths.get("/work/dir")
+        def fileOutParam = Mock(FileOutParam)
+        fileOutParam.type >> 'file'
+        fileOutParam.getFilePatterns(_, _) >> ['output.txt']
+
+        def task = Mock(TaskRun) {
+            getInputFilesMap() >> [:]
+            getOutputFilesNames() >> []
+            getOutputsByType(FileOutParam) >> [(fileOutParam): null]
+            getWorkDir() >> workDir
+        }
+        task.getName() >> 'tes-task'
+        def config = Mock(TaskConfig)
+        config.get('storeDir') >> null
+        task.getConfig() >> config
+        task.getContainer() >> 'foo'
+        def handler = new TesTaskHandler(task, executor)
+
+        when:
+        def tesTask = handler.newTesTask()
+
+        then:
+        def outputs = tesTask.getOutputs()
+
+        def workDirOutputs = outputs.findAll { it.url.startsWith(workDir.toUriString()) }
+        workDirOutputs.size() >= 1
+
+        outputs.every { it.url.startsWith(workDir.toUriString()) }
+    }
+
 }