Merge pull request #2573 from opencb/TASK-7508

TASK-7508 - Fail to download files programmatically

Author: pfurio

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java

@@ -23,6 +23,8 @@
 import org.opencb.opencga.core.response.RestResponse;
 import org.slf4j.Logger;
 
+import java.io.DataInputStream;
+
 public class CustomFilesCommandExecutor extends CustomCommandExecutor {
 
     public CustomFilesCommandExecutor(ObjectMap options, String token, ClientConfiguration clientConfiguration,
@@ -52,4 +54,9 @@ public RestResponse<File> upload(CustomFilesCommandOptions.UploadCommandOptions
         return openCGAClient.getFileClient().upload(options);
     }
 
+    public RestResponse<DataInputStream> download(CustomFilesCommandOptions.DownloadCommandOptions commandOptions) throws Exception {
+        options.put("OPENCGA_DESTINY", commandOptions.to);
+        return openCGAClient.getFileClient().download(commandOptions.file, options);
+    }
+
 }

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandOptions.java

@@ -17,6 +17,7 @@ public class CustomFilesCommandOptions {
     public JCommander jCommander;
     public CommonCommandOptions commonCommandOptions;
     public UploadCommandOptions uploadCommandOptions;
+    public DownloadCommandOptions downloadCommandOptions;
 
     public CustomFilesCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) {
 
@@ -82,4 +83,22 @@ public class UploadCommandOptions {
                 " UUID", required = false, arity = 1)
         public String study;
     }
+
+    @Parameters(commandNames = {"download"}, commandDescription ="Download file")
+    public class DownloadCommandOptions {
+
+        @ParametersDelegate
+        public CommonCommandOptions commonOptions = commonCommandOptions;
+
+        @Parameter(names = {"--file"}, description = "File id, name or path. Paths must be separated by : instead of /", required = true, arity = 1)
+        public String file;
+
+        @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1)
+        public String study;
+
+        @Parameter(names = {"--to"}, description = "Path where the file will be downloaded", arity = 1, required = true)
+        public String to;
+
+    }
+
 }

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java

@@ -737,15 +737,15 @@ private RestResponse<File> updateAnnotationSetsAnnotations() throws Exception {
     private RestResponse<DataInputStream> download() throws Exception {
         logger.debug("Executing download in Files command line");
 
-        FilesCommandOptions.DownloadCommandOptions commandOptions = filesCommandOptions.downloadCommandOptions;
-
+        CustomFilesCommandOptions.DownloadCommandOptions commandOptions = filesCommandOptions.downloadCommandOptions;
         ObjectMap queryParams = new ObjectMap();
+        queryParams.putIfNotEmpty("file", commandOptions.file);
         queryParams.putIfNotEmpty("study", commandOptions.study);
         if (queryParams.get("study") == null && OpencgaMain.isShellMode()) {
             queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy());
         }
-
-        return openCGAClient.getFileClient().download(commandOptions.file, queryParams);
+        CustomFilesCommandExecutor customFilesCommandExecutor = new CustomFilesCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger());
+        return customFilesCommandExecutor.download(commandOptions);
     }
 
     private RestResponse<FileContent> grep() throws Exception {

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/io/TextOutputWriter.java

@@ -188,8 +188,7 @@ private void manageEvents(final List<Event> events) {
     }
 
     private boolean isNotAnIdOrMessage(List<DataResult> queryResultList) {
-        return !(queryResultList.get(0).getResultType().equals("java.lang.String") &&
-                ((OpenCGAResult) queryResultList.get(0)).getNumMatches() == 1);
+        return !"java.lang.String".equals(queryResultList.get(0).getResultType()) && queryResultList.get(0).getNumMatches() == 1;
     }
 
     private boolean isEdition(List<DataResult> queryResultList) {

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/FilesCommandOptions.java

@@ -870,20 +870,6 @@ public class UpdateAnnotationSetsAnnotationsCommandOptions {
 
     }
 
-    @Parameters(commandNames = {"download"}, commandDescription ="Download file")
-    public class DownloadCommandOptions {
-    
-        @ParametersDelegate
-        public CommonCommandOptions commonOptions = commonCommandOptions;
-    
-        @Parameter(names = {"--file"}, description = "File id, name or path. Paths must be separated by : instead of /", required = true, arity = 1)
-        public String file; 
-    
-        @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1)
-        public String study; 
-    
-    }
-
     @Parameters(commandNames = {"grep"}, commandDescription ="Filter lines of the file containing the pattern")
     public class GrepCommandOptions {
 

opencga-client/src/main/python/pyopencga/commons.py

@@ -176,23 +176,28 @@ def _fetch(config, sid, category, resource, method, subcategory=None, query_id=N
             raise OpencgaInvalidToken(r.content)
         elif r.status_code == 403:
             raise OpencgaAuthorisationError(r.content)
-
         elif r.status_code != 200:
             raise Exception(r.content)
 
-        try:
-            response = r.json()
-
-            # TODO Remove deprecated response and result in future release. Added for backwards compatibility
-            if 'response' in response:
-                response['responses'] = response['response']
-            for query_result in response['responses']:
-                if 'result' in query_result:
-                    query_result['results'] = query_result['result']
-
-        except ValueError:
-            msg = 'Bad JSON format retrieved from server'
-            raise ValueError(msg)
+        if r.headers['Content-Type'] == 'application/json':
+            try:
+                response = r.json()
+
+                # TODO Remove deprecated response and result in future release. Added for backwards compatibility
+                if 'response' in response:
+                    response['responses'] = response['response']
+                for query_result in response['responses']:
+                    if 'result' in query_result:
+                        query_result['results'] = query_result['result']
+
+            except ValueError:
+                raise ValueError('Bad JSON format retrieved from server')
+        elif r.headers['Content-Type'] == 'application/octet-stream':
+            return r.content
+        else:
+            raise ValueError('Unexpected content type retrieved from server ("{}"): "{}"'.format(
+                r.headers['Content-Type'], r.content)
+            )
 
         # Setting up final_response
         if final_response is None:
@@ -324,7 +329,7 @@ def execute(config, sid, category, resource, method, subcategory=None, query_id=
                                          'options': options})
             # Setting threads as "daemon" allows main program to exit eventually
             # even if these do not finish correctly
-            t.setDaemon(True)
+            t.daemon = True
             t.start()
 
         # Loading up the queue with index and id batches for each job

opencga-client/src/main/python/pyopencga/rest_clients/_parent_rest_clients.py

@@ -1,3 +1,5 @@
+import json
+
 from pyopencga.commons import execute
 from pyopencga.rest_response import RestResponse
 from pyopencga.retry import retry
@@ -66,7 +68,11 @@ def notify_retry(exc_type, exc_val, exc_tb):
 
         if self.auto_refresh:
             self._refresh_token_client()
-        return RestResponse(response)
+
+        if isinstance(response, dict):
+            return RestResponse(response)
+        else:  # Not a JSON (e.g. /{apiVersion}/files/{file}/download)
+            return response
 
     def _get(self, category, resource, query_id=None, subcategory=None,
              second_query_id=None, **options):

opencga-server/src/main/resources/cli-config.yaml

@@ -84,6 +84,9 @@ apiConfig:
         - name: upload
           executorExtended: True
           optionExtended: True
+        - name: download
+          executorExtended: True
+          optionExtended: True
         - name: update
           subcommands:
             - name: annotationSetsAction