Implement multifile downloads

Author: kylstevenson

README.md

@@ -148,13 +148,24 @@ Map<String, DatasetSeries> members = fusion.listDatasetMembers("my-catalog", "my
 ```java
 Map<String, Distribution> distributions = fusion.listDistributions("my-catalog", "my-dataset", "my-series-member");
 ```
-6. Download as a file
+6. List the files available in a distribution
+```java
+List<String> files = fusion.listDistributionFiles("my-dataset", "my-series-member", "csv", "my-catalog", 0);
+```
+7. Download all files in a distribution
 ```java
 fusion.download("my-catalog", "my-dataset", "my-series-member", "csv", "/downloads/distributions");
 ```
-7. Download as a stream
+8. Download specific files in a distribution
+```java
+List<String> fileNames = Arrays.asList("file1", "file2");
+fusion.download("my-catalog", "my-dataset", "my-series-member", "csv", "/downloads/distributions", fileNames);
+```
+9. Download as a stream (returns a Map of file names to InputStreams)
 ```java
-InputStream is = fusion.downloadStream("my-catalog", "my-dataset", "my-series-member", "csv");
+Map<String, InputStream> streams = fusion.downloadStream("my-catalog", "my-dataset", "my-series-member", "csv");
+// Access individual file streams
+InputStream fileStream = streams.get("file1");
 ```
 
 #### Logging

src/main/java/io/github/jpmorganchase/fusion/Fusion.java

@@ -2,6 +2,8 @@
 
 import static io.github.jpmorganchase.fusion.filter.DatasetFilter.filterDatasets;
 
+import com.google.gson.JsonArray;
+import com.google.gson.JsonParser;
 import io.github.jpmorganchase.fusion.api.APIManager;
 import io.github.jpmorganchase.fusion.api.FusionAPIManager;
 import io.github.jpmorganchase.fusion.api.exception.APICallException;
@@ -24,13 +26,15 @@
 import io.github.jpmorganchase.fusion.parsing.DefaultGsonConfig;
 import io.github.jpmorganchase.fusion.parsing.GsonAPIResponseParser;
 import io.github.jpmorganchase.fusion.parsing.ParsingException;
-import java.io.IOException;
 import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
 import java.nio.file.Files;
-import java.nio.file.InvalidPathException;
 import java.nio.file.Paths;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -281,6 +285,46 @@ public Map<String, Dataset> listDatasets(String catalogName, String contains, bo
         return filterDatasets(responseParser.parseDatasetResponse(json, catalogName), contains, idContains);
     }
 
+    /**
+     * List distribution file names for a given dataset series and format.
+     * <p>
+     * This calls the Fusion API, parses the JSON response, and returns a list of
+     * file names.
+     * </p>
+     *
+     * @param dataset     dataset identifier
+     * @param series      series member identifier
+     * @param fileFormat  file format / distribution identifier
+     * @param catalog     catalog identifier
+     * @param maxResults  maximum number of file names to return; if = 0, all are returned
+     * @return list of file names
+     * @throws APICallException   if the call to the Fusion API fails
+     * @throws ParsingException   if the response cannot be parsed
+     * @throws OAuthException     if a token could not be retrieved for authentication
+     */
+    public List<String> listDistributionFiles(
+            String dataset, String series, String fileFormat, String catalog, int maxResults) {
+
+        String url = String.format(
+                "%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s/files",
+                this.rootURL, catalog, dataset, series, fileFormat);
+
+        String json = this.api.callAPI(url);
+
+        if (json == null || json.trim().isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        JsonArray arr = JsonParser.parseString(json).getAsJsonArray();
+        List<String> files = new ArrayList<>();
+
+        for (int i = 0; i < arr.size() && (maxResults <= 0 || i < maxResults); i++) {
+            files.add(arr.get(i).getAsString());
+        }
+
+        return files;
+    }
+
     /**
      * Get a list of the datasets in the specified catalog
      *
@@ -498,7 +542,7 @@ public Map<String, Distribution> listDistributions(String dataset, String series
      * @throws OAuthException if a token could not be retrieved for authentication
      */
     public void download(String catalogName, String dataset, String seriesMember, String distribution, String path) {
-        download(catalogName, dataset, seriesMember, distribution, path, new HashMap<>());
+        download(catalogName, dataset, seriesMember, distribution, path, (List<String>) null);
     }
 
     /**
@@ -509,31 +553,82 @@ public void download(String catalogName, String dataset, String seriesMember, St
      * @param seriesMember a String representing the series member identifier.
      * @param distribution a String representing the distribution identifier, this is the file extension.
      * @param path         the absolute file path where the file should be written.
-     * @param headers       http headers to be provided in the request.  For headers with multiple instances, the value should be a csv list
+     * @param fileNames    a list of file names to download.
      * @throws APICallException if the call to the Fusion API fails
      * @throws FileDownloadException if there is an issue handling the response from Fusion API
      * @throws OAuthException if a token could not be retrieved for authentication
      */
+    public void download(
+            String catalogName,
+            String dataset,
+            String seriesMember,
+            String distribution,
+            String path,
+            List<String> fileNames) {
+        download(catalogName, dataset, seriesMember, distribution, path, new HashMap<>(), fileNames);
+    }
+
     public void download(
             String catalogName,
             String dataset,
             String seriesMember,
             String distribution,
             String path,
             Map<String, String> headers) {
+        download(catalogName, dataset, seriesMember, distribution, path, headers, null);
+    }
+
+    public void download(
+            String catalogName,
+            String dataset,
+            String seriesMember,
+            String distribution,
+            String path,
+            Map<String, String> headers,
+            List<String> fileNames) {
+
+        if (fileNames == null || fileNames.isEmpty()) {
+            fileNames = listDistributionFiles(dataset, seriesMember, distribution, catalogName, 0);
+        }
+
+        if (fileNames == null || fileNames.isEmpty()) {
+            throw new FusionException(String.format(
+                    "No files found to download for catalog=%s, dataset=%s, series=%s, distribution=%s",
+                    catalogName, dataset, seriesMember, distribution));
+        }
 
-        String url = String.format(
-                "%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s",
-                this.rootURL, catalogName, dataset, seriesMember, distribution);
         try {
             Files.createDirectories(Paths.get(path));
-        } catch (InvalidPathException | IOException e) {
-            throw new FusionException(String.format("Unable to save to target path %s", path), e);
+        } catch (Exception e) {
+            throw new FusionException("Unable to save to " + path, e);
+        }
+
+        for (String fileName : fileNames) {
+
+            //            String safeFileName = (fileName.equalsIgnoreCase(distribution)
+            //                            ? String.format("%s_%s_%s", catalogName, dataset, seriesMember)
+            //                            : fileName)
+            //                    .replaceAll("[^a-zA-Z0-9_.\\-]", "_");
+            String safeFileName = fileName.replaceAll("[^a-zA-Z0-9_.\\-]", "_");
+            String fullPath = path + "/" + safeFileName + "." + distribution;
+
+            String url = null;
+            try {
+                url = String.format(
+                        "%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s?file=%s",
+                        this.rootURL,
+                        catalogName,
+                        dataset,
+                        seriesMember,
+                        distribution,
+                        URLEncoder.encode(fileName, "UTF-8"));
+
+            } catch (UnsupportedEncodingException e) {
+                throw new RuntimeException(e);
+            }
+
+            this.api.callAPIFileDownload(url, fullPath, catalogName, dataset, headers);
         }
-        String fileName =
-                String.format("%s_%s_%s", catalogName, dataset, seriesMember).replaceAll("[^a-zA-Z0-9_\\-]", "_");
-        String filepath = String.format("%s/%s.%s", path, fileName, distribution);
-        this.api.callAPIFileDownload(url, filepath, catalogName, dataset, headers);
     }
 
     /**
@@ -548,7 +643,24 @@ public void download(
      * @throws OAuthException if a token could not be retrieved for authentication
      */
     public void download(String catalogName, String dataset, String seriesMember, String distribution) {
-        this.download(catalogName, dataset, seriesMember, distribution, getDefaultPath(), new HashMap<>());
+        this.download(catalogName, dataset, seriesMember, distribution, getDefaultPath(), new HashMap<>(), null);
+    }
+
+    /**
+     * Download a single distribution to the local filesystem. By default, this will write to downloads folder.
+     *
+     * @param catalogName  identifier of the catalog to be queried
+     * @param dataset      a String representing the dataset identifier to download.
+     * @param seriesMember a String representing the series member identifier.
+     * @param distribution a String representing the distribution identifier, this is the file extension.
+     * @param fileNames    a list of file names to download.
+     * @throws APICallException if the call to the Fusion API fails
+     * @throws FileDownloadException  if there is an issue handling the response from Fusion API
+     * @throws OAuthException if a token could not be retrieved for authentication
+     */
+    public void download(
+            String catalogName, String dataset, String seriesMember, String distribution, List<String> fileNames) {
+        this.download(catalogName, dataset, seriesMember, distribution, getDefaultPath(), new HashMap<>(), fileNames);
     }
 
     /**
@@ -565,7 +677,30 @@ public void download(String catalogName, String dataset, String seriesMember, St
      */
     public void download(
             String catalogName, String dataset, String seriesMember, String distribution, Map<String, String> headers) {
-        this.download(catalogName, dataset, seriesMember, distribution, getDefaultPath());
+        download(catalogName, dataset, seriesMember, distribution, headers, null);
+    }
+
+    /**
+     * Download a single distribution to the local filesystem. By default, this will write to downloads folder.
+     *
+     * @param catalogName  identifier of the catalog to be queried
+     * @param dataset      a String representing the dataset identifier to download.
+     * @param seriesMember a String representing the series member identifier.
+     * @param distribution a String representing the distribution identifier, this is the file extension.
+     * @param headers       http headers to be provided in the request.  For headers with multiple instances, the value should be a csv list
+     * @param fileNames    a list of file names to download.
+     * @throws APICallException if the call to the Fusion API fails
+     * @throws FileDownloadException  if there is an issue handling the response from Fusion API
+     * @throws OAuthException if a token could not be retrieved for authentication
+     */
+    public void download(
+            String catalogName,
+            String dataset,
+            String seriesMember,
+            String distribution,
+            Map<String, String> headers,
+            List<String> fileNames) {
+        this.download(catalogName, dataset, seriesMember, distribution, getDefaultPath(), fileNames);
     }
 
     /**
@@ -597,11 +732,9 @@ public void download(String catalogName, String dataset, List<String> seriesMemb
      * @throws FileDownloadException if there is an issue handling the response from Fusion API
      * @throws OAuthException if a token could not be retrieved for authentication
      */
-    public InputStream downloadStream(String catalogName, String dataset, String seriesMember, String distribution) {
-        String url = String.format(
-                "%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s",
-                this.rootURL, catalogName, dataset, seriesMember, distribution);
-        return downloadStream(catalogName, dataset, seriesMember, distribution, new HashMap<>());
+    public Map<String, InputStream> downloadStream(
+            String catalogName, String dataset, String seriesMember, String distribution) {
+        return downloadStream(catalogName, dataset, seriesMember, distribution, (List<String>) null);
     }
 
     /**
@@ -613,17 +746,72 @@ public InputStream downloadStream(String catalogName, String dataset, String ser
      * @param dataset      a String representing the dataset identifier to download.
      * @param seriesMember a String representing the series member identifier.
      * @param distribution a String representing the distribution identifier, this is the file extension.
-     * @param headers       http headers to be provided in the request.  For headers with multiple instances, the value should be a csv list
+     * @param fileNames    a list of file names to download.
      * @throws APICallException if the call to the Fusion API fails
      * @throws FileDownloadException if there is an issue handling the response from Fusion API
      * @throws OAuthException if a token could not be retrieved for authentication
      */
-    public InputStream downloadStream(
-            String catalogName, String dataset, String seriesMember, String distribution, Map<String, String> headers) {
+    public Map<String, InputStream> downloadStream(
+            String catalogName, String dataset, String seriesMember, String distribution, List<String> fileNames) {
         String url = String.format(
                 "%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s",
                 this.rootURL, catalogName, dataset, seriesMember, distribution);
-        return this.api.callAPIFileDownload(url, catalogName, dataset, headers);
+
+        return downloadStream(catalogName, dataset, seriesMember, distribution, new HashMap<>(), fileNames);
+    }
+
+    public Map<String, InputStream> downloadStream(
+            String catalogName, String dataset, String seriesMember, String distribution, Map<String, String> headers) {
+        return downloadStream(catalogName, dataset, seriesMember, distribution, headers, null);
+    }
+
+    public Map<String, InputStream> downloadStream(
+            String catalogName,
+            String dataset,
+            String seriesMember,
+            String distribution,
+            Map<String, String> headers,
+            List<String> fileNames) {
+
+        if (fileNames == null || fileNames.isEmpty()) {
+            fileNames = listDistributionFiles(dataset, seriesMember, distribution, catalogName, 0);
+        }
+
+        if (fileNames == null || fileNames.isEmpty()) {
+            throw new FusionException(String.format(
+                    "No files found to download for catalog=%s, dataset=%s, series=%s, distribution=%s",
+                    catalogName, dataset, seriesMember, distribution));
+        }
+
+        Map<String, InputStream> result = new HashMap<>();
+
+        for (String rawName : fileNames) {
+
+            if (rawName == null || rawName.trim().isEmpty()) {
+                continue;
+            }
+
+            String cleaned = rawName.replaceAll("[/\\\\]+$", "");
+
+            String url = null;
+            try {
+                url = String.format(
+                        "%scatalogs/%s/datasets/%s/datasetseries/%s/distributions/%s/files/operationType/download?file=%s",
+                        this.rootURL,
+                        catalogName,
+                        dataset,
+                        seriesMember,
+                        distribution,
+                        URLEncoder.encode(cleaned, "UTF-8"));
+            } catch (UnsupportedEncodingException e) {
+                throw new RuntimeException(e);
+            }
+
+            InputStream stream = this.api.callAPIFileDownload(url, catalogName, dataset, headers);
+            result.put(cleaned, stream);
+        }
+
+        return result;
     }
 
     /**