Merge pull request #176 from mwang87/workbench-native

Workbench native support

Author: mwang87

download.py

@@ -11,6 +11,9 @@
 from tqdm import tqdm
 from time import sleep
 
+from download_msv import _resolve_msv_usi
+from download_workbench import _resolve_metabolomicsworkbench_usi
+
 def _get_usi_display_filename(usi):
     usi_splits = usi.split(":")
 
@@ -65,54 +68,6 @@ def _usi_to_local_filename(usi):
 
 
 
-
-def _resolve_msv_usi(usi, force_massive=False):
-    """
-    
-
-    Args:
-        usi ([type]): [description]
-        force_massive (bool, optional): [description]. Defaults to False, we try to create the url given the USI, usually for non mzML/RAW files, e.g. CDF files
-
-    Returns:
-        [type]: [description]
-    """
-
-    usi_splits = usi.split(':')
-
-    msv_usi = usi
-    if len(usi.split(":")) == 3:
-        msv_usi = "{}:scan:1".format(usi)
-    
-    lookup_url = f'https://massive.ucsd.edu/ProteoSAFe/QuerySpectrum?id={msv_usi}'
-    lookup_request = requests.get(lookup_url)
-
-    try:
-        resolution_json = lookup_request.json()
-
-        remote_path = None
-        
-        mzML_resolutions = [resolution for resolution in resolution_json["row_data"] if os.path.splitext(resolution["file_descriptor"])[1] == ".mzML"]
-        mzXML_resolutions = [resolution for resolution in resolution_json["row_data"] if os.path.splitext(resolution["file_descriptor"])[1] == ".mzXML"]
-        raw_resolutions = [resolution for resolution in resolution_json["row_data"] if os.path.splitext(resolution["file_descriptor"])[1].lower() == ".raw"]
-
-        if len(mzML_resolutions) > 0:
-            remote_path = mzML_resolutions[0]["file_descriptor"]
-        elif len(mzXML_resolutions) > 0:
-            remote_path = mzXML_resolutions[0]["file_descriptor"]
-        elif len(raw_resolutions) > 0:
-            remote_path = raw_resolutions[0]["file_descriptor"]
-
-        # Format into FTP link
-        remote_link = f"ftp://massive.ucsd.edu/{remote_path[2:]}"
-    except:
-        # We did not successfully look it up, this is the fallback try
-        if force_massive:
-            return f"ftp://massive.ucsd.edu/{usi_splits[1]}/{usi_splits[2]}"
-        raise
-
-    return remote_link
-
 def _resolve_gnps_usi(usi):
     usi_splits = usi.split(':')
 
@@ -148,23 +103,7 @@ def _resolve_mtbls_usi(usi):
 
     return remote_link
 
-def _resolve_metabolomicsworkbench_usi(usi):
-    usi_splits = usi.split(':')
-
-    # First looking 
-    dataset_accession = usi_splits[1]
-    filename = usi_splits[2]
-        
-    # Query Accession
-    url = "https://massive.ucsd.edu/ProteoSAFe/QueryDatasets?task=N%2FA&file=&pageSize=30&offset=0&query=%257B%2522full_search_input%2522%253A%2522%2522%252C%2522table_sort_history%2522%253A%2522createdMillis_dsc%2522%252C%2522query%2522%253A%257B%257D%252C%2522title_input%2522%253A%2522{}%2522%257D&target=&_=1606254845533".format(dataset_accession)
-    r = requests.get(url)
-    data_json = r.json()
-
-    msv_accession = data_json["row_data"][0]["dataset"]
-
-    msv_usi = "mzspec:{}:{}:scan:1".format(msv_accession, filename)
 
-    return _resolve_msv_usi(msv_usi)
 
 def _resolve_pxd_usi(usi):
     usi_splits = usi.split(':')

download_msv.py

@@ -0,0 +1,50 @@
+import requests
+import os
+import sys
+
+
+def _resolve_msv_usi(usi, force_massive=False):
+    """
+
+    Args:
+        usi ([type]): [description]
+        force_massive (bool, optional): [description]. Defaults to False. This is to force the url in massive even is the USI resolver from MassIVE didn't return successfully, mostly the case in CDF files
+
+    Returns:
+        [type]: [description]
+    """
+
+    usi_splits = usi.split(':')
+
+    msv_usi = usi
+    if len(usi.split(":")) == 3:
+        msv_usi = "{}:scan:1".format(usi)
+    
+    lookup_url = f'https://massive.ucsd.edu/ProteoSAFe/QuerySpectrum?id={msv_usi}'
+    lookup_request = requests.get(lookup_url)
+
+    try:
+        resolution_json = lookup_request.json()
+
+        remote_path = None
+        
+        mzML_resolutions = [resolution for resolution in resolution_json["row_data"] if os.path.splitext(resolution["file_descriptor"])[1] == ".mzML"]
+        mzXML_resolutions = [resolution for resolution in resolution_json["row_data"] if os.path.splitext(resolution["file_descriptor"])[1] == ".mzXML"]
+        raw_resolutions = [resolution for resolution in resolution_json["row_data"] if os.path.splitext(resolution["file_descriptor"])[1].lower() == ".raw"]
+
+        if len(mzML_resolutions) > 0:
+            remote_path = mzML_resolutions[0]["file_descriptor"]
+        elif len(mzXML_resolutions) > 0:
+            remote_path = mzXML_resolutions[0]["file_descriptor"]
+        elif len(raw_resolutions) > 0:
+            remote_path = raw_resolutions[0]["file_descriptor"]
+
+        # Format into FTP link
+        remote_link = f"ftp://massive.ucsd.edu/{remote_path[2:]}"
+    except:
+        # We did not successfully look it up, this is the fallback try
+        if force_massive:
+            return f"ftp://massive.ucsd.edu/{usi_splits[1]}/{usi_splits[2]}"
+        raise
+
+    return remote_link

download_workbench.py

@@ -0,0 +1,29 @@
+import requests
+from download_msv import _resolve_msv_usi
+
+def _resolve_metabolomicsworkbench_usi(usi):
+    usi_splits = usi.split(':')
+
+    # First looking 
+    dataset_accession = usi_splits[1]
+    filename = usi_splits[2]
+
+    try:
+        # Checking if Data is in Metabolomics Workbench
+        dataset_list_url = "https://www.metabolomicsworkbench.org/data/show_archive_contents_json.php?STUDY_ID={}".format(dataset_accession)
+        mw_file_list = requests.get(dataset_list_url).json()
+        for file_obj in mw_file_list:
+            if filename in file_obj["FILENAME"]:
+                return file_obj["URL"]
+    except:
+        pass
+
+    # Checking if Data is in MSV
+    url = "https://massive.ucsd.edu/ProteoSAFe/QueryDatasets?task=N%2FA&file=&pageSize=30&offset=0&query=%257B%2522full_search_input%2522%253A%2522%2522%252C%2522table_sort_history%2522%253A%2522createdMillis_dsc%2522%252C%2522query%2522%253A%257B%257D%252C%2522title_input%2522%253A%2522{}%2522%257D&target=&_=1606254845533".format(dataset_accession)
+    r = requests.get(url)
+    data_json = r.json()
+    
+    msv_accession = data_json["row_data"][0]["dataset"]
+    msv_usi = "mzspec:{}:{}:scan:1".format(msv_accession, filename)
+
+    return _resolve_msv_usi(msv_usi)

layout_misc.py

@@ -77,7 +77,9 @@
 
             html.A("LCMS from Metabolights", href="/?usi=mzspec:MTBLS1124:QC07.mzML"),
             html.Br(),
-            html.A("LCMS from Metabolomics Workbench that include MS/MS", href="/?usi=mzspec:ST000763:20160411_MB_CS00000074-1_P.mzXML"),
+            html.A("LCMS from Metabolomics Workbench Native", href="/?usi=mzspec:ST001709:Sample_01___neg.mzXML"),
+            html.Br(),
+            html.A("LCMS from Metabolomics Workbench Imported into GNPS", href="/?usi=mzspec:ST000763:20160411_MB_CS00000074-1_P.mzXML"),
             html.Br(),
             html.A("Thermo LCMS from GNPS Analysis Classical Molecular Networking Task", href="/?usi=mzspec:GNPS:TASK-5ecfcf81cb3c471698995b194d8246a0-f.MSV000085444/ccms_peak/peak/Hui_N1_fe.mzML#%7B%7D"),
             html.Br(),

test/test_usi_resolution.py

@@ -10,7 +10,7 @@ def test_resolve_remote_url():
     for record in df.to_dict(orient="records"):
         print(record["usi"])
         remote_link = download._resolve_usi_remotelink(record["usi"])
-
+        print(remote_link)
         assert(len(remote_link) > 0)
 
 # These might not work on a standard system because they include raw files
@@ -23,11 +23,11 @@ def test_resolve_remote_url():
 #         assert(os.path.exists(local_filename))
 
 
-# def test_resolve_filename():
-#     df = pd.read_csv("usi_list.tsv", sep='\t')
-#     for record in df.to_dict(orient="records"):
-#         converted_filename = download._usi_to_local_filename(record["usi"])
-#         print(record["usi"], converted_filename)
+def test_resolve_filename():
+    df = pd.read_csv("usi_list.tsv", sep='\t')
+    for record in df.to_dict(orient="records"):
+        converted_filename = download._usi_to_local_filename(record["usi"])
+        print(record["usi"], converted_filename)
 
 # def test_raw_filename():
 #     converted_filename = download._usi_to_local_filename("mzspec:PXD007600:20150416_41_F1_S28_ZT_1_4.raw")  # Should be in PRIDE

test/usi_list.tsv

@@ -30,4 +30,5 @@ mzspec:MSV000086729:ccms_peak/raw/Identification/UP_Fusion_AcX_SAT_ob_pool_1_1.m
 mzspec:PXD002854:20150414_QEp1_LC7_GaPI_SA_Serum_DT_03_150416181741.mzXML:scan:2308:[+314.188]-QQKPGQAPR/2
 mzspec:PXD002854:20150414_QEp1_LC7_GaPI_SA_Serum_DT_03_150416181741:scan:2308:[+314.188]-QQKPGQAPR/2
 mzspec:PXD002854:20150414_QEp1_LC7_GaPI_SA_Serum_DT_03_150416181741.raw:scan:2308:[+314.188]-QQKPGQAPR/2
-mzspec:PXD023650:03552_GA1_P_041575_P00_A00_30min_R1.raw
+mzspec:PXD023650:03552_GA1_P_041575_P00_A00_30min_R1.raw
+mzspec:ST001709:Sample_01___neg.mzXML