Merge pull request #301 from Wang-Bioinformatics-Lab/norman

supporting norman

Author: mwang87

download.py

@@ -11,6 +11,7 @@
 from download_workbench import _resolve_metabolomicsworkbench_usi
 import download_zenodo
 import download_glycopost
+import download_norman
 
 try:
     import utils_conversion
@@ -69,6 +70,9 @@ def _usi_to_local_filename(usi):
         converted_local_filename = werkzeug.utils.secure_filename(":".join(usi_splits[:3])) + ".mzML"
         converted_local_filename = converted_local_filename.replace(".mzML.mzML", ".mzML")
 
+    elif "NORMAN" in usi_splits[1]:
+        converted_local_filename = werkzeug.utils.secure_filename(":".join(usi_splits[:3]).split("?")[0]) + ".mzML"
+
     # Cleaning it up
     if len(converted_local_filename) > 250:
         converted_local_filename = "convertedfile_" + str(hash(converted_local_filename)) + "_" + converted_local_filename[-150:]
@@ -220,6 +224,9 @@ def _resolve_usi_remotelink(usi):
     elif "ZENODO" in usi_splits[1]:
         remote_link = download_zenodo._resolve_zenodo_usi(usi)
         resource = "ZENODO"
+    elif "NORMAN" in usi_splits[1]:
+        remote_link = download_norman._resolve_norman_usi(usi)
+        resource = "NORMAN"
     elif "PXD" in usi_splits[1]:
         # First lets try resolving it at MSV
         remote_link = ""
@@ -348,7 +355,12 @@ def _resolve_usi(usi, temp_folder="temp", cleanup=True):
     print("DEBUG: Remote Link", remote_link, file=sys.stderr, flush=True)
 
     # Getting Data Local, TODO: likely should serialize it
-    local_filename = os.path.join(temp_folder, "temp_" + str(uuid.uuid4()) + "_" + werkzeug.utils.secure_filename(remote_link)[-150:])
+    if "NORMAN" in usi_splits[1]:
+        # remove all the url parmaeters from url
+        remote_link_stripped = remote_link.split("?")[0]
+        local_filename = os.path.join(temp_folder, "temp_" + str(uuid.uuid4()) + "_" + werkzeug.utils.secure_filename(remote_link_stripped)[-150:])
+    else:
+        local_filename = os.path.join(temp_folder, "temp_" + str(uuid.uuid4()) + "_" + werkzeug.utils.secure_filename(remote_link)[-150:])
     filename, file_extension = os.path.splitext(local_filename)
 
     # This is the download temporary filename
@@ -372,6 +384,7 @@ def _resolve_usi(usi, temp_folder="temp", cleanup=True):
         os.system(wget_cmd)
 
     temp_msconvert_filename = os.path.join(temp_folder, "msconvert_out_" + str(uuid.uuid4()) + ".mzML")
+
     # Lets do a conversion
     # TODO: Setting timeouts to kill child processes
     if file_extension.lower() == ".cdf":

download_msv.py

@@ -22,7 +22,7 @@ def _resolve_msv_usi(usi, force_massive=False):
         msv_usi = "{}:scan:1".format(usi)
 
     lookup_url = f'https://massive.ucsd.edu/ProteoSAFe/QuerySpectrum?id={msv_usi}'
-    lookup_request = requests.get(lookup_url)
+    lookup_request = requests.get(lookup_url, verify=False) # we don't verify because massive always goes down on TLS
 
     try:
         resolution_json = lookup_request.json()

download_norman.py

@@ -0,0 +1,11 @@
+import requests
+
+def _resolve_norman_usi(usi):
+    # mzspec:NORMAN-27df0a3e-3578-4a30-b9e4-1505f9da010d:webform/sample/52/uoa_pos_25_lc-esi-qtof_life_apex_16_roach_muscle_from_river_thames_shepperton-sunbury_london_uk_01.01.2016_life_apex_34794.mzml?VersionId=dHajIPC5bwhZM1gzQPvtfR8HNfqkstJL
+    usi_splits = usi.split(':')
+
+    # First looking 
+    dataset_accession = usi_splits[1].replace("NORMAN-", "")
+    filename = usi_splits[2]
+
+    return "https://files.dsfp.norman-data.eu/{}".format(filename)

tasks_conversion.py

@@ -33,6 +33,7 @@ def _download_convert_file(usi, temp_folder="temp"):
     """
 
     return_val = download._resolve_usi(usi, temp_folder=temp_folder)
+
     _convert_file_feather.delay(usi, temp_folder=temp_folder)
 
     return return_val