Skip to content

Commit 967bcd3

Browse files
mwang87mwang87
authored
Merge pull request #280 from mwang87/https-usage
Using HTTPS for as many things as possible
2 parents 2e52862 + 3968635 commit 967bcd3

File tree

8 files changed

+61
-27
lines changed

8 files changed

+61
-27
lines changed

‎Dockerfile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ RUN conda install -c conda-forge openjdk=11.0.9.1
3333
COPY requirements.txt .
3434
RUN pip install -r requirements.txt
3535

36+
# Install Aspera
37+
# RUN wget -qO- https://ak-delivery04-mul.dhe.ibm.com/sar/CMA/OSA/0a07f/0/ibm-aspera-connect_4.1.0.46-linux_x86_64.tar.gz | tar xvz
38+
# RUN chmod +x ibm-aspera-connect_4.1.0.46-linux_x86_64.sh
39+
# RUN ./ibm-aspera-connect_4.1.0.46-linux_x86_64.sh
40+
41+
3642
COPY . /app
3743
WORKDIR /app
3844

‎app.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,7 @@
485485
dbc.Col(
486486
dbc.InputGroup(
487487
[
488-
dbc.InputGroupText("XIC Tolerance (Da)"),
488+
dbc.InputGroupText("Tolerance (Da)"),
489489
dbc.Input(id='xic_tolerance', placeholder="Enter Da Tolerance", value="0.5"),
490490
],
491491
className="mb-3",
@@ -494,7 +494,7 @@
494494
dbc.Col(
495495
dbc.InputGroup(
496496
[
497-
dbc.InputGroupText("XIC Tolerance (ppm)"),
497+
dbc.InputGroupText("Tolerance (ppm)"),
498498
dbc.Input(id='xic_ppm_tolerance', placeholder="Enter Da Tolerance", value="10"),
499499
],
500500
className="mb-3",
@@ -503,7 +503,7 @@
503503
dbc.Col(
504504
dbc.Row(
505505
[
506-
dbc.Label("XIC Tolerance Unit", width=4.8, style={"width":"150px"}),
506+
dbc.Label("Tolerance Unit", width=4.8, style={"width":"150px"}),
507507
dcc.Dropdown(
508508
id='xic_tolerance_unit',
509509
options=[
@@ -525,7 +525,7 @@
525525
dbc.Col(
526526
dbc.InputGroup(
527527
[
528-
dbc.InputGroupText("XIC Retention Time View/Integration Limits"),
528+
dbc.InputGroupText("Retention Time View/Integration Limits"),
529529
dbc.Input(id='xic_rt_window', placeholder="Enter RT Window (e.g. 1-2 or 1.5)", value=""),
530530
],
531531
className="mb-3",
@@ -536,7 +536,7 @@
536536
dbc.Col(
537537
dbc.Row(
538538
[
539-
dbc.Label("XIC Integration", width=4.8, style={"width":"120px"}),
539+
dbc.Label("Integration", width=4.8, style={"width":"120px"}),
540540
dcc.Dropdown(
541541
id='xic_integration_type',
542542
options=[
@@ -558,7 +558,7 @@
558558
dbc.Col(
559559
dbc.Row(
560560
[
561-
dbc.Label("XIC Normalization", html_for="xic_norm", width=4.8, style={"width":"140px"}),
561+
dbc.Label("Normalization", html_for="xic_norm", width=4.8, style={"width":"140px"}),
562562
dbc.Col(
563563
daq.ToggleSwitch(
564564
id='xic_norm',
@@ -577,7 +577,7 @@
577577
dbc.Col(
578578
dbc.Row(
579579
[
580-
dbc.Label("XIC Grouping", width=4.8, style={"width":"120px"}),
580+
dbc.Label("Grouping", width=4.8, style={"width":"120px"}),
581581
dcc.Dropdown(
582582
id='xic_file_grouping',
583583
options=[
@@ -613,7 +613,7 @@
613613
dbc.Col(
614614
dbc.Row(
615615
[
616-
dbc.Label("TIC option", width=4.8, style={"width":"100px"}),
616+
dbc.Label("Option", width=4.8, style={"width":"100px"}),
617617
dcc.Dropdown(
618618
id='tic_option',
619619
options=[
@@ -635,7 +635,7 @@
635635
dbc.Col(
636636
dbc.Row(
637637
[
638-
dbc.Label("Show Multiple TICs", width=4.8, style={"width":"150px"}),
638+
dbc.Label("Show Multiple", width=4.8, style={"width":"150px"}),
639639
dbc.Col(
640640
daq.ToggleSwitch(
641641
id='show_multiple_tic',

‎docker-compose-dev.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ version: '3'
22
services:
33
gnpslcms-dash:
44
command: /app/run_dev_server.sh
5-
environment:
6-
VIRTUAL_HOST: mingwangbeta.ucsd.edu,gnps-lcms.wanglab.science,gnps-lcms.gnps2.org
7-
VIRTUAL_PORT: 5000
8-
LETSENCRYPT_HOST: mingwangbeta.ucsd.edu,gnps-lcms.wanglab.science,gnps-lcms.gnps2.org
9-
LETSENCRYPT_EMAIL: miw023@ucsd.edu
5+
# environment:
6+
# VIRTUAL_HOST: mingwangbeta.ucsd.edu,gnps-lcms.wanglab.science,gnps-lcms.gnps2.org
7+
# VIRTUAL_PORT: 5000
8+
# LETSENCRYPT_HOST: mingwangbeta.ucsd.edu,gnps-lcms.wanglab.science,gnps-lcms.gnps2.org
9+
# LETSENCRYPT_EMAIL: miw023@ucsd.edu
1010

1111

1212

‎download.py‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,8 @@ def _resolve_gnps_usi(usi):
9191
spectrum_dict = r.json()
9292
task = spectrum_dict["spectruminfo"]["task"]
9393
source_file = os.path.basename(spectrum_dict["spectruminfo"]["source_file"])
94+
95+
# TODO: update this to the API
9496
remote_link = "ftp://ccms-ftp.ucsd.edu/GNPS_Library_Provenance/{}/{}".format(task, source_file)
9597

9698
return remote_link
@@ -100,7 +102,15 @@ def _resolve_mtbls_usi(usi):
100102

101103
dataset_accession = usi_splits[1]
102104
filename = usi_splits[2]
103-
remote_link = "ftp://ftp.ebi.ac.uk/pub/databases/metabolights/studies/public/{}/{}".format(dataset_accession, filename)
105+
106+
# FTP Deprecated
107+
#remote_link = "ftp://ftp.ebi.ac.uk/pub/databases/metabolights/studies/public/{}/{}".format(dataset_accession, filename)
108+
109+
# HTTPS Download
110+
# Getting obfuscation code
111+
r = requests.get("https://www.ebi.ac.uk/metabolights/ws/studies/{}/files?include_raw_data=false".format(dataset_accession))
112+
obfuscation_code = r.json()["obfuscationCode"]
113+
remote_link = "https://www.ebi.ac.uk/metabolights/ws/studies/{}/download/{}?file={}".format(dataset_accession, obfuscation_code, filename)
104114

105115
return remote_link
106116

@@ -206,10 +216,12 @@ def _usi_to_ccms_path(usi):
206216
return None
207217

208218
if "MSV" in usi_splits[1]:
209-
msv_ftp = _resolve_msv_usi(usi)
210-
msv_ftp = msv_ftp.replace("ftp://massive.ucsd.edu/", "")
219+
# TODO: Update this so it works
220+
msv_url = _resolve_msv_usi(usi)
221+
msv_url = msv_url.replace("ftp://massive.ucsd.edu/", "")
222+
msv_url = msv_url.replace("https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=", "")
211223

212-
return "f.{}".format(msv_ftp)
224+
return "f.{}".format(msv_url)
213225

214226
if "GNPS" in usi_splits[1]:
215227
if "TASK-" in usi_splits[2]:
@@ -223,10 +235,11 @@ def _usi_to_ccms_path(usi):
223235
return None
224236

225237
if "ST" in usi_splits[1]:
226-
msv_ftp = _resolve_metabolomicsworkbench_usi(usi)
227-
msv_ftp = msv_ftp.replace("ftp://massive.ucsd.edu/", "")
238+
st_url = _resolve_metabolomicsworkbench_usi(usi)
239+
st_url = st_url.replace("ftp://massive.ucsd.edu/", "")
240+
st_url = st_url.replace("https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=", "")
228241

229-
return "f.{}".format(msv_ftp)
242+
return "f.{}".format(st_url)
230243

231244
if "PXD" in usi_splits[1]:
232245
return None
@@ -303,7 +316,10 @@ def _resolve_usi(usi, temp_folder="temp", cleanup=True):
303316
wget_cmd = "wget '{}' --referer '{}' -O {} --no-check-certificate 2> /dev/null".format(remote_link, remote_link, temp_filename)
304317
else:
305318
wget_cmd = "wget '{}' --referer '{}' -O {} 2> /dev/null".format(remote_link, remote_link, temp_filename)
306-
print("ZZZZZZZZZZZZZ", wget_cmd, file=sys.stderr, flush=True)
319+
320+
# DEBUG COMMAND
321+
print("DOWNLOAD WGET CMD", wget_cmd, file=sys.stderr, flush=True)
322+
307323
os.system(wget_cmd)
308324
os.rename(temp_filename, local_filename)
309325

‎download_msv.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,18 @@ def _resolve_msv_usi(usi, force_massive=False):
4141
elif len(raw_resolutions) > 0:
4242
remote_path = raw_resolutions[0]["file_descriptor"]
4343

44-
# Format into FTP link
45-
remote_link = f"ftp://massive.ucsd.edu/{remote_path[2:]}"
44+
# Format into FTP link, we are deprecating FTP
45+
# remote_link = f"ftp://massive.ucsd.edu/{remote_path[2:]}"
46+
47+
# Format into HTTPS
48+
remote_link = f"https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file={remote_path}"
4649
except:
4750
# We did not successfully look it up, this is the fallback try
4851
if force_massive:
49-
return f"ftp://massive.ucsd.edu/{usi_splits[1]}/{usi_splits[2]}"
50-
raise
52+
#return f"ftp://massive.ucsd.edu/{usi_splits[1]}/{usi_splits[2]}"
53+
remote_link = f"https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f.{usi_splits[1]}/{usi_splits[2]}"
54+
else:
55+
raise
5156

57+
print(remote_link)
5258
return remote_link

‎test/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ all:
33
pytest --durations=0 test_2d.py test_usi_resolution.py test_xic.py -n 8
44

55
resolve:
6+
pytest test_usi_resolution.py::test_resolve_remote_url -v --capture=tee-sys
67
pytest test_usi_resolution.py -v
8+
79

810
benchmark:
911
pytest --durations=0 -n 4 speed_test.py -v --profile-svg

‎test/test_usi_resolution.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ def test_resolve_remote_url():
1313
for record in df.to_dict(orient="records"):
1414
print(record["usi"])
1515
remote_link, resource_name = download._resolve_usi_remotelink(record["usi"])
16+
print("RESOLVED URL", remote_link, resource_name)
1617
assert(len(remote_link) > 0)
1718

1819
# Testing conversion

‎test/usi_list.tsv‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,7 @@ mzspec:PXD002854:20150414_QEp1_LC7_GaPI_SA_Serum_DT_03_150416181741:scan:2308:[+
3131
mzspec:ST001709:Sample_01___neg.mzXML
3232
mzspec:MSV000084754:ccms_peak/raw/Toronamide_MS3_DDA_2.mzML
3333
mzspec:MSV000086834:raw/20210210 GNPS LMCS/PA1MeOH1.aia.CDF
34-
mzspec:MSV000084169:raw/FractA1/0205045.CDF
34+
mzspec:MSV000084169:raw/FractA1/0205045.CDF
35+
mzspec:PXD002854:20150414_QEp1_LC7_GaPI_SA_Serum_DT_03_150416181741.mzXML
36+
mzspec:PXD028694:QEp21-1189_Song_SYDE2_Rep_aniso+inhib_FT.raw
37+
mzspec:MTBLS338:0307_63.CDF

0 commit comments

Comments
 (0)