I26 use urls

osprey/processes/wps_parameters.py

@@ -17,12 +17,13 @@
     log_level,
     nc_output,
 )
-from osprey.utils import logger
+from osprey.utils import logger, replace_urls
 
 # Library imports
 import os
 import json
 from datetime import datetime
+from pkg_resources import resource_filename
 
 
 class Parameters(Process):
@@ -97,6 +98,7 @@ def _handler(self, request, response):
             logger.info(version.short_version)
 
         (config, np, loglevel) = self.collect_args(request)
+        replace_urls(config, self.workdir)
         log_handler(
             self,
             response,

osprey/utils.py

@@ -3,7 +3,9 @@
 import logging
 import os
 import json
+import requests
 from datetime import datetime, timedelta
+from tempfile import NamedTemporaryFile
 
 logger = logging.getLogger("PYWPS")
 logger.setLevel(logging.NOTSET)
@@ -34,6 +36,38 @@ def replace_filenames(config, temp_config):
     temp_config.writelines(newdata)
 
 
+def replace_urls(config, outdir):
+    """
+    Copy https URLs to local storage and replace URLs
+    with local paths in config file.
+    Parameters:
+        config (str): Config file
+        outdir (str): Output directory
+    """
+    read_config = open(config, "r")
+    filedata = read_config.readlines()
+    read_config.close()
+
+    for i in range(len(filedata)):
+        if "https" in filedata[i]:
+            url = filedata[i].split(" ")[-1]  # https url is last word in line
+            url = url.rstrip()  # remove \n character at end
+            r = requests.get(url)
+            filename = url.split("/")[-1]
+            prefix, suffix = filename.split(".")
+            suffix = "." + suffix
+            local_file = NamedTemporaryFile(
+                suffix=suffix, prefix=prefix, dir=outdir, delete=False
+            )
+            local_file.write(r.content)
+            filedata[i] = filedata[i].replace(url, local_file.name)
+
+    write_config = open(config, "w")
+    for line in filedata:
+        write_config.write(f"{line}\n")
+    write_config.close()
+
+
 def config_hander(workdir, unprocessed, config_template):
     """
     This function enables users to provide dictionary-like string for Configuration input.

requirements_dev.txt

@@ -1,6 +1,7 @@
 pytest
 flake8
 pytest-flake8
+requests-mock
 ipython
 pytest-notebook
 nbsphinx

tests/configs/parameter_opendap.cfg → tests/configs/parameter_https.cfg

@@ -73,7 +73,7 @@ SEARCH_FOR_CHANNEL: False
 #-- Path to Pour Points File (char) --#
 # A comma separated file of outlets to route to [lons, lats] - one coordinate pair per line (order not important)
 # May optionally include a column [names] - which will (if not aggregating) be included in param file
-FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/RVIC/sample_pour.txt
+FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets/RVIC/sample_pour.txt
 
 #-- ====================================== --#
 
@@ -83,7 +83,7 @@ FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/da
 # This defines the unit hydrograph to rout flow to the edge of each grid cell.
 # A comma separated file of [time in seconds, unit hydrograph ordinate] - one timestep per line
 # The timestep should be 1hr (3600 sec) or less.
-FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/RVIC/uhbox.csv
+FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets/RVIC/uhbox.csv
 
 #-- Number of Header lines to ignore in [UH_BOX]FILE_NAME (INT) --#
 HEADER_LINES = 1
@@ -92,7 +92,7 @@ HEADER_LINES = 1
 [ROUTING]
 #-- ====================================== --#
 #-- Path to routing inputs netcdf (char) --#
-FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/RVIC/sample_flow_parameters.nc
+FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets/RVIC/sample_flow_parameters.nc
 
 #-- netCDF Variable Names --#
 LONGITUDE_VAR: lon
@@ -124,7 +124,7 @@ CELL_FLOWDAYS: 4
 [DOMAIN]
 #-- ====================================== --#
 #-- Path to cesm compliant domain file (char) --#
-FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/RVIC/sample_routing_domain.nc
+FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets/RVIC/sample_routing_domain.nc
 
 #-- netCDF Variable Names --#
 LONGITUDE_VAR: lon

tests/conftest.py

@@ -0,0 +1,18 @@
+import pytest
+from pkg_resources import resource_filename
+
+
+@pytest.fixture
+def make_mock_urls(config, requests_mock):
+    read_config = open(config, "r")
+    config_data = read_config.readlines()
+    read_config.close()
+    for line in config_data:
+        if "https" in line:
+            url = line.split(" ")[-1]  # https url is last word in line
+            url = url.rstrip()  # remove \n character at end
+            filename = url.split("/")[-1]
+            f = open(resource_filename(__name__, f"data/samples/{filename}"), "rb")
+            filedata = f.read()
+            f.close()
+            requests_mock.get(url, content=filedata)

tests/test_wps_parameters.py

@@ -21,3 +21,19 @@ def test_parameters_local(config):
         temp_config.read()
         params = f"config={temp_config.name};"
         run_wps_process(Parameters(), params)
+
+
+@pytest.mark.online
+@pytest.mark.parametrize(
+    ("config"), [resource_filename(__name__, "configs/parameter_https.cfg")],
+)
+def test_parameters_https(config, make_mock_urls):
+    config_name = os.path.splitext(config)[0]  # Remove .cfg extension
+    with NamedTemporaryFile(
+        suffix=".cfg", prefix=os.path.basename(config_name), mode="w+t"
+    ) as temp_config:  # Avoid permanent replacement of https URLs
+        read_config = open(config, "r")
+        temp_config.writelines(read_config.read())
+        temp_config.read()
+        params = f"config={temp_config.name};"
+        run_wps_process(Parameters(), params)