I26 use urls

notebooks/demo_individual/demo_parameters.ipynb

@@ -9,7 +9,10 @@
     "from birdy import WPSClient\n",
     "from pkg_resources import resource_filename\n",
     "\n",
-    "import requests"
+    "import requests\n",
+    "\n",
+    "# For mixed demo\n",
+    "from tempfile import NamedTemporaryFile"
    ]
   },
   {
@@ -43,7 +46,7 @@
      "data": {
       "text/plain": [
        "\u001b[0;31mType:\u001b[0m            WPSClient\n",
-       "\u001b[0;31mString form:\u001b[0m     <birdy.client.base.WPSClient object at 0x7fa7f84f42e8>\n",
+       "\u001b[0;31mString form:\u001b[0m     <birdy.client.base.WPSClient object at 0x7f07302baeb8>\n",
        "\u001b[0;31mFile:\u001b[0m            ~/osprey-venv/lib/python3.6/site-packages/birdy/client/base.py\n",
        "\u001b[0;31mDocstring:\u001b[0m      \n",
        "A Web Processing Service for Climate Data Analysis.\n",
@@ -145,8 +148,8 @@
     }
    ],
    "source": [
-    "cfg_file = resource_filename(\"tests\", \"data/samples/sample_parameter_config.cfg\")\n",
-    "print(cfg_file)"
+    "cfg_file_local = resource_filename(\"tests\", \"data/samples/sample_parameter_config.cfg\")\n",
+    "print(cfg_file_local)"
    ]
   },
   {
@@ -157,7 +160,7 @@
    "source": [
     "# run parameters\n",
     "output = osprey.parameters(\n",
-    "    config = cfg_file\n",
+    "    config = cfg_file_local\n",
     ")"
    ]
   },
@@ -170,7 +173,7 @@
      "data": {
       "text/plain": [
        "parametersResponse(\n",
-       "    output='http://localhost:5002/outputs/22f2799c-dc02-11ea-a870-c86000e3f2fd/sample.rvic.prm.COLUMBIA.20200811.nc'\n",
+       "    output='http://localhost:5002/outputs/c70edd66-e893-11ea-a870-c86000e3f2fd/sample.rvic.prm.COLUMBIA.20200827.nc'\n",
        ")"
       ]
      },
@@ -182,6 +185,100 @@
    "source": [
     "output.get()"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/home/slim/osprey/tests/configs/parameter_mixed.cfg\n"
+     ]
+    }
+   ],
+   "source": [
+    "# FILE_PATHS are a mix of local paths and https urls\n",
+    "cfg_file_mixed = resource_filename(\"tests\", \"configs/parameter_mixed.cfg\")\n",
+    "print(cfg_file_mixed)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config_name = os.path.splitext(cfg_file_mixed)[0]  # Remove .cfg extension\n",
+    "with NamedTemporaryFile(\n",
+    "    suffix=\".cfg\", prefix=os.path.basename(config_name), mode=\"w+t\"\n",
+    ") as temp_config:  # Avoid permanent replacement of https URLs\n",
+    "    read_config = open(cfg_file_mixed, \"r\")\n",
+    "    temp_config.writelines(read_config.read())\n",
+    "    temp_config.read()\n",
+    "    output = osprey.parameters(\n",
+    "        config = temp_config.name\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "parametersResponse(\n",
+       "    output='http://localhost:5002/outputs/ce1ee182-e893-11ea-a870-c86000e3f2fd/sample.rvic.prm.COLUMBIA.20200827.nc'\n",
+       ")"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "output.get()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

osprey/cli.py

@@ -152,7 +152,7 @@ def stop():
 @click.option(
     "--parallelprocesses",
     metavar="INT",
-    default="2",
+    default="20",
     help="parallelprocesses in PyWPS configuration.",
 )
 @click.option(

osprey/processes/wps_parameters.py

@@ -17,12 +17,13 @@
     log_level,
     nc_output,
 )
-from osprey.utils import logger
+from osprey.utils import logger, replace_urls
 
 # Library imports
 import os
 import json
 from datetime import datetime
+from pkg_resources import resource_filename
 
 
 class Parameters(Process):
@@ -97,6 +98,7 @@ def _handler(self, request, response):
             logger.info(version.short_version)
 
         (config, np, loglevel) = self.collect_args(request)
+        replace_urls(config, self.workdir)
         log_handler(
             self,
             response,

osprey/testing.py

@@ -0,0 +1,28 @@
+from pkg_resources import resource_filename
+
+
+def make_mock_urls(config, requests_mock):
+    """Create mock get requests for urls in
+    config file
+
+    Since it is possible for a test file to not
+    exist on THREDDS, requests_mock is used to
+    get these urls, and their content is the corresponding
+    data coming from the same file in local storage.
+
+    Parameters:
+        config (str): Path to config file
+        requests_mock: requests_mock fixture
+    """
+    read_config = open(config, "r")
+    config_data = read_config.readlines()
+    read_config.close()
+    for line in config_data:
+        if "https" in line:
+            url = line.split(" ")[-1]  # https url is last word in line
+            url = url.rstrip()  # remove \n character at end
+            filename = url.split("/")[-1]
+            f = open(resource_filename("tests", f"data/samples/{filename}"), "rb")
+            filedata = f.read()
+            f.close()
+            requests_mock.get(url, content=filedata)

osprey/utils.py

@@ -3,7 +3,9 @@
 import logging
 import os
 import json
+import requests
 from datetime import datetime, timedelta
+from tempfile import NamedTemporaryFile
 
 logger = logging.getLogger("PYWPS")
 logger.setLevel(logging.NOTSET)
@@ -34,6 +36,38 @@ def replace_filenames(config, temp_config):
     temp_config.writelines(newdata)
 
 
+def replace_urls(config, outdir):
+    """
+    Copy https URLs to local storage and replace URLs
+    with local paths in config file.
+    Parameters:
+        config (str): Config file
+        outdir (str): Output directory
+    """
+    read_config = open(config, "r")
+    filedata = read_config.readlines()
+    read_config.close()
+
+    for i in range(len(filedata)):
+        if "https" in filedata[i]:
+            url = filedata[i].split(" ")[-1]  # https url is last word in line
+            url = url.rstrip()  # remove \n character at end
+            r = requests.get(url)
+            filename = url.split("/")[-1]
+            prefix, suffix = filename.split(".")
+            suffix = "." + suffix
+            local_file = NamedTemporaryFile(
+                suffix=suffix, prefix=prefix, dir=outdir, delete=False
+            )
+            local_file.write(r.content)
+            filedata[i] = filedata[i].replace(url, local_file.name)
+
+    write_config = open(config, "w")
+    for line in filedata:
+        write_config.write(f"{line}")
+    write_config.close()
+
+
 def config_hander(workdir, unprocessed, config_template):
     """
     This function enables users to provide dictionary-like string for Configuration input.

requirements.txt

@@ -2,6 +2,6 @@ pywps>=4.2
 jinja2
 click
 psutil
-rvic==1.1.0post1
+rvic==1.1.1
 nchelpers==5.5.7
 wps-tools==0.1.2

requirements_dev.txt

@@ -1,6 +1,7 @@
 pytest
 flake8
 pytest-flake8
+requests-mock
 ipython
 pytest-notebook
 nbsphinx

tests/configs/parameter_opendap.cfg → tests/configs/parameter_https.cfg

@@ -73,7 +73,7 @@ SEARCH_FOR_CHANNEL: False
 #-- Path to Pour Points File (char) --#
 # A comma separated file of outlets to route to [lons, lats] - one coordinate pair per line (order not important)
 # May optionally include a column [names] - which will (if not aggregating) be included in param file
-FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/RVIC/sample_pour.txt
+FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets/RVIC/sample_pour.txt
 
 #-- ====================================== --#
 
@@ -83,7 +83,7 @@ FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/da
 # This defines the unit hydrograph to rout flow to the edge of each grid cell.
 # A comma separated file of [time in seconds, unit hydrograph ordinate] - one timestep per line
 # The timestep should be 1hr (3600 sec) or less.
-FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/RVIC/uhbox.csv
+FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets/RVIC/uhbox.csv
 
 #-- Number of Header lines to ignore in [UH_BOX]FILE_NAME (INT) --#
 HEADER_LINES = 1
@@ -92,7 +92,7 @@ HEADER_LINES = 1
 [ROUTING]
 #-- ====================================== --#
 #-- Path to routing inputs netcdf (char) --#
-FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/RVIC/sample_flow_parameters.nc
+FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets/RVIC/sample_flow_parameters.nc
 
 #-- netCDF Variable Names --#
 LONGITUDE_VAR: lon
@@ -124,7 +124,7 @@ CELL_FLOWDAYS: 4
 [DOMAIN]
 #-- ====================================== --#
 #-- Path to cesm compliant domain file (char) --#
-FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/dodsC/datasets/RVIC/sample_routing_domain.nc
+FILE_NAME: https://docker-dev03.pcic.uvic.ca/twitcher/ows/proxy/thredds/fileServer/datasets/RVIC/sample_routing_domain.nc
 
 #-- netCDF Variable Names --#
 LONGITUDE_VAR: lon