Skip to content

Commit 7742f14

Browse files
authored
Merge pull request #787 from crim-ca/cov-params
2 parents e0e195e + c9f9c89 commit 7742f14

File tree

14 files changed

+149
-44
lines changed

14 files changed

+149
-44
lines changed

.github/workflows/tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
strategy:
4747
matrix:
4848
os: [ubuntu-latest]
49-
python-version: ["3.9", "3.10", "3.11", "3.12"]
49+
python-version: ["3.10", "3.11", "3.12"]
5050
allow-failure: [false]
5151
test-case: [test-unit-only, test-func-only]
5252
include:

CHANGES.rst

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,15 @@ Changes
1212

1313
Changes:
1414
--------
15-
- No change.
15+
- Update ``owslib==0.32.1`` for parameters fixes employed by *Collection Input* with ``format=ogc-coverage-collection``.
16+
- Drop support of Python 3.9 (required for ``owslib==0.32.1`` dependency).
1617

1718
Fixes:
1819
------
19-
- No change.
20+
- Fix parsing of *Collection Input* ``format=ogc-coverage-collection`` and ``format=ogc-map-collection``
21+
to provide additional parameters to the remote collection request.
22+
- Update ``pygeofilter>=0.3.1`` to resolve ``filter-lang=FES`` parser as per other filters
23+
(relates to `geopython/pygeofilter#102 <https://github.com/geopython/pygeofilter/pull/102>`_).
2024

2125
.. _changes_6.2.0:
2226

README.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@ for each process.
5454
* - releases
5555
- | |version| |commits-since| |docker_image|
5656

57-
.. |py_ver| image:: https://img.shields.io/badge/python-3.9%2B-blue.svg?logo=python
58-
:alt: Requires Python 3.9+
57+
.. |py_ver| image:: https://img.shields.io/badge/python-3.10%2B-blue.svg?logo=python
58+
:alt: Requires Python 3.10+
5959
:target: https://www.python.org/getit
6060

6161
.. |commits-since| image:: https://img.shields.io/github/commits-since/crim-ca/weaver/6.2.0.svg?logo=github

requirements.txt

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,11 +71,10 @@ mako
7171
# employed by cwltool -> schema-salad -> mistune
7272
#mistune>=2.0.3,<2.1
7373
mypy_boto3_s3
74-
numpy>=1.22.2,<2; python_version < "3.10"
75-
numpy>=1.22.2; python_version >= "3.10"
74+
numpy>=1.22.2
7675
# esgf-compute-api (cwt) needs oauthlib but doesn't add it in their requirements
7776
oauthlib
78-
owslib==0.29.3
77+
owslib==0.32.1
7978
PasteDeploy>=3.1.0; python_version >= "3.12"
8079
pint
8180
psutil
@@ -86,7 +85,7 @@ psutil
8685
# https://github.com/tschaub/ogcapi-features/tree/json-array-expression/extensions/cql/jfe
8786
# - extra 'backend-native' uses shapely for Python objects
8887
# - support FES, CQL(TEXT,JSON), CQL2(TEXT,JSON), ECQL, JFE
89-
pygeofilter[fes,backend-native]
88+
pygeofilter[fes,backend-native]>=0.3.1
9089
# pymongo>=4 breaks for some kombu combinations corresponding to pinned Celery
9190
# - https://github.com/crim-ca/weaver/issues/386
9291
# - https://github.com/celery/kombu/pull/1536
@@ -119,10 +118,7 @@ schema-salad>=8.3.20221209165047,<9
119118
shapely
120119
simplejson
121120
# urllib3 not directly required, pinned by Snyk to avoid CVE-2024-37891
122-
# Python<3.10 error via pip, avoid endless package install lookup error with botocore
123-
# (https://github.com/pypa/pip/issues/12827#issuecomment-2211291150)
124-
urllib3>=2.2.2 ; python_version >= "3.10"
125-
urllib3==1.26.19 ; python_version < "3.10" # pyup: ignore
121+
urllib3>=2.2.2
126122
urlmatch
127123
xmltodict
128124
webob

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ def _parse_requirements(file_path, requirements, links):
120120
package_data={"": ["*.mako"]},
121121
zip_safe=False,
122122
test_suite="tests",
123-
python_requires=">=3.9, <4",
123+
python_requires=">=3.10, <4",
124124
install_requires=REQUIREMENTS,
125125
dependency_links=LINKS,
126126
extras_require={

tests/functional/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
import os
22

33
APP_PKG_ROOT = os.path.join(os.path.dirname(__file__), "application-packages")
4+
TEST_DATA_ROOT = os.path.join(os.path.dirname(__file__), "test-data")

tests/functional/test-data/test.nc

39.1 KB
Binary file not shown.

tests/functional/test_wps_package.py

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from parameterized import parameterized
2929

3030
from tests import resources
31+
from tests.functional import TEST_DATA_ROOT
3132
from tests.functional.utils import ResourcesUtil, WpsConfigBase
3233
from tests.utils import (
3334
MOCK_AWS_REGION,
@@ -2354,7 +2355,6 @@ def test_execute_job_with_collection_input_ogc_features(self, filter_method, fil
23542355
filter_match = responses.matchers.query_param_matcher({
23552356
"filter": filter_value,
23562357
"filter-lang": filter_lang,
2357-
"timeout": "10", # added internally by collection processor
23582358
})
23592359
else:
23602360
filter_match = responses.matchers.json_params_matcher(filter_value)
@@ -2489,6 +2489,89 @@ def test_execute_job_with_collection_input_stac_items(self):
24892489
assert "features" in out_data and isinstance(out_data["features"], list)
24902490
assert len(out_data["features"]) == 1
24912491

2492+
@parameterized.expand([
2493+
(
2494+
{"subset": "Lat(10:20),Lon(30:40)", "datetime": "2025-01-01/2025-01-02"},
2495+
"?subset=Lat(10:20),Lon(30:40)&datetime=2025-01-01/2025-01-02",
2496+
),
2497+
(
2498+
{"subset": {"Lat": [10, 20], "Lon": [30, 40]}, "datetime": ["2025-01-01", "2025-01-02"]},
2499+
"?subset=Lat(10:20),Lon(30:40)&datetime=2025-01-01/2025-01-02",
2500+
),
2501+
])
2502+
def test_execute_job_with_collection_input_coverages_netcdf(self, coverage_parameters, coverage_request):
2503+
# type: (JSON, str) -> None
2504+
proc_name = "DockerNetCDF2Text"
2505+
body = self.retrieve_payload(proc_name, "deploy", local=True)
2506+
cwl = self.retrieve_payload(proc_name, "package", local=True)
2507+
body["executionUnit"] = [{"unit": cwl}]
2508+
proc_id = self.fully_qualified_test_name(self._testMethodName)
2509+
self.deploy_process(body, describe_schema=ProcessSchema.OGC, process_id=proc_id)
2510+
2511+
with contextlib.ExitStack() as stack:
2512+
tmp_host = "https://mocked-file-server.com" # must match collection prefix hostnames
2513+
tmp_svr = stack.enter_context(responses.RequestsMock(assert_all_requests_are_fired=False))
2514+
test_file = "test.nc"
2515+
test_data = stack.enter_context(open(os.path.join(TEST_DATA_ROOT, test_file), mode="rb")).read()
2516+
2517+
# coverage request expected with resolved query parameters matching submitted collection input parameters
2518+
col_url = f"{tmp_host}/collections/climate-data"
2519+
col_cov_url = f"{col_url}/coverage"
2520+
col_cov_req = f"{col_cov_url}{coverage_request}"
2521+
tmp_svr.add("GET", col_cov_req, body=test_data)
2522+
2523+
col_exec_body = {
2524+
"mode": ExecuteMode.ASYNC,
2525+
"response": ExecuteResponse.DOCUMENT,
2526+
"inputs": {
2527+
"input_nc": {
2528+
"collection": col_url,
2529+
"format": ExecuteCollectionFormat.OGC_COVERAGE, # NOTE: this is the test!
2530+
"type": ContentType.APP_NETCDF, # must align with process input media-type
2531+
**coverage_parameters,
2532+
}
2533+
}
2534+
}
2535+
2536+
for mock_exec in mocked_execute_celery():
2537+
stack.enter_context(mock_exec)
2538+
proc_url = f"/processes/{proc_id}/execution"
2539+
resp = mocked_sub_requests(self.app, "post_json", proc_url, timeout=5,
2540+
data=col_exec_body, headers=self.json_headers, only_local=True)
2541+
assert resp.status_code in [200, 201], f"Failed with: [{resp.status_code}]\nReason:\n{resp.json}"
2542+
2543+
status_url = resp.json["location"]
2544+
results = self.monitor_job(status_url)
2545+
assert "output_txt" in results
2546+
2547+
job_id = status_url.rsplit("/", 1)[-1]
2548+
log_url = f"{status_url}/logs"
2549+
log_txt = self.app.get(log_url, headers={"Accept": ContentType.TEXT_PLAIN}).text
2550+
cov_col = "coverage.nc" # file name applied by 'collection_processor' (resolved by 'format' + 'type' extension)
2551+
cov_out = "coverage.txt" # extension modified by invoked process from input file name, literal copy of NetCDF
2552+
assert cov_col in log_txt, "Resolved NetCDF file from collection handler should have been logged."
2553+
assert cov_out in log_txt, "Chained NetCDF copied by the process as text should have been logged."
2554+
2555+
wps_dir = get_wps_output_dir(self.settings)
2556+
job_dir = os.path.join(wps_dir, job_id)
2557+
job_out = os.path.join(job_dir, "output_txt", cov_out)
2558+
assert os.path.isfile(job_out), f"Invalid output file not found: [{job_out}]"
2559+
with open(job_out, mode="rb") as out_fd: # output, although ".txt" is actually a copy of the submitted NetCDF
2560+
out_data = out_fd.read(3)
2561+
assert out_data == b"CDF", "Output file from (collection + process) chain should contain the NetCDF header."
2562+
2563+
for file_path in [
2564+
os.path.join(job_dir, cov_col),
2565+
os.path.join(job_dir, "inputs", cov_col),
2566+
os.path.join(job_dir, "output_txt", cov_col),
2567+
os.path.join(job_out, cov_col),
2568+
os.path.join(job_out, "inputs", cov_col),
2569+
os.path.join(job_out, "output_txt", cov_col),
2570+
]:
2571+
assert not os.path.exists(file_path), (
2572+
f"Intermediate collection coverage file should not exist: [{file_path}]"
2573+
)
2574+
24922575
def test_execute_job_with_context_output_dir(self):
24932576
cwl = {
24942577
"cwlVersion": "v1.0",

weaver/compat.py

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,3 @@ def patch(self) -> int:
3838
@property
3939
def micro(self) -> int:
4040
return self.patch
41-
42-
try:
43-
from functools import cache # pylint: disable=unused-import # definition for other modules to import
44-
except ImportError: # python<3.9 # pragma: no cover
45-
from functools import lru_cache
46-
cache = lru_cache(maxsize=None)

weaver/formats.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import os
66
import re
77
import socket
8+
from functools import cache
89
from typing import TYPE_CHECKING, cast, overload
910
from urllib.error import HTTPError, URLError
1011
from urllib.request import urlopen
@@ -17,7 +18,6 @@
1718
from requests.exceptions import ConnectionError
1819

1920
from weaver.base import Constants, classproperty
20-
from weaver.compat import cache
2121

2222
if TYPE_CHECKING:
2323
from typing import Any, AnyStr, Dict, List, Optional, Tuple, TypeAlias, TypeVar, Union

0 commit comments

Comments
 (0)