diff --git a/speasy/core/hapi/__init__.py b/speasy/core/hapi/__init__.py new file mode 100644 index 00000000..0166988b --- /dev/null +++ b/speasy/core/hapi/__init__.py @@ -0,0 +1,15 @@ +from .provider import HapiProvider +from .exceptions import ( + HapiError, + HapiRequestError, + HapiServerError, + HapiNoData, +) + +__all__ = [ + "HapiProvider", + "HapiError", + "HapiRequestError", + "HapiServerError", + "HapiNoData", +] diff --git a/speasy/core/hapi/client.py b/speasy/core/hapi/client.py new file mode 100644 index 00000000..002eebe6 --- /dev/null +++ b/speasy/core/hapi/client.py @@ -0,0 +1,148 @@ +from enum import Enum +import io +from json import JSONDecodeError +from typing import Dict, List, Mapping, Optional +from urllib.parse import urlencode + +from speasy.core import http +from speasy.core.hapi.parser import _parse_hapi_csv +from speasy.products.variable import SpeasyVariable + +from .exceptions import ( + HapiRequestError, HapiServerError, HapiNoData +) + + +class HapiEndpoint(Enum): + CAPABILITIES = "capabilities" + CATALOG = "catalog" + ABOUT = "about" + INFO = "info" + DATA = "data" + + +def _fetch_response(url: str): + response = http.get(url) + _check_response(response) + return response + + +def _check_hapi_status(data: Dict) -> None: + code = data["status"]["code"] + message = data["status"]["message"] + if code == 1201: + raise HapiNoData() + elif 1400 <= code < 1500: + raise HapiRequestError(code, message) + elif code >= 1500: + raise HapiServerError(code, message) + + +def _check_http_status(status_code: int, text: str) -> None: + if 400 <= status_code < 500: + raise HapiRequestError(status_code, text) + elif status_code >= 500: + raise HapiServerError(status_code, text) + + +def _check_response(response) -> None: + try: + _check_hapi_status(response.json()) + except (JSONDecodeError, KeyError): + _check_http_status(response.status_code, response.text) + + +class HapiClient: + def __init__(self, server_url: str): + self.server_url = server_url.rstrip('/') + self._dataset_param_name = self._init_dataset_param_name() + + def _init_dataset_param_name(self) -> str: + """ Starting from HAPI-3.0, 'id' parameter becomes 'dataset' + set by major version number + """ + version = self.get_capabilities().get("HAPI") + + if not version: + raise RuntimeError("HAPI version not provided by server") + + major = int(version.split('.')[0]) + + if major == 2: + return "id" + elif major == 3: + return "dataset" + + raise RuntimeError(f"Unsupported HAPI version: {version}") + + def _fetch_variables(self, query_parameters: Dict) -> Mapping[str, SpeasyVariable]: + parameters = query_parameters.get("parameters", []) + url = self._build_url(HapiEndpoint.DATA, query_parameters) + f = io.BytesIO(_fetch_response(url).text.encode("utf-8")) + return _parse_hapi_csv(f, parameters) + + def _build_url( + self, + endpoint: Optional[HapiEndpoint] = None, + query_parameters: Optional[Dict] = None + ) -> str: + base = f"{self.server_url}/hapi" + url = f"{base}/{endpoint.value}" if endpoint else base + + # Flatten "parameters" into a comma-separated query string + if query_parameters: + query_params_copy = query_parameters.copy() + parameters = query_params_copy.get("parameters") + if parameters: + query_params_copy["parameters"] = ",".join(parameters) + else: + query_params_copy.pop("parameters", None) + + url = f"{url}?{urlencode(query_params_copy)}" + + return url + + def _endpoint_to_json( + self, + endpoint: HapiEndpoint, + query_parameters: Optional[Dict] = None + ) -> Dict: + url = self._build_url(endpoint, query_parameters) + return _fetch_response(url).json() + + def get_hapi(self) -> str: + url = self._build_url() + with http.urlopen(url) as response: + html_page = response.text + return html_page + + def get_capabilities(self) -> Dict: + return self._endpoint_to_json(HapiEndpoint.CAPABILITIES) + + def get_catalog(self) -> Dict: + return self._endpoint_to_json(HapiEndpoint.CATALOG) + + def get_about(self) -> Dict: + return self._endpoint_to_json(HapiEndpoint.ABOUT) + + def get_info(self, dataset: str, parameters: Optional[List[str]] = None) -> Dict: + query_params = { + self._dataset_param_name: dataset, + } + if parameters is not None: + query_params["parameters"] = parameters # List[str], will be flatten in _build_url + + return self._endpoint_to_json(HapiEndpoint.INFO, query_params) + + def get_data( + self, dataset: str, start: str, stop: str, parameters: List[str] + ) -> Mapping[str, SpeasyVariable]: + query_params = { + self._dataset_param_name: dataset, + "parameters": parameters, + "start": start, + "stop": stop, + "format": "csv", + "include": "header", + } + return self._fetch_variables(query_params) diff --git a/speasy/core/hapi/exceptions.py b/speasy/core/hapi/exceptions.py new file mode 100644 index 00000000..49c88f58 --- /dev/null +++ b/speasy/core/hapi/exceptions.py @@ -0,0 +1,23 @@ +class HapiError(Exception): + pass + + +class HapiRequestError(HapiError): + """4xx — Client Error (unkown dataset, bad date format, ...) """ + def __init__(self, code: int, message: str): + self.code = code + self.message = message + super().__init__(message) + + +class HapiServerError(HapiError): + """5xx — Server Error """ + def __init__(self, code: int, message: str): + self.code = code + self.message = message + super().__init__(message) + + +class HapiNoData(HapiError): + """1201 — Valid request, but no available data on time range""" + pass diff --git a/speasy/core/hapi/parser.py b/speasy/core/hapi/parser.py new file mode 100644 index 00000000..68329a92 --- /dev/null +++ b/speasy/core/hapi/parser.py @@ -0,0 +1,21 @@ +import io +from typing import List, Mapping +from speasy.core.codecs.codec_interface import CodecInterface +from speasy.core.codecs.codecs_registry import get_codec +from speasy.core.hapi.exceptions import HapiError +from speasy.products.variable import SpeasyVariable + + +def _parse_hapi_csv( + file: io.IOBase, parameters: List[str] +) -> Mapping[str, SpeasyVariable]: + """Converts the CSV returned by /data into a SpeasyVariable. + """ + if not parameters: + raise HapiError( + f"Wrong 'parameters' argument to hapi.load_variables: {parameters}" + ) + hapi_csv_codec: CodecInterface = get_codec('hapi/csv') + variables = hapi_csv_codec.load_variables(file=file, variables=parameters, + disable_cache=True) + return variables diff --git a/speasy/core/hapi/provider.py b/speasy/core/hapi/provider.py new file mode 100644 index 00000000..a7531f64 --- /dev/null +++ b/speasy/core/hapi/provider.py @@ -0,0 +1,29 @@ +from typing import List, Mapping, Optional + +from speasy.products.variable import SpeasyVariable + +from .client import HapiClient + + +class HapiProvider: + def __init__(self, server_url: str): + self.hapi_client = HapiClient(server_url) + + def hapi(self) -> str: + return self.hapi_client.get_hapi() + + def capabilities(self) -> dict: + return self.hapi_client.get_capabilities() + + def catalog(self) -> dict: + return self.hapi_client.get_catalog() + + def about(self) -> dict: + return self.hapi_client.get_about() + + def info(self, dataset: str, parameters: Optional[List] = None) -> dict: + return self.hapi_client.get_info(dataset, parameters) + + def data(self, dataset: str, start: str, stop: str, + parameters: List[str]) -> Mapping[str, SpeasyVariable]: + return self.hapi_client.get_data(dataset, start, stop, parameters) diff --git a/tests/test_hapi.py b/tests/test_hapi.py new file mode 100644 index 00000000..5660594d --- /dev/null +++ b/tests/test_hapi.py @@ -0,0 +1,168 @@ +import unittest +from ddt import data, ddt, unpack + +from speasy.core.hapi.client import HapiClient, HapiEndpoint +from speasy.core.hapi.provider import HapiProvider +from speasy.core.hapi.exceptions import HapiRequestError, HapiServerError, HapiNoData +from speasy.products.variable import SpeasyVariable + + +AMDA_SERVER_ROOT = "https://amda.irap.omp.eu/service" +HAPITEST33_SERVER_ROOT = "https://hapi-server.org/servers/TestData3.3" +CDAWEB_SERVER_ROOT = "https://cdaweb.gsfc.nasa.gov" + +@ddt +class TestHapiClient(unittest.TestCase): + + def test_build_url(self): + hapi_client = HapiClient(AMDA_SERVER_ROOT) + amda_about_url = hapi_client._build_url(endpoint=HapiEndpoint.ABOUT) + self.assertEqual(f"{AMDA_SERVER_ROOT}/hapi/about", amda_about_url) + + def test_get_info_bad_dataset(self): + hapi_client = HapiClient(HAPITEST33_SERVER_ROOT) + with self.assertRaises(HapiRequestError) as rc: + hapi_client.get_info('wrong_dataset_name') + self.assertEqual(1406, rc.exception.code) + + def test_get_info_bad_parameter(self): + hapi_client = HapiClient(HAPITEST33_SERVER_ROOT) + with self.assertRaises(HapiRequestError) as rc: + hapi_client.get_info('dataset1', ['wrong_parameter']) + self.assertEqual(1407, rc.exception.code) + + @data( + ('dataset1', '1970-01-02', '1970-01-01', [], 1404 ), # start time >= stop time + ('dataset1', '1970-01-01', '1970-01-02', [], 1405 ), # times outside valid ranges + ('wrong_dataset_name', '1970-01-01', '1970-01-02', [], 1406 ), # wrong dataset id + ('dataset1', '1970-01-01', '1970-01-02', ['no_such_parameter'], 1407 ) # wrong dataset parameter id + ) + @unpack + def test_get_data_bad_request(self, dataset, start, stop, parameters, expected_err_code): + hapi_client = HapiClient(HAPITEST33_SERVER_ROOT) + with self.assertRaises(HapiRequestError) as rc: + hapi_client.get_data(dataset, start, stop, parameters) + self.assertEqual(expected_err_code, rc.exception.code) + + def test_get_data_good_request(self): + hapi_client = HapiClient(HAPITEST33_SERVER_ROOT) + result = hapi_client.get_data('dataset1', '1970-01-01Z', '1970-01-01T00:01:11Z', ['vector']) + self.assertIsInstance(result['vector'], SpeasyVariable) + + +@ddt +class TestHapiProvider(unittest.TestCase): + + def test_capabilities(self): + hapi_provider = HapiProvider(CDAWEB_SERVER_ROOT) + result = hapi_provider.capabilities() + self.assertEqual( result["HAPI"], "2.0") + self.assertCountEqual( result["outputFormats"], ['json', 'csv', 'binary']) + + def test_about(self): + hapi_provider = HapiProvider(HAPITEST33_SERVER_ROOT) + result = hapi_provider.about() + self.assertEqual(result["HAPI"], "3.3") + self.assertEqual( result["status"]["code"], 1200) + self.assertIsInstance(result["note"], list) + + @data ( + (HAPITEST33_SERVER_ROOT, ["data", "info"]), + (AMDA_SERVER_ROOT, ["data", "info"]), + (CDAWEB_SERVER_ROOT, ["data", "info"]), + ) + @unpack + def test_hapi(self, root_url, contents): + hapi_provider = HapiProvider(root_url) + html_hapi = hapi_provider.hapi() + self.assertIn("= stop time + ('dataset1', '1970-01-01', '1970-01-02', [], 1405 ), # times outside valid ranges + ('wrong_dataset_name', '1970-01-01', '1970-01-02', [], 1406 ), # wrong dataset id + ('dataset1', '1970-01-01', '1970-01-02', ['no_such_parameter'], 1407 ) # wrong dataset parameter id + ) + @unpack + def test_data_bad_request(self, dataset, start, stop, parameters, expected_err_code): + hapi_provider = HapiProvider(HAPITEST33_SERVER_ROOT) + with self.assertRaises(HapiRequestError) as rc: + hapi_provider.data(dataset, start, stop, parameters) + self.assertEqual(expected_err_code, rc.exception.code) + + def test_data_good_request(self): + hapi_provider = HapiProvider(HAPITEST33_SERVER_ROOT) + result = hapi_provider.data('dataset1', '1970-01-01Z', '1970-01-01T00:01:11Z', ['vector']) + + self.assertEqual(60, len(result['vector'].time)) + self.assertIsInstance(result['vector'], SpeasyVariable)