From 9fd9a6bdaed7d851877b32c3ccfbd702e9c07f3b Mon Sep 17 00:00:00 2001 From: cauriol Date: Mon, 23 Mar 2026 09:28:05 +0100 Subject: [PATCH 01/16] feat: authentification for desp cache --- .../plugins/authentication/openid_connect.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index d8a7368316..1b185e005c 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -252,8 +252,9 @@ class OIDCAuthorizationCodeFlowAuth(OIDCRefreshTokenBase): * :attr:`~eodag.config.PluginConfig.token_key` (``str``): The key pointing to the token in the json response to the POST request to the token server * :attr:`~eodag.config.PluginConfig.token_provision` (``str``) (**mandatory**): One of - ``qs`` or ``header``. This is how the token obtained will be used to authenticate the - user on protected requests. If ``qs`` is chosen, then ``token_qs_key`` is mandatory + ``qs``, ``header`` or ``basic``. This is how the token obtained will be used to authenticate the + user on protected requests. If ``qs`` is chosen, then ``token_qs_key`` is mandatory. If ``basic`` is chosen, + the token is used as password with username "anonymous". * :attr:`~eodag.config.PluginConfig.login_form_xpath` (``str``) (**mandatory**): The xpath to the HTML form element representing the user login form * :attr:`~eodag.config.PluginConfig.authentication_uri_source` (``str``) (**mandatory**): Where @@ -301,9 +302,9 @@ def __init__(self, provider: str, config: PluginConfig) -> None: def validate_config_credentials(self) -> None: """Validate configured credentials""" super(OIDCAuthorizationCodeFlowAuth, self).validate_config_credentials() - if getattr(self.config, "token_provision", None) not in ("qs", "header"): + if getattr(self.config, "token_provision", None) not in ("qs", "header", "basic"): raise MisconfiguredError( - 'Provider config parameter "token_provision" must be one of "qs" or "header"' + 'Provider config parameter "token_provision" must be one of "qs", "header", or "basic"' ) if self.config.token_provision == "qs" and not getattr( self.config, "token_qs_key", "" @@ -317,10 +318,19 @@ def authenticate(self) -> CodeAuthorizedAuth: """Authenticate""" self._get_access_token() + if getattr(self.config, 'zarr', False): + return CodeAuthorizedAuth( + self.refresh_token, + "basic", + key=None, + refresh_token=self.refresh_token + ) + return CodeAuthorizedAuth( self.access_token, self.config.token_provision, key=getattr(self.config, "token_qs_key", None), + refresh_token=self.refresh_token ) def _request_new_token(self) -> dict[str, str]: @@ -583,10 +593,11 @@ def compute_state() -> str: class CodeAuthorizedAuth(AuthBase): """CodeAuthorizedAuth custom authentication class to be used with requests module""" - def __init__(self, token: str, where: str, key: Optional[str] = None) -> None: + def __init__(self, token: str, where: str, key: Optional[str] = None, refresh_token: Optional[str] = None) -> None: self.token = token self.where = where self.key = key + self.refresh_token = refresh_token def __call__(self, request: PreparedRequest) -> PreparedRequest: """Perform the actual authentication""" @@ -601,6 +612,12 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: elif self.where == "header": request.headers["Authorization"] = "Bearer {}".format(self.token) + + elif self.where == "basic": + import base64 + auth_str = base64.b64encode(f"anonymous:{self.token}".encode()).decode() + request.headers["Authorization"] = f"Basic {auth_str}" + logger.debug( re.sub( r"'Bearer [^']+'", From 640e6462a21c9853dc85e8b549f2ce305bbc4c55 Mon Sep 17 00:00:00 2001 From: jlahovnik Date: Mon, 23 Mar 2026 14:52:23 +0100 Subject: [PATCH 02/16] refactor: remove zarr config parameter --- .../plugins/authentication/openid_connect.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 1b185e005c..6187613b75 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -302,7 +302,11 @@ def __init__(self, provider: str, config: PluginConfig) -> None: def validate_config_credentials(self) -> None: """Validate configured credentials""" super(OIDCAuthorizationCodeFlowAuth, self).validate_config_credentials() - if getattr(self.config, "token_provision", None) not in ("qs", "header", "basic"): + if getattr(self.config, "token_provision", None) not in ( + "qs", + "header", + "basic", + ): raise MisconfiguredError( 'Provider config parameter "token_provision" must be one of "qs", "header", or "basic"' ) @@ -318,19 +322,11 @@ def authenticate(self) -> CodeAuthorizedAuth: """Authenticate""" self._get_access_token() - if getattr(self.config, 'zarr', False): - return CodeAuthorizedAuth( - self.refresh_token, - "basic", - key=None, - refresh_token=self.refresh_token - ) - return CodeAuthorizedAuth( self.access_token, self.config.token_provision, key=getattr(self.config, "token_qs_key", None), - refresh_token=self.refresh_token + refresh_token=self.refresh_token, ) def _request_new_token(self) -> dict[str, str]: @@ -593,7 +589,13 @@ def compute_state() -> str: class CodeAuthorizedAuth(AuthBase): """CodeAuthorizedAuth custom authentication class to be used with requests module""" - def __init__(self, token: str, where: str, key: Optional[str] = None, refresh_token: Optional[str] = None) -> None: + def __init__( + self, + token: str, + where: str, + key: Optional[str] = None, + refresh_token: Optional[str] = None, + ) -> None: self.token = token self.where = where self.key = key @@ -615,7 +617,10 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: elif self.where == "basic": import base64 - auth_str = base64.b64encode(f"anonymous:{self.token}".encode()).decode() + + auth_str = base64.b64encode( + f"anonymous:{self.refresh_token}".encode() + ).decode() request.headers["Authorization"] = f"Basic {auth_str}" logger.debug( From a8ecb5007530818f84d7c915eca3c78374489bc6 Mon Sep 17 00:00:00 2001 From: cauriol Date: Tue, 24 Mar 2026 13:00:29 +0100 Subject: [PATCH 03/16] feat: get auth headers --- .../plugins/authentication/openid_connect.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 6187613b75..128785eb8e 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -18,6 +18,7 @@ from __future__ import annotations import datetime as dt +import base64 import logging import re import string @@ -601,6 +602,19 @@ def __init__( self.key = key self.refresh_token = refresh_token + def get_auth_headers(self) -> dict[str, str]: + """Build auth headers for header-based token provisioning.""" + if self.where == "header": + return {"Authorization": f"Bearer {self.token}"} + + if self.where == "basic" and self.refresh_token is not None: + auth_str = base64.b64encode( + f"anonymous:{self.refresh_token}".encode() + ).decode() + return {"Authorization": f"Basic {auth_str}"} + + return {} + def __call__(self, request: PreparedRequest) -> PreparedRequest: """Perform the actual authentication""" if self.where == "qs": @@ -612,16 +626,8 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: request.prepare_url(url_without_args, query_dict) - elif self.where == "header": - request.headers["Authorization"] = "Bearer {}".format(self.token) - - elif self.where == "basic": - import base64 - - auth_str = base64.b64encode( - f"anonymous:{self.refresh_token}".encode() - ).decode() - request.headers["Authorization"] = f"Basic {auth_str}" + else: + request.headers.update(self.get_auth_headers()) logger.debug( re.sub( From 29d551e85121c4d8088d0947c85d2764fd0009ad Mon Sep 17 00:00:00 2001 From: cauriol Date: Wed, 25 Mar 2026 14:26:49 +0100 Subject: [PATCH 04/16] feat: tests --- tests/units/test_auth_plugins.py | 55 +++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tests/units/test_auth_plugins.py b/tests/units/test_auth_plugins.py index dcd5529439..c048f05fbd 100644 --- a/tests/units/test_auth_plugins.py +++ b/tests/units/test_auth_plugins.py @@ -17,6 +17,7 @@ # limitations under the License. import datetime as dt +import base64 import pickle import unittest from unittest import mock @@ -2202,7 +2203,7 @@ def test_plugins_auth_codeflowauth_validate_credentials(self): with self.assertRaises(MisconfiguredError) as context: auth_plugin.validate_config_credentials() self.assertTrue( - '"token_provision" must be one of "qs" or "header"' + '"token_provision" must be one of "qs", "header", or "basic"' in str(context.exception) ) # `token_provision=="qs"` but `token_qs_key` is missing @@ -3113,3 +3114,55 @@ def test_plugins_auth_codeflowauth_exchange_code_for_token_error( with self.assertRaises(TimeoutError): auth_plugin.exchange_code_for_token(authorized_url, state) + + +class TestCodeAuthorizedAuth(unittest.TestCase): + def test_get_auth_headers_for_header(self): + """CodeAuthorizedAuth.get_auth_headers must build a Bearer Authorization header in header mode.""" + auth = CodeAuthorizedAuth(token="obtained-token", where="header") + + self.assertEqual( + auth.get_auth_headers(), + {"Authorization": "Bearer obtained-token"}, + ) + + def test_get_auth_headers_for_basic(self): + """CodeAuthorizedAuth.get_auth_headers must build a Basic Authorization header from the refresh token.""" + auth = CodeAuthorizedAuth( + token="obtained-token", + where="basic", + refresh_token="obtained-refresh-token", + ) + + self.assertEqual( + auth.get_auth_headers(), + { + "Authorization": "Basic " + + base64.b64encode(b"anonymous:obtained-refresh-token").decode() + }, + ) + + def test_call_injects_basic_auth_header(self): + """CodeAuthorizedAuth.__call__ must inject the computed Basic Authorization header into the request.""" + auth = CodeAuthorizedAuth( + token="obtained-token", + where="basic", + refresh_token="obtained-refresh-token", + ) + req = Request( + "GET", + "https://httpbin.org/get", + headers={"existing-header": "value"}, + ).prepare() + + auth(req) + + self.assertEqual(req.url, "https://httpbin.org/get") + self.assertEqual( + req.headers, + { + "Authorization": "Basic " + + base64.b64encode(b"anonymous:obtained-refresh-token").decode(), + "existing-header": "value", + }, + ) From ae75a9637ae566598be0efe31e1c99b19414db10 Mon Sep 17 00:00:00 2001 From: cauriol Date: Wed, 1 Apr 2026 11:25:28 +0200 Subject: [PATCH 05/16] feat: handle zarr in eodag + tests --- eodag/api/product/_product.py | 49 +++++++++++ .../plugins/authentication/openid_connect.py | 5 +- pyproject.toml | 3 + tests/units/test_auth_plugins.py | 54 +++++++++++- tests/units/test_eoproduct.py | 88 +++++++++++++++++++ 5 files changed, 195 insertions(+), 4 deletions(-) diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py index 70ca600565..33a2bcd4e4 100644 --- a/eodag/api/product/_product.py +++ b/eodag/api/product/_product.py @@ -19,6 +19,7 @@ import base64 import datetime as dt +import json import logging import os import re @@ -660,6 +661,54 @@ def stream_download( **kwargs, ) + def _get_auth_headers(self, auth: object) -> dict[str, str]: + """Return headers exposed by an auth object when available.""" + get_auth_headers = getattr(auth, "get_auth_headers", None) + if callable(get_auth_headers): + return cast(dict[str, str], get_auth_headers()) + return {} + + def request_asset( + self, + url: str, + auth: Optional[object] = None, + ) -> requests.Response: + """Perform a GET request to the given URL with authentication if provided.""" + headers = self._get_auth_headers(auth) if auth is not None else {} + return requests.get(url, auth=auth, headers=headers, stream=True) + + def list_zarr_files_from_metadata( + self, + base_url: str, + auth: Optional[object] = None, + ) -> list[str]: + """List file paths from a Zarr store metadata file.""" + import fsspec + + headers = self._get_auth_headers(auth) if auth is not None else {} + mapper = fsspec.get_mapper( + base_url, + client_kwargs={"headers": headers, "trust_env": False}, + ) + + if ".zmetadata" in mapper: + meta = json.loads(mapper[".zmetadata"]) + return [".zmetadata", *meta["metadata"].keys()] + + """TO DO + .zmetadata is present for zarr v2, but not for v3, we should support both. + For Zarr v3, there is no .zmetata but zarr.json file instead, + we can try to read it and parse the metadata from it to list files in the store. + for exemple: + if "zarr.json" in mapper: + meta = json.loads(mapper["zarr.json"]) + return ["zarr.json", , *meta["metadata"].keys()] + We don't have v3 zarr files for testing yet, so we will implement this when we have some. + But it should be straightforward as the structure of zarr.json is similar to .zmetadata + """ + + raise ValueError(f"No Zarr metadata file found at {base_url}") + def _init_progress_bar( self, progress_callback: Optional[ProgressCallback], diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 128785eb8e..947bd59000 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -23,6 +23,7 @@ import re import string from random import SystemRandom +from threading import Lock from typing import TYPE_CHECKING, Any, Optional from urllib.parse import parse_qs, urlparse @@ -77,6 +78,7 @@ class OIDCRefreshTokenBase(Authentication): def __init__(self, provider: str, config: PluginConfig) -> None: super(OIDCRefreshTokenBase, self).__init__(provider, config) + self._auth_lock = Lock() self.access_token = "" self.access_token_expiration = dt.datetime.min.replace(tzinfo=dt.timezone.utc) @@ -321,7 +323,8 @@ def validate_config_credentials(self) -> None: def authenticate(self) -> CodeAuthorizedAuth: """Authenticate""" - self._get_access_token() + with self._auth_lock: + self._get_access_token() return CodeAuthorizedAuth( self.access_token, diff --git a/pyproject.toml b/pyproject.toml index 8072157641..2a99713749 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,9 @@ dependencies = [ "typing_extensions >= 4.8.0", "urllib3", "zipstream-ng", + "fsspec", + "aiohttp", + "requests" ] dynamic = ["version"] diff --git a/tests/units/test_auth_plugins.py b/tests/units/test_auth_plugins.py index c048f05fbd..ff6582715e 100644 --- a/tests/units/test_auth_plugins.py +++ b/tests/units/test_auth_plugins.py @@ -20,6 +20,7 @@ import base64 import pickle import unittest +from datetime import datetime, timedelta, timezone from unittest import mock import boto3 @@ -2187,12 +2188,18 @@ def get_auth_plugin(self, provider): "jwks_uri": "http://foo.bar/auth/realms/myrealm/protocol/openid-connect/certs", "id_token_signing_alg_values_supported": ["RS256", "HS512"], } - mock_request.return_value.json.side_effect = [oidc_config, oidc_config] + mock_request.return_value.json.return_value = oidc_config auth_plugin = super( TestAuthPluginOIDCAuthorizationCodeFlowAuth, self ).get_auth_plugin(provider) - # reset token info - auth_plugin.token_info = {} + auth_plugin.access_token = "" + auth_plugin.refresh_token = "" + auth_plugin.access_token_expiration = datetime.min.replace( + tzinfo=timezone.utc + ) + auth_plugin.refresh_token_expiration = datetime.min.replace( + tzinfo=timezone.utc + ) return auth_plugin def test_plugins_auth_codeflowauth_validate_credentials(self): @@ -2439,6 +2446,47 @@ def test_plugins_auth_codeflowauth_authenticate_token_qs_key_ok( self.assertEqual(auth.where, "qs") self.assertEqual(auth.key, auth_plugin.config.token_qs_key) + @mock.patch( + "eodag.plugins.authentication.openid_connect.OIDCRefreshTokenBase.decode_jwt_token", + autospec=True, + ) + @mock.patch( + "eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth._request_new_token", + autospec=True, + ) + def test_plugins_auth_codeflowauth_authenticate_basic_ok( + self, + mock_request_new_token, + mock_decode, + ): + """OIDCAuthorizationCodeFlowAuth.authenticate must return a basic auth object with the refresh token.""" + auth_plugin = self.get_auth_plugin("provider_ok") + auth_plugin.config.token_provision = "basic" + json_response = { + "access_token": "obtained-access-token", + "expires_in": "3600", + "refresh_expires_in": "7200", + "refresh_token": "obtained-refresh-token", + } + mock_request_new_token.return_value = json_response + mock_decode.return_value = { + "exp": (now_in_utc() + timedelta(seconds=3600)).timestamp() + } + + auth = auth_plugin.authenticate() + + self.assertIsInstance(auth, CodeAuthorizedAuth) + self.assertEqual(auth.token, json_response["access_token"]) + self.assertEqual(auth.where, "basic") + self.assertEqual(auth.refresh_token, json_response["refresh_token"]) + self.assertEqual( + auth.get_auth_headers(), + { + "Authorization": "Basic " + + base64.b64encode(b"anonymous:obtained-refresh-token").decode() + }, + ) + @mock.patch( "eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth.authenticate_user", autospec=True, diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py index 702ceb6f6e..f665329d4a 100644 --- a/tests/units/test_eoproduct.py +++ b/tests/units/test_eoproduct.py @@ -16,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import os import pathlib @@ -548,6 +549,93 @@ def test_eoproduct_download_http_dynamic_options(self): product_zip_file = "{}.zip".format(product_dir_path) self.assertTrue(os.path.isfile(product_zip_file)) + def test_eoproduct_get_auth_headers_uses_auth_object_helper(self): + """EOProduct._get_auth_headers must expose auth headers when available.""" + product = self._dummy_product() + auth = mock.Mock() + auth.get_auth_headers.return_value = {"Authorization": "Basic abc"} + + self.assertEqual( + product._get_auth_headers(auth), + {"Authorization": "Basic abc"}, + ) + + @mock.patch("eodag.api.product._product.requests.get") + def test_eoproduct_request_asset_passes_auth_and_headers(self, mock_get): + """EOProduct.request_asset must pass auth headers returned by the auth object.""" + product = self._dummy_product() + auth = mock.Mock() + auth.get_auth_headers.return_value = {"Authorization": "Basic abc"} + + product.request_asset("https://example.com/zarr/.zmetadata", auth) + + mock_get.assert_called_once_with( + "https://example.com/zarr/.zmetadata", + auth=auth, + headers={"Authorization": "Basic abc"}, + stream=True, + ) + + @mock.patch("fsspec.get_mapper") + def test_eoproduct_list_zarr_files_from_zmetadata(self, mock_get_mapper): + """EOProduct.list_zarr_files_from_metadata must return files listed in `.zmetadata`.""" + product = self._dummy_product() + mapper = { + ".zmetadata": json.dumps( + { + "metadata": { + ".zgroup": {}, + ".zattrs": {}, + "foo/.zarray": {}, + } + } + ) + } + mock_get_mapper.return_value = mapper + + files = product.list_zarr_files_from_metadata("https://example.com/zarr") + + mock_get_mapper.assert_called_once_with( + "https://example.com/zarr", + client_kwargs={"headers": {}, "trust_env": False}, + ) + self.assertEqual(files, [".zmetadata", ".zgroup", ".zattrs", "foo/.zarray"]) + + @mock.patch("fsspec.get_mapper") + def test_eoproduct_list_zarr_files_from_zmetadata_with_basic_auth( + self, mock_get_mapper + ): + """EOProduct.list_zarr_files_from_metadata must forward auth headers to fsspec for `.zmetadata`.""" + product = self._dummy_product() + auth = mock.Mock() + auth.get_auth_headers.return_value = {"Authorization": "Basic abc"} + mock_get_mapper.return_value = {".zmetadata": json.dumps({"metadata": {}})} + + files = product.list_zarr_files_from_metadata( + "https://example.com/zarr", + auth, + ) + + mock_get_mapper.assert_called_once_with( + "https://example.com/zarr", + client_kwargs={ + "headers": {"Authorization": "Basic abc"}, + "trust_env": False, + }, + ) + self.assertEqual(files, [".zmetadata"]) + + @mock.patch("fsspec.get_mapper") + def test_eoproduct_list_zarr_files_from_metadata_raises_when_missing( + self, mock_get_mapper + ): + """EOProduct.list_zarr_files_from_metadata must fail when metadata files are missing.""" + product = self._dummy_product() + mock_get_mapper.return_value = {} + + with self.assertRaises(ValueError): + product.list_zarr_files_from_metadata("https://example.com/zarr") + @responses.activate def test_eoproduct_download_progress_bar(self): """eoproduct.download must show a progress bar""" From 6210b1b55cd01856f652a7d4f0afbf6572be27a7 Mon Sep 17 00:00:00 2001 From: cauriol Date: Wed, 1 Apr 2026 14:25:51 +0200 Subject: [PATCH 06/16] fix: linter --- eodag/api/product/_product.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py index 33a2bcd4e4..59e8e9846d 100644 --- a/eodag/api/product/_product.py +++ b/eodag/api/product/_product.py @@ -671,7 +671,7 @@ def _get_auth_headers(self, auth: object) -> dict[str, str]: def request_asset( self, url: str, - auth: Optional[object] = None, + auth: Optional[AuthBase] = None, ) -> requests.Response: """Perform a GET request to the given URL with authentication if provided.""" headers = self._get_auth_headers(auth) if auth is not None else {} @@ -683,7 +683,7 @@ def list_zarr_files_from_metadata( auth: Optional[object] = None, ) -> list[str]: """List file paths from a Zarr store metadata file.""" - import fsspec + import fsspec # type: ignore[import-untyped] headers = self._get_auth_headers(auth) if auth is not None else {} mapper = fsspec.get_mapper( @@ -695,18 +695,10 @@ def list_zarr_files_from_metadata( meta = json.loads(mapper[".zmetadata"]) return [".zmetadata", *meta["metadata"].keys()] - """TO DO - .zmetadata is present for zarr v2, but not for v3, we should support both. - For Zarr v3, there is no .zmetata but zarr.json file instead, - we can try to read it and parse the metadata from it to list files in the store. - for exemple: - if "zarr.json" in mapper: - meta = json.loads(mapper["zarr.json"]) - return ["zarr.json", , *meta["metadata"].keys()] - We don't have v3 zarr files for testing yet, so we will implement this when we have some. - But it should be straightforward as the structure of zarr.json is similar to .zmetadata - """ - + # TODO: Support Zarr v3 when test data becomes available. + # Zarr v2 uses `.zmetadata`, while Zarr v3 exposes `zarr.json`. + # The implementation should be straightforward once we can validate it + # against real examples. raise ValueError(f"No Zarr metadata file found at {base_url}") def _init_progress_bar( From 58c8d23396e685ea207fe28831498f12617435b2 Mon Sep 17 00:00:00 2001 From: cauriol Date: Thu, 2 Apr 2026 14:47:22 +0200 Subject: [PATCH 07/16] feat: move EOProduct._get_storage_options from eodag-cube to eodag --- eodag/api/product/_product.py | 79 ++++-- .../plugins/authentication/openid_connect.py | 23 +- eodag/utils/exceptions.py | 5 + tests/context.py | 3 + tests/units/test_auth_plugins.py | 60 ----- tests/units/test_eoproduct.py | 238 ++++++++++++++++-- 6 files changed, 297 insertions(+), 111 deletions(-) diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py index 59e8e9846d..a8ade5cd4d 100644 --- a/eodag/api/product/_product.py +++ b/eodag/api/product/_product.py @@ -39,11 +39,15 @@ import orjson import requests from pystac import Item -from requests import RequestException +from boto3 import Session +from boto3.resources.base import ServiceResource +from requests import PreparedRequest, RequestException from requests.auth import AuthBase +from requests.structures import CaseInsensitiveDict from shapely import geometry from shapely.errors import ShapelyError +from eodag.plugins.authentication.aws_auth import AwsAuth from eodag.types.queryables import CommonStacMetadata from eodag.types.stac_metadata import create_stac_metadata_model @@ -84,7 +88,12 @@ _import_stac_item_from_known_provider, _import_stac_item_from_unknown_provider, ) -from eodag.utils.exceptions import DownloadError, MisconfiguredError, ValidationError +from eodag.utils.exceptions import ( + DatasetCreationError, + DownloadError, + MisconfiguredError, + ValidationError, +) from eodag.utils.repr import dict_to_html_table if TYPE_CHECKING: @@ -661,31 +670,73 @@ def stream_download( **kwargs, ) - def _get_auth_headers(self, auth: object) -> dict[str, str]: - """Return headers exposed by an auth object when available.""" - get_auth_headers = getattr(auth, "get_auth_headers", None) - if callable(get_auth_headers): - return cast(dict[str, str], get_auth_headers()) - return {} + def get_storage_options( + self, + asset_key: Optional[str] = None, + ) -> dict[str, Any]: + """ + Get fsspec storage_options keyword arguments + """ + auth = self.downloader_auth.authenticate() if self.downloader_auth else None + if self.downloader is None: + return {} + + # default url and headers + try: + url = self.assets[asset_key]["href"] if asset_key else self.location + except KeyError as e: + raise DatasetCreationError(f"{asset_key} not found in {self} assets") from e + headers = {**USER_AGENT} + + if isinstance(auth, ServiceResource) and isinstance( + self.downloader_auth, AwsAuth + ): + auth_kwargs: dict[str, Any] = dict() + # AwsAuth + if s3_endpoint := getattr(self.downloader_auth.config, "s3_endpoint", None): + auth_kwargs["client_kwargs"] = {"endpoint_url": s3_endpoint} + if creds := cast( + Session, self.downloader_auth.s3_session + ).get_credentials(): + auth_kwargs["key"] = creds.access_key + auth_kwargs["secret"] = creds.secret_key + if creds.token: + auth_kwargs["token"] = creds.token + if requester_pays := getattr( + self.downloader_auth.config, "requester_pays", False + ): + auth_kwargs["requester_pays"] = requester_pays + else: + auth_kwargs["anon"] = True + return {"path": url, **auth_kwargs} + + if isinstance(auth, AuthBase): + # update url and headers with auth + req = PreparedRequest() + req.url = url + req.headers = CaseInsensitiveDict(headers) + if auth: + auth(req) + return {"path": req.url, "headers": dict(req.headers)} + + return {"path": url} def request_asset( self, url: str, - auth: Optional[AuthBase] = None, ) -> requests.Response: - """Perform a GET request to the given URL with authentication if provided.""" - headers = self._get_auth_headers(auth) if auth is not None else {} - return requests.get(url, auth=auth, headers=headers, stream=True) + """Perform a GET request to the given URL using product's authentication headers.""" + headers = self.get_storage_options().get("headers", {}) + return requests.get(url, headers=headers, stream=True) def list_zarr_files_from_metadata( self, base_url: str, - auth: Optional[object] = None, ) -> list[str]: """List file paths from a Zarr store metadata file.""" import fsspec # type: ignore[import-untyped] - headers = self._get_auth_headers(auth) if auth is not None else {} + headers = self.get_storage_options().get("headers", {}) mapper = fsspec.get_mapper( base_url, client_kwargs={"headers": headers, "trust_env": False}, diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 947bd59000..495b630fef 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -605,19 +605,6 @@ def __init__( self.key = key self.refresh_token = refresh_token - def get_auth_headers(self) -> dict[str, str]: - """Build auth headers for header-based token provisioning.""" - if self.where == "header": - return {"Authorization": f"Bearer {self.token}"} - - if self.where == "basic" and self.refresh_token is not None: - auth_str = base64.b64encode( - f"anonymous:{self.refresh_token}".encode() - ).decode() - return {"Authorization": f"Basic {auth_str}"} - - return {} - def __call__(self, request: PreparedRequest) -> PreparedRequest: """Perform the actual authentication""" if self.where == "qs": @@ -629,8 +616,14 @@ def __call__(self, request: PreparedRequest) -> PreparedRequest: request.prepare_url(url_without_args, query_dict) - else: - request.headers.update(self.get_auth_headers()) + elif self.where == "header": + request.headers["Authorization"] = "Bearer {}".format(self.token) + + if self.where == "basic" and self.refresh_token is not None: + auth_str = base64.b64encode( + f"anonymous:{self.refresh_token}".encode() + ).decode() + request.headers["Authorization"] = f"Basic {auth_str}" logger.debug( re.sub( diff --git a/eodag/utils/exceptions.py b/eodag/utils/exceptions.py index 8ccb28b987..c5d91d25be 100644 --- a/eodag/utils/exceptions.py +++ b/eodag/utils/exceptions.py @@ -168,3 +168,8 @@ def raise_if_quota_exceeded(error: Exception, provider: str) -> None: raise QuotaExceededError( f"Too many requests on provider {provider}, please check your quota." ) + + +class DatasetCreationError(EodagError): + """An error indicating that :class:`xarray.Dataset` or :class:`eodag_cube.types.XarrayDict` could not be created""" + diff --git a/tests/context.py b/tests/context.py index da1522810b..527d045aee 100644 --- a/tests/context.py +++ b/tests/context.py @@ -58,6 +58,8 @@ from eodag.plugins.authentication.aws_auth import AwsAuth from eodag.plugins.authentication.header import HeaderAuth from eodag.plugins.authentication.openid_connect import CodeAuthorizedAuth +from eodag.plugins.authentication.header import HTTPHeaderAuth +from eodag.plugins.authentication.qsauth import HttpQueryStringAuth from eodag.plugins.base import PluginTopic from eodag.plugins.crunch.filter_date import FilterDate from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName @@ -136,6 +138,7 @@ UnsupportedProvider, ValidationError, InvalidDataError, + DatasetCreationError, ) from eodag.utils.stac_reader import fetch_stac_items, _TextOpener from tests import TEST_RESOURCES_PATH diff --git a/tests/units/test_auth_plugins.py b/tests/units/test_auth_plugins.py index ff6582715e..95dbe9ea22 100644 --- a/tests/units/test_auth_plugins.py +++ b/tests/units/test_auth_plugins.py @@ -17,7 +17,6 @@ # limitations under the License. import datetime as dt -import base64 import pickle import unittest from datetime import datetime, timedelta, timezone @@ -2479,13 +2478,6 @@ def test_plugins_auth_codeflowauth_authenticate_basic_ok( self.assertEqual(auth.token, json_response["access_token"]) self.assertEqual(auth.where, "basic") self.assertEqual(auth.refresh_token, json_response["refresh_token"]) - self.assertEqual( - auth.get_auth_headers(), - { - "Authorization": "Basic " - + base64.b64encode(b"anonymous:obtained-refresh-token").decode() - }, - ) @mock.patch( "eodag.plugins.authentication.openid_connect.OIDCAuthorizationCodeFlowAuth.authenticate_user", @@ -3162,55 +3154,3 @@ def test_plugins_auth_codeflowauth_exchange_code_for_token_error( with self.assertRaises(TimeoutError): auth_plugin.exchange_code_for_token(authorized_url, state) - - -class TestCodeAuthorizedAuth(unittest.TestCase): - def test_get_auth_headers_for_header(self): - """CodeAuthorizedAuth.get_auth_headers must build a Bearer Authorization header in header mode.""" - auth = CodeAuthorizedAuth(token="obtained-token", where="header") - - self.assertEqual( - auth.get_auth_headers(), - {"Authorization": "Bearer obtained-token"}, - ) - - def test_get_auth_headers_for_basic(self): - """CodeAuthorizedAuth.get_auth_headers must build a Basic Authorization header from the refresh token.""" - auth = CodeAuthorizedAuth( - token="obtained-token", - where="basic", - refresh_token="obtained-refresh-token", - ) - - self.assertEqual( - auth.get_auth_headers(), - { - "Authorization": "Basic " - + base64.b64encode(b"anonymous:obtained-refresh-token").decode() - }, - ) - - def test_call_injects_basic_auth_header(self): - """CodeAuthorizedAuth.__call__ must inject the computed Basic Authorization header into the request.""" - auth = CodeAuthorizedAuth( - token="obtained-token", - where="basic", - refresh_token="obtained-refresh-token", - ) - req = Request( - "GET", - "https://httpbin.org/get", - headers={"existing-header": "value"}, - ).prepare() - - auth(req) - - self.assertEqual(req.url, "https://httpbin.org/get") - self.assertEqual( - req.headers, - { - "Authorization": "Basic " - + base64.b64encode(b"anonymous:obtained-refresh-token").decode(), - "existing-header": "value", - }, - ) diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py index f665329d4a..7f6bb8f30a 100644 --- a/tests/units/test_eoproduct.py +++ b/tests/units/test_eoproduct.py @@ -38,9 +38,14 @@ from tests.context import ( DEFAULT_SHAPELY_GEOMETRY, NOT_AVAILABLE, + USER_AGENT, + AwsAuth, + DatasetCreationError, DatasetDriver, Download, EOProduct, + HTTPHeaderAuth, + HttpQueryStringAuth, ProgressCallback, mock, ) @@ -549,30 +554,41 @@ def test_eoproduct_download_http_dynamic_options(self): product_zip_file = "{}.zip".format(product_dir_path) self.assertTrue(os.path.isfile(product_zip_file)) - def test_eoproduct_get_auth_headers_uses_auth_object_helper(self): - """EOProduct._get_auth_headers must expose auth headers when available.""" + @mock.patch("eodag.api.product._product.requests.get") + def test_eoproduct_request_asset(self, mock_get): + """EOProduct.request_asset must perform a GET request with storage options headers.""" product = self._dummy_product() - auth = mock.Mock() - auth.get_auth_headers.return_value = {"Authorization": "Basic abc"} - self.assertEqual( - product._get_auth_headers(auth), - {"Authorization": "Basic abc"}, + product.request_asset("https://example.com/zarr/.zmetadata") + + mock_get.assert_called_once_with( + "https://example.com/zarr/.zmetadata", + headers={}, + stream=True, ) @mock.patch("eodag.api.product._product.requests.get") - def test_eoproduct_request_asset_passes_auth_and_headers(self, mock_get): - """EOProduct.request_asset must pass auth headers returned by the auth object.""" + def test_eoproduct_request_asset_with_auth_headers(self, mock_get): + """EOProduct.request_asset must forward authentication headers from get_storage_options.""" product = self._dummy_product() - auth = mock.Mock() - auth.get_auth_headers.return_value = {"Authorization": "Basic abc"} + # Mock downloader and auth + mock_downloader = mock.MagicMock() + mock_auth = mock.MagicMock() + product.register_downloader(mock_downloader, mock_auth) + + # Mock get_storage_options to return auth headers + product.get_storage_options = mock.MagicMock( + return_value={ + "path": "https://example.com/zarr/.zmetadata", + "headers": {"Authorization": "Bearer token123"}, + } + ) - product.request_asset("https://example.com/zarr/.zmetadata", auth) + product.request_asset("https://example.com/zarr/.zmetadata") mock_get.assert_called_once_with( "https://example.com/zarr/.zmetadata", - auth=auth, - headers={"Authorization": "Basic abc"}, + headers={"Authorization": "Bearer token123"}, stream=True, ) @@ -602,24 +618,19 @@ def test_eoproduct_list_zarr_files_from_zmetadata(self, mock_get_mapper): self.assertEqual(files, [".zmetadata", ".zgroup", ".zattrs", "foo/.zarray"]) @mock.patch("fsspec.get_mapper") - def test_eoproduct_list_zarr_files_from_zmetadata_with_basic_auth( - self, mock_get_mapper - ): - """EOProduct.list_zarr_files_from_metadata must forward auth headers to fsspec for `.zmetadata`.""" + def test_eoproduct_list_zarr_files_from_zmetadata_headers(self, mock_get_mapper): + """EOProduct.list_zarr_files_from_metadata must forward storage options headers to fsspec.""" product = self._dummy_product() - auth = mock.Mock() - auth.get_auth_headers.return_value = {"Authorization": "Basic abc"} mock_get_mapper.return_value = {".zmetadata": json.dumps({"metadata": {}})} files = product.list_zarr_files_from_metadata( "https://example.com/zarr", - auth, ) mock_get_mapper.assert_called_once_with( "https://example.com/zarr", client_kwargs={ - "headers": {"Authorization": "Basic abc"}, + "headers": {}, "trust_env": False, }, ) @@ -892,3 +903,186 @@ def test_eoproduct_from_pystac(self): pystac_item = Item.from_dict(product.as_dict()) product_from_pystac = EOProduct.from_pystac(pystac_item) self.assertIsInstance(product_from_pystac, EOProduct) + + def test_get_storage_options_http_headers(self): + """get_storage_options should be adapted to the provider config""" + product = EOProduct( + self.provider, self.eoproduct_props, collection=self.collection + ) + # http headers auth + product.register_downloader( + Download("foo", PluginConfig()), + HTTPHeaderAuth( + "foo", + PluginConfig.from_mapping( + { + "type": "Download", + "credentials": {"apikey": "foo"}, + "headers": {"X-API-Key": "{apikey}"}, + } + ), + ), + ) + self.assertDictEqual( + product.get_storage_options(), + { + "path": self.download_url, + "headers": {"X-API-Key": "foo", **USER_AGENT}, + }, + ) + + def test_get_storage_options_http_no_auth(self): + """get_storage_options should return path when no auth""" + product = EOProduct( + self.provider, self.eoproduct_props, collection=self.collection + ) + # http headers auth + product.register_downloader( + Download("foo", PluginConfig()), + None, + ) + self.assertDictEqual( + product.get_storage_options(), + { + "path": self.download_url, + }, + ) + + def test_get_storage_options_http_qs(self): + """get_storage_options should be adapted to the provider config""" + product = EOProduct( + self.provider, self.eoproduct_props, collection=self.collection + ) + # http qs auth + product.register_downloader( + Download("foo", PluginConfig()), + HttpQueryStringAuth( + "foo", + PluginConfig.from_mapping( + { + "type": "Download", + "credentials": {"apikey": "foo"}, + } + ), + ), + ) + self.assertDictEqual( + product.get_storage_options(), + { + "path": f"{self.download_url}?apikey=foo", + "headers": USER_AGENT, + }, + ) + + @mock.patch("eodag.api.product._product.ServiceResource", new=object) + def test_get_storage_options_s3_credentials_endpoint(self): + """get_storage_options should be adapted to the provider config using s3 credentials and endpoint""" + product = EOProduct( + self.provider, self.eoproduct_props, collection=self.collection + ) + auth_plugin = AwsAuth( + "foo", + PluginConfig.from_mapping( + { + "type": "Authentication", + "s3_endpoint": "http://foo.bar", + "credentials": { + "aws_access_key_id": "foo", + "aws_secret_access_key": "bar", + "aws_session_token": "baz", + }, + "requester_pays": True, + } + ), + ) + auth_plugin.s3_session = mock.MagicMock() + auth_plugin.s3_session.get_credentials.return_value = mock.Mock( + access_key="foo", + secret_key="bar", + token="baz", + ) + auth_plugin.authenticate = mock.MagicMock(return_value=object()) + product.register_downloader(Download("foo", PluginConfig()), auth_plugin) + self.assertDictEqual( + product.get_storage_options(), + { + "path": self.download_url, + "key": "foo", + "secret": "bar", + "token": "baz", + "client_kwargs": {"endpoint_url": "http://foo.bar"}, + "requester_pays": True, + }, + ) + + @mock.patch("eodag.api.product._product.ServiceResource", new=object) + def test_get_storage_options_s3_credentials(self): + """get_storage_options should be adapted to the provider config using s3 credentials""" + product = EOProduct( + self.provider, self.eoproduct_props, collection=self.collection + ) + auth_plugin = AwsAuth( + "foo", + PluginConfig.from_mapping( + { + "type": "Authentication", + "credentials": { + "aws_access_key_id": "foo", + "aws_secret_access_key": "bar", + "aws_session_token": "baz", + }, + } + ), + ) + auth_plugin.s3_session = mock.MagicMock() + auth_plugin.s3_session.get_credentials.return_value = mock.Mock( + access_key="foo", + secret_key="bar", + token="baz", + ) + auth_plugin.authenticate = mock.MagicMock(return_value=object()) + product.register_downloader(Download("foo", PluginConfig()), auth_plugin) + self.assertDictEqual( + product.get_storage_options(), + { + "path": self.download_url, + "key": "foo", + "secret": "bar", + "token": "baz", + }, + ) + + @mock.patch("eodag.api.product._product.ServiceResource", new=object) + def test_get_storage_options_s3_anon(self): + """get_storage_options should be adapted to the provider config using anonymous s3 access""" + product = EOProduct( + self.provider, self.eoproduct_props, collection=self.collection + ) + auth_plugin = AwsAuth( + "foo", + PluginConfig.from_mapping( + {"type": "Authentication", "requester_pays": True} + ), + ) + auth_plugin.s3_session = mock.MagicMock() + auth_plugin.s3_session.get_credentials.return_value = None + auth_plugin.authenticate = mock.MagicMock(return_value=object()) + product.register_downloader(Download("foo", PluginConfig()), auth_plugin) + self.assertDictEqual( + product.get_storage_options(), + { + "path": self.download_url, + "anon": True, + }, + ) + + def test_get_storage_options_error(self): + """get_storage_options should raise when the asset key is missing""" + product = EOProduct( + self.provider, self.eoproduct_props, collection=self.collection + ) + product.downloader = mock.MagicMock() + with self.assertRaises( + DatasetCreationError, msg=f"foo not found in {product} assets" + ): + product.get_storage_options(asset_key="foo") From a772c504ab47ff832fe4c2f089dccfa08ebee32f Mon Sep 17 00:00:00 2001 From: cauriol Date: Fri, 10 Apr 2026 14:11:30 +0200 Subject: [PATCH 08/16] fix: zarr --- eodag/api/product/_product.py | 24 --------------- pyproject.toml | 3 -- tests/units/test_eoproduct.py | 55 ----------------------------------- 3 files changed, 82 deletions(-) diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py index a8ade5cd4d..24fd06a5e6 100644 --- a/eodag/api/product/_product.py +++ b/eodag/api/product/_product.py @@ -19,7 +19,6 @@ import base64 import datetime as dt -import json import logging import os import re @@ -729,29 +728,6 @@ def request_asset( headers = self.get_storage_options().get("headers", {}) return requests.get(url, headers=headers, stream=True) - def list_zarr_files_from_metadata( - self, - base_url: str, - ) -> list[str]: - """List file paths from a Zarr store metadata file.""" - import fsspec # type: ignore[import-untyped] - - headers = self.get_storage_options().get("headers", {}) - mapper = fsspec.get_mapper( - base_url, - client_kwargs={"headers": headers, "trust_env": False}, - ) - - if ".zmetadata" in mapper: - meta = json.loads(mapper[".zmetadata"]) - return [".zmetadata", *meta["metadata"].keys()] - - # TODO: Support Zarr v3 when test data becomes available. - # Zarr v2 uses `.zmetadata`, while Zarr v3 exposes `zarr.json`. - # The implementation should be straightforward once we can validate it - # against real examples. - raise ValueError(f"No Zarr metadata file found at {base_url}") - def _init_progress_bar( self, progress_callback: Optional[ProgressCallback], diff --git a/pyproject.toml b/pyproject.toml index 2a99713749..8072157641 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,9 +59,6 @@ dependencies = [ "typing_extensions >= 4.8.0", "urllib3", "zipstream-ng", - "fsspec", - "aiohttp", - "requests" ] dynamic = ["version"] diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py index 7f6bb8f30a..ee66d98fbb 100644 --- a/tests/units/test_eoproduct.py +++ b/tests/units/test_eoproduct.py @@ -592,61 +592,6 @@ def test_eoproduct_request_asset_with_auth_headers(self, mock_get): stream=True, ) - @mock.patch("fsspec.get_mapper") - def test_eoproduct_list_zarr_files_from_zmetadata(self, mock_get_mapper): - """EOProduct.list_zarr_files_from_metadata must return files listed in `.zmetadata`.""" - product = self._dummy_product() - mapper = { - ".zmetadata": json.dumps( - { - "metadata": { - ".zgroup": {}, - ".zattrs": {}, - "foo/.zarray": {}, - } - } - ) - } - mock_get_mapper.return_value = mapper - - files = product.list_zarr_files_from_metadata("https://example.com/zarr") - - mock_get_mapper.assert_called_once_with( - "https://example.com/zarr", - client_kwargs={"headers": {}, "trust_env": False}, - ) - self.assertEqual(files, [".zmetadata", ".zgroup", ".zattrs", "foo/.zarray"]) - - @mock.patch("fsspec.get_mapper") - def test_eoproduct_list_zarr_files_from_zmetadata_headers(self, mock_get_mapper): - """EOProduct.list_zarr_files_from_metadata must forward storage options headers to fsspec.""" - product = self._dummy_product() - mock_get_mapper.return_value = {".zmetadata": json.dumps({"metadata": {}})} - - files = product.list_zarr_files_from_metadata( - "https://example.com/zarr", - ) - - mock_get_mapper.assert_called_once_with( - "https://example.com/zarr", - client_kwargs={ - "headers": {}, - "trust_env": False, - }, - ) - self.assertEqual(files, [".zmetadata"]) - - @mock.patch("fsspec.get_mapper") - def test_eoproduct_list_zarr_files_from_metadata_raises_when_missing( - self, mock_get_mapper - ): - """EOProduct.list_zarr_files_from_metadata must fail when metadata files are missing.""" - product = self._dummy_product() - mock_get_mapper.return_value = {} - - with self.assertRaises(ValueError): - product.list_zarr_files_from_metadata("https://example.com/zarr") - @responses.activate def test_eoproduct_download_progress_bar(self): """eoproduct.download must show a progress bar""" From 18e4d2576bbc09450190d4348716c3a21a8e043e Mon Sep 17 00:00:00 2001 From: cauriol Date: Fri, 10 Apr 2026 16:37:02 +0200 Subject: [PATCH 09/16] fix: precomit --- eodag/api/product/_product.py | 2 +- tests/units/test_eoproduct.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py index 24fd06a5e6..415673b564 100644 --- a/eodag/api/product/_product.py +++ b/eodag/api/product/_product.py @@ -37,9 +37,9 @@ import geojson import orjson import requests -from pystac import Item from boto3 import Session from boto3.resources.base import ServiceResource +from pystac import Item from requests import PreparedRequest, RequestException from requests.auth import AuthBase from requests.structures import CaseInsensitiveDict diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py index ee66d98fbb..fecc8706f4 100644 --- a/tests/units/test_eoproduct.py +++ b/tests/units/test_eoproduct.py @@ -16,7 +16,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import logging import os import pathlib From a5c677f141b6c21d56895e01f19fbd50e18d32be Mon Sep 17 00:00:00 2001 From: cauriol Date: Mon, 13 Apr 2026 09:42:38 +0200 Subject: [PATCH 10/16] fix: exception class --- eodag/api/product/_product.py | 4 ++-- tests/context.py | 1 - tests/units/test_eoproduct.py | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py index 415673b564..9844113cfc 100644 --- a/eodag/api/product/_product.py +++ b/eodag/api/product/_product.py @@ -88,7 +88,7 @@ _import_stac_item_from_unknown_provider, ) from eodag.utils.exceptions import ( - DatasetCreationError, + AddressNotFound, DownloadError, MisconfiguredError, ValidationError, @@ -684,7 +684,7 @@ def get_storage_options( try: url = self.assets[asset_key]["href"] if asset_key else self.location except KeyError as e: - raise DatasetCreationError(f"{asset_key} not found in {self} assets") from e + raise AddressNotFound(f"{asset_key} not found in {self} assets") from e headers = {**USER_AGENT} if isinstance(auth, ServiceResource) and isinstance( diff --git a/tests/context.py b/tests/context.py index 527d045aee..d780556af6 100644 --- a/tests/context.py +++ b/tests/context.py @@ -138,7 +138,6 @@ UnsupportedProvider, ValidationError, InvalidDataError, - DatasetCreationError, ) from eodag.utils.stac_reader import fetch_stac_items, _TextOpener from tests import TEST_RESOURCES_PATH diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py index fecc8706f4..f30200dea6 100644 --- a/tests/units/test_eoproduct.py +++ b/tests/units/test_eoproduct.py @@ -38,8 +38,8 @@ DEFAULT_SHAPELY_GEOMETRY, NOT_AVAILABLE, USER_AGENT, + AddressNotFound, AwsAuth, - DatasetCreationError, DatasetDriver, Download, EOProduct, @@ -1027,6 +1027,6 @@ def test_get_storage_options_error(self): ) product.downloader = mock.MagicMock() with self.assertRaises( - DatasetCreationError, msg=f"foo not found in {product} assets" + AddressNotFound, msg=f"foo not found in {product} assets" ): product.get_storage_options(asset_key="foo") From 541535579794c12586beb4c470c1ad5524223150 Mon Sep 17 00:00:00 2001 From: cauriol Date: Wed, 22 Apr 2026 09:42:25 +0200 Subject: [PATCH 11/16] fix: compute state OIDCAuthorizationCodeFlowAuth --- eodag/plugins/authentication/openid_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py index 495b630fef..b383a1c5a7 100644 --- a/eodag/plugins/authentication/openid_connect.py +++ b/eodag/plugins/authentication/openid_connect.py @@ -336,7 +336,7 @@ def authenticate(self) -> CodeAuthorizedAuth: def _request_new_token(self) -> dict[str, str]: """Fetch the access token with a new authentication""" logger.debug("Fetching access token from %s", self.token_endpoint) - state = self.compute_state() + state = OIDCAuthorizationCodeFlowAuth.compute_state() authentication_response = self.authenticate_user(state) exchange_url = authentication_response.url From 7707d99730b2f309755db7e6a7e9a783751856d6 Mon Sep 17 00:00:00 2001 From: cauriol Date: Thu, 23 Apr 2026 10:43:54 +0200 Subject: [PATCH 12/16] feat: add desp_cache provider --- eodag/resources/collections.yml | 31 +++++++++++++--- eodag/resources/providers/desp_cache.yml | 45 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 eodag/resources/providers/desp_cache.yml diff --git a/eodag/resources/collections.yml b/eodag/resources/collections.yml index 07279a2ed7..3d84c174d0 100644 --- a/eodag/resources/collections.yml +++ b/eodag/resources/collections.yml @@ -6622,10 +6622,6 @@ DT_CLIMATE_G2_BASELINE_CONT_ICON_R1: the pages [Climate DT Phase2 CLTE Parameters](https://confluence.ecmwf.int/display/DDCZ/DestinE+ClimateDT+Phase+2+clte+Parameters) and [Climate DT Phase2 CLMN Parameters](https://confluence.ecmwf.int/display/DDCZ/DestinE+ClimateDT+Phase+2+clmn+Parameters)". This Generation-2 Collection gives access to 'Control Simulation' data based on the 'ICON' model. - instruments: [] - constellation: Digital Twin - platform: DT - processing:level: keywords: ["DT", "DE", "LUMI", "Destination-Earth", "Digital-Twin", "Climate Change", "baseline", "ICON", "cont", "Phase 2"] eodag:sensor_type: ATMOSPHERIC license: CC-BY-4.0 @@ -7322,6 +7318,33 @@ DT_CLIMATE_G2_STORY_NUDGING_TPLUS2_0K_IFS_FESOM_R5: title: Climate Change Adaptation Digital Twin (Climate Adaptation DT) - Storyline Simulation Future Climate - IFS-FESOM - Generation-2 - Realization 5 extent: {"spatial": {"bbox": [[-180.0, -90.0, 180.0, 90.0]]}, "temporal": {"interval": [["2017-01-01T00:00:00Z", "2025-12-31T23:59:59Z"]]}} +# Digital Twin data cubes +DT_CLIMATE_ADAPTATION_IFS_NEMO: + description: | + A selection of datasets derived from the DestinE Climate Change Adaptation Digital Twin according to the output of the IFS-NEMO model coupled with the Shared Socioeconomic Pathway 3-7.0 (experiment: SSP3-7.0) as defined in the Scenario Model Intercomparison Project (activity: ScenarioMIP) + instruments: [] + constellation: Digital Twin + platform: DT + processing:level: + keywords: ["DT", "Digital-Twin", "Climate Change", "Atmosphere", "Sea Ice", "SCENARIOMIP", "IFS-NEMO","Socioeconomic", "SSP3-7.0"] + eodag:sensor_type: ATMOSPHERIC + license: CC-BY-4.0 + title: DestinE Climate Adaptation DT, activity ScenarioMIP, experiment SSP3-7.0, model IFS-NEMO + extent: {"spatial": {"bbox": [[-180.0, -90.0, 180.0, 90.0]]}, "temporal": {"interval": [["2020-07-01T00:00:00Z", "2040-01-01T00:00:00Z"]]}} + +DT_CLIMATE_ADAPTATION_ICON: + description: | + A selection of datasets derived from the DestinE Climate Change Adaptation Digital Twin according to the output of the ICON model coupled with the Shared Socioeconomic Pathway 3-7.0 (experiment: SSP3-7.0) as defined in the Scenario Model Intercomparison Project (activity: ScenarioMIP) + instruments: [] + constellation: Digital Twin + platform: DT + processing:level: + keywords: ["DT", "Digital-Twin", "Climate Change", "Atmosphere", "Sea Ice", "SCENARIOMIP", "ICON","Socioeconomic", "SSP3-7.0"] + eodag:sensor_type: ATMOSPHERIC + license: CC-BY-4.0 + title: DestinE Climate Adaptation DT, activity ScenarioMIP, experiment SSP3-7.0, model ICON + extent: {"spatial": {"bbox": [[-180.0, -90.0, 180.0, 90.0]]}, "temporal": {"interval": [["2021-07-01T00:00:00Z", "2030-12-31T00:00:00Z"]]}} + # MARK: METOP -------------------------------------------------------------------------- METOP_AMSU_L1: description: | diff --git a/eodag/resources/providers/desp_cache.yml b/eodag/resources/providers/desp_cache.yml new file mode 100644 index 0000000000..cef3f64648 --- /dev/null +++ b/eodag/resources/providers/desp_cache.yml @@ -0,0 +1,45 @@ +desp_cache: + name: desp_cache + group: dedl + priority: 2 + roles: + - host + description: Destination Earth Digital Twin for Climate Change Adaptation through Earth Observation (DESP) - Cache + url: https://data.destination-earth.eu + search: + type: StacSearch + api_endpoint: https://hda.data.destination-earth.eu/stac/v2/search + pagination: + max_items_per_page: 1000 + next_page_token_key: token + sort: + sort_by_tpl: '{{"sortby": [ {{"field": "{sort_param}", "direction": "{sort_order}" }} ] }}' + sort_order_mapping: + ascending: asc + descending: desc + sort_param_mapping: + id: id + start_datetime: properties.datetime + end_datetime: properties.end_datetime + download: + type: HTTPDownload + ssl_verify: true + no_auth_download: True + auth_error_code: 401 + auth: + type: OIDCAuthorizationCodeFlowAuth + oidc_config_url: https://auth.destine.eu/realms/desp/.well-known/openid-configuration + redirect_uri: https://cacheb.dcms.destine.eu/ + client_id: edh-public + user_consent_needed: false + token_exchange_post_data_method: data + login_form_xpath: //form[@id='kc-form-login'] + authentication_uri_source: login-form + token_provision: basic + products: + GENERIC_COLLECTION: + _collection: "{collection}" + DT_CLIMATE_ADAPTATION_IFS_NEMO: + _collection: EO.ECMWF.DAT.DT_CLIMATE_ADAPTATION_IFS-NEMO + DT_CLIMATE_ADAPTATION_ICON: + _collection: EO.ECMWF.DAT.DT_CLIMATE_ADAPTATION_ICON \ No newline at end of file From 80f8a66bb552a784b43ab0195104eb462d4a8cbe Mon Sep 17 00:00:00 2001 From: cauriol Date: Thu, 23 Apr 2026 11:35:53 +0200 Subject: [PATCH 13/16] fix: tests with desp_cache provider --- eodag/resources/providers/desp_cache.yml | 3 +-- eodag/resources/user_conf_template.yml | 9 +++++++++ tests/units/test_core.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/eodag/resources/providers/desp_cache.yml b/eodag/resources/providers/desp_cache.yml index cef3f64648..5af3257820 100644 --- a/eodag/resources/providers/desp_cache.yml +++ b/eodag/resources/providers/desp_cache.yml @@ -1,7 +1,6 @@ desp_cache: name: desp_cache - group: dedl - priority: 2 + priority: 0 roles: - host description: Destination Earth Digital Twin for Climate Change Adaptation through Earth Observation (DESP) - Cache diff --git a/eodag/resources/user_conf_template.yml b/eodag/resources/user_conf_template.yml index c4e67799e1..0e8c6492a8 100644 --- a/eodag/resources/user_conf_template.yml +++ b/eodag/resources/user_conf_template.yml @@ -140,6 +140,15 @@ dlr_eoc_geoservice: credentials: username: password: +desp_cache: + priority: # Lower value means lower priority (Default: 0) + search: + download: + output_dir: + auth: + credentials: + username: + password: earth_search: priority: # Lower value means lower priority (Default: 0) search: # Search parameters configuration diff --git a/tests/units/test_core.py b/tests/units/test_core.py index 40f162fa19..d564474678 100644 --- a/tests/units/test_core.py +++ b/tests/units/test_core.py @@ -189,6 +189,8 @@ class TestCore(TestCoreBase): "DT_CLIMATE_G2_BASELINE_HIST_IFS_FESOM_R1": ["dedt_lumi"], "DT_CLIMATE_G2_PROJECTIONS_SSP3_7_0_ICON_R1": ["dedt_lumi"], "DT_CLIMATE_G2_PROJECTIONS_SSP3_7_0_IFS_FESOM_R1": ["dedt_lumi"], + "DT_CLIMATE_ADAPTATION_IFS_NEMO": ["desp_cache"], + "DT_CLIMATE_ADAPTATION_ICON": ["desp_cache"], "EEA_HRL_TCF": ["wekeo_main"], "EFAS_FORECAST": ["cop_ewds", "dedl"], "EFAS_HISTORICAL": ["cop_ewds", "dedl"], @@ -920,6 +922,7 @@ class TestCore(TestCoreBase): "dedt_lumi", "dedt_mn5", "dlr_eoc_geoservice", + "desp_cache", "earth_search", "earth_search_gcs", "ecmwf", @@ -2293,6 +2296,13 @@ def test_available_sortables(self, mock_auth_session_request): "eo:cloud_cover", ], "max_sort_params": None, + "desp_cache": { + "max_sort_params": None, + "sortables": [ + "id", + "start_datetime", + "end_datetime", + ], }, "earth_search": { "sortables": [ From 4cca34baf53f32d606df9c3459cd94efb2afbd46 Mon Sep 17 00:00:00 2001 From: Julia Date: Wed, 22 Jul 2026 11:34:24 +0200 Subject: [PATCH 14/16] fix: syntax error --- tests/units/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/units/test_core.py b/tests/units/test_core.py index d564474678..affe721465 100644 --- a/tests/units/test_core.py +++ b/tests/units/test_core.py @@ -2296,6 +2296,7 @@ def test_available_sortables(self, mock_auth_session_request): "eo:cloud_cover", ], "max_sort_params": None, + }, "desp_cache": { "max_sort_params": None, "sortables": [ From ec3699d7ad69bcb4e7254052afd4e9fe39a3f5a7 Mon Sep 17 00:00:00 2001 From: Julia Date: Fri, 24 Jul 2026 17:11:58 +0200 Subject: [PATCH 15/16] fix: re-add data lost in rebase --- eodag/resources/collections.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eodag/resources/collections.yml b/eodag/resources/collections.yml index 3d84c174d0..3d2457b952 100644 --- a/eodag/resources/collections.yml +++ b/eodag/resources/collections.yml @@ -6622,6 +6622,10 @@ DT_CLIMATE_G2_BASELINE_CONT_ICON_R1: the pages [Climate DT Phase2 CLTE Parameters](https://confluence.ecmwf.int/display/DDCZ/DestinE+ClimateDT+Phase+2+clte+Parameters) and [Climate DT Phase2 CLMN Parameters](https://confluence.ecmwf.int/display/DDCZ/DestinE+ClimateDT+Phase+2+clmn+Parameters)". This Generation-2 Collection gives access to 'Control Simulation' data based on the 'ICON' model. + instruments: [] + constellation: Digital Twin + platform: DT + processing:level: keywords: ["DT", "DE", "LUMI", "Destination-Earth", "Digital-Twin", "Climate Change", "baseline", "ICON", "cont", "Phase 2"] eodag:sensor_type: ATMOSPHERIC license: CC-BY-4.0 From d4f4729e297e9b523ff0f550508964d836e4e1ff Mon Sep 17 00:00:00 2001 From: Julia Date: Fri, 24 Jul 2026 18:16:29 +0200 Subject: [PATCH 16/16] refactor: remove provider desp cache --- eodag/resources/providers/dedl.yml | 8 +++++ eodag/resources/providers/desp_cache.yml | 44 ------------------------ 2 files changed, 8 insertions(+), 44 deletions(-) delete mode 100644 eodag/resources/providers/desp_cache.yml diff --git a/eodag/resources/providers/dedl.yml b/eodag/resources/providers/dedl.yml index 9c9930488c..7b03a141e6 100644 --- a/eodag/resources/providers/dedl.yml +++ b/eodag/resources/providers/dedl.yml @@ -432,6 +432,14 @@ dedl: metadata_mapping: order:status: '{$.null#replace_str("Not Available","orderable")}' <<: *orderable_mm + DT_CLIMATE_ADAPTATION_IFS_NEMO: + _collection: EO.ECMWF.DAT.DT_CLIMATE_ADAPTATION_IFS-NEMO + metadata_mapping: + assets: $.assets + DT_CLIMATE_ADAPTATION_ICON: + _collection: EO.ECMWF.DAT.DT_CLIMATE_ADAPTATION_ICON + metadata_mapping: + assets: $.assets # AERIS AERIS_IAGOS: _collection: EO.AERIS.DAT.IAGOS diff --git a/eodag/resources/providers/desp_cache.yml b/eodag/resources/providers/desp_cache.yml deleted file mode 100644 index 5af3257820..0000000000 --- a/eodag/resources/providers/desp_cache.yml +++ /dev/null @@ -1,44 +0,0 @@ -desp_cache: - name: desp_cache - priority: 0 - roles: - - host - description: Destination Earth Digital Twin for Climate Change Adaptation through Earth Observation (DESP) - Cache - url: https://data.destination-earth.eu - search: - type: StacSearch - api_endpoint: https://hda.data.destination-earth.eu/stac/v2/search - pagination: - max_items_per_page: 1000 - next_page_token_key: token - sort: - sort_by_tpl: '{{"sortby": [ {{"field": "{sort_param}", "direction": "{sort_order}" }} ] }}' - sort_order_mapping: - ascending: asc - descending: desc - sort_param_mapping: - id: id - start_datetime: properties.datetime - end_datetime: properties.end_datetime - download: - type: HTTPDownload - ssl_verify: true - no_auth_download: True - auth_error_code: 401 - auth: - type: OIDCAuthorizationCodeFlowAuth - oidc_config_url: https://auth.destine.eu/realms/desp/.well-known/openid-configuration - redirect_uri: https://cacheb.dcms.destine.eu/ - client_id: edh-public - user_consent_needed: false - token_exchange_post_data_method: data - login_form_xpath: //form[@id='kc-form-login'] - authentication_uri_source: login-form - token_provision: basic - products: - GENERIC_COLLECTION: - _collection: "{collection}" - DT_CLIMATE_ADAPTATION_IFS_NEMO: - _collection: EO.ECMWF.DAT.DT_CLIMATE_ADAPTATION_IFS-NEMO - DT_CLIMATE_ADAPTATION_ICON: - _collection: EO.ECMWF.DAT.DT_CLIMATE_ADAPTATION_ICON \ No newline at end of file