feat(conf): split providers.yml file (#2228)

Author: timothey-cs

docs/_static/collections_information.csv

@@ -93,7 +93,6 @@ EUSTAT_SOIL_SEALING_INDEX,Eurostat,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,,,,,
 EUSTAT_SURFACE_TERRESTRIAL_PROTECTED_AREAS,Eurostat,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,,,,,
 FIRE_HISTORICAL,CEMS,,,,,,,available,,,,,available,,,,,,,,,,,,,,,,,,,,available,
 FIRE_SEASONAL,CEMS,,,,,,,available,,,,,,,,,,,,,,,,,,,,,,,,,,
-GENERIC_PRODUCT_TYPE,,,,,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,,
 GHS_BUILT_C,Sentinel-2,,,,,,,,available,,,,,,,,,,,,,,,,,,,,,,,,,
 GHS_BUILT_H,Sentinel-2,,,,,,,,available,,,,,,,,,,,,,,,,,,,,,,,,,
 GHS_BUILT_LAUSTAT,,,,,,,,,available,,,,,,,,,,,,,,,,,,,,,,,,,
@@ -253,7 +252,7 @@ S1_SAR_SLC,SENTINEL1,L1,,,,available,available,,,,available,available,available,
 S2_MSI_L1C,SENTINEL2,L1,available,,,available,available,,,,available,available,available,,,,available,available,,,,,available,available,,,,available,,available,,,,available
 S2_MSI_L2A,SENTINEL2,L2,available,,,available,available,,,,available,available,available,,,,,,,,,,,,,,available,available,,,,,,available
 S2_MSI_L2A_COG,SENTINEL2,L2,,,,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,
-S2_MSI_L2A_MAJA,SENTINEL2,L2,,,,,,,,,,,,,,,,,,,,,available,available,,,,,,,,,,
+S2_MSI_L2A_MAJA,SENTINEL2,L2,,,,,,,,,,,,,,available,,,,,,,available,available,,,,,,,,,,
 S2_MSI_L2B_MAJA_SNOW,SENTINEL2,L2,,,,,,,,,,,,,,,,,,,,,available,available,,,,,,,,,,
 S2_MSI_L2B_MAJA_WATER,SENTINEL2,L2,,,,,,,,,,,,,,,,,,,,,available,available,,,,,,,,,,
 S3_EFR,SENTINEL3,L1,,,,available,available,,,,available,available,available,,,,,,,,available,,,,,,,available,,,,,,available
@@ -345,5 +344,6 @@ SPACENET_BUILDINGS_DETECTION_V2,SpaceNet,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,,,
 SPACENET_OFF_NADIR_BUILDING,SpaceNet,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,,,,,
 SPACENET_ROADS_NETWORK_DETECTION,SpaceNet,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,,,,,
 SPACENET_ROADS_NETWORK_ROUTE_TRAVEL,SpaceNet,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,,,,,
+SUPERSITES,,,,,,,,,,,,,,,,available,,,,,,,,,,,,,,,,,,
 TIGGE_CF_SFC,TIGGE,,,,,,,,,,,,,,,,,,available,,,,,,,,,,,,,,,
 UERRA_EUROPE_SL,SURFEX,,,,available,,,,,,,,available,,,,,,,,,,,,,,,,,,,,available,

docs/add_collection.rst

@@ -42,8 +42,8 @@ as ``instruments``, ``processing:level``, ``constellation``, etc.
 Add collection to a provider
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In the file ``eodag/resources/providers.yml``, add the collection to the ``products``
-entry of a provider:
+In the directory ``eodag/resources/providers/``, you will find a YAML file for each provider supported by EODAG.
+To make EODAG aware of the new collection, you need to add the collection to the ``products`` entry of a provider:
 
 .. code-block:: yaml
 

docs/add_provider.rst

@@ -72,15 +72,18 @@ See more details about how to create a new plugin in :ref:`this dedicated sectio
 Providers pre-configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-All the providers are pre-configured in ``eodag`` in a YAML file.
+All the providers are pre-configured in ``eodag`` in YAML files located in the ``eodag/resources/providers/`` directory.
+Each file corresponds to a provider and contains the configuration of the plugins used to access this provider.
+
+For example, the file ``aws_eos.yml`` contains the configuration of the provider and associated plugins.
 Click on the link below to display its full content.
 
 .. raw:: html
 
    <details>
-   <summary><a>providers.yml</a></summary>
+   <summary><a>aws_eos.yml</a></summary>
 
-.. include:: ../eodag/resources/providers.yml
+.. include:: ../eodag/resources/providers/aws_eos.yml
    :start-line: 17
    :code: yaml
 

docs/getting_started_guide/configure.rst

@@ -148,8 +148,10 @@ Some EODAG core settings can be overriden using environment variables:
 * ``EODAG_LOCS_CFG_FILE`` for defining the desired path to the
   `locations <../notebooks/api_user_guide/3_search.ipynb#Locations-search>`_
   configuration file in place of `~/.config/eodag/locations.yml`.
-* ``EODAG_PROVIDERS_CFG_FILE`` for defining the desired path to the providers configuration file in place of
+* ``EODAG_PROVIDERS_CFG_FILE`` **(deprecated)** for defining the desired path to the providers configuration file in place of
   `<python-site-packages>/eodag/resources/providers.yml`.
+* ``EODAG_PROVIDERS_CFG_DIR`` for defining the desired path to the directory containing providers configuration files in place of
+  `<python-site-packages>/eodag/resources/providers/`.
 * ``EODAG_COLLECTIONS_CFG_FILE`` for defining the desired path to the collections configuration file in place of
   `<python-site-packages>/eodag/resources/collections.yml`.
 * ``EODAG_EXT_COLLECTIONS_CFG_FILE`` for defining the desired path to the `external collections configuration file\

docs/params_mapping.rst

@@ -13,7 +13,7 @@ EODAG maps each provider's specific metadata parameters to a common model using
 `OGC OpenSearch Extension for Earth Observation <http://docs.opengeospatial.org/is/13-026r9/13-026r9.html>`_.
 Extra parameters having no equivalent in this model are mapped as is.
 
-Depending on the provider, some parameters are queryable or not. This is configured in `providers.yml`:
+Depending on the provider, some parameters are queryable or not. This is configured in each configuration file located in the ``eodag/resources/providers/`` directory.
 
 * If a parameter metadata-mapping is a list, the first element will help constructing the query \
   (using `format() <https://docs.python.org/fr/3/library/string.html#string.Formatter.format>`_), and the 2nd will \

eodag/config.py

@@ -19,6 +19,8 @@
 
 import logging
 import os
+import pathlib
+import warnings
 from importlib.resources import files as res_files
 from typing import TYPE_CHECKING, Annotated, Any, Literal, Optional, Union
 
@@ -682,16 +684,35 @@ def credentials_in_auth(auth_conf: PluginConfig) -> bool:
 def load_default_config() -> dict[str, ProviderConfig]:
     """Load the providers configuration into a dictionary.
 
-    Load from eodag `resources/providers.yml` or `EODAG_PROVIDERS_CFG_FILE` environment
+    Load from eodag `resources/providers` files or `EODAG_PROVIDERS_CFG_DIR` environment
     variable if exists.
 
+    For backwards compatibility, if `EODAG_PROVIDERS_CFG_FILE` environment variable is set, load from it instead of
+    `EODAG_PROVIDERS_CFG_DIR`.
+
     :returns: The default provider's configuration
     """
-    eodag_providers_cfg_file = os.getenv("EODAG_PROVIDERS_CFG_FILE") or str(
-        res_files("eodag") / "resources" / "providers.yml"
-    )
+    eodag_providers_cfg_file = os.getenv("EODAG_PROVIDERS_CFG_FILE")
+    eodag_providers_cfg_dir = os.getenv("EODAG_PROVIDERS_CFG_DIR")
+
+    if eodag_providers_cfg_file:
+        warnings.warn(
+            "Usage of deprecated environment variable EODAG_PROVIDERS_CFG_FILE. "
+            "(Please use EODAG_PROVIDERS_CFG_DIR instead)"
+            " -- Deprecated since v4.5.0",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+        return load_config(eodag_providers_cfg_file)
 
-    return load_config(eodag_providers_cfg_file)
+    config_dir = eodag_providers_cfg_dir or str(
+        res_files("eodag") / "resources" / "providers"
+    )
+    providers: dict[str, ProviderConfig] = {}
+    for f in pathlib.Path(config_dir).glob("*.yml"):
+        if f.is_file():
+            providers.update(load_config(str(f)))
+    return dict(sorted(providers.items()))
 
 
 def load_config(config_path: str) -> dict[str, ProviderConfig]:

eodag/plugins/manager.py

@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 import logging
+import pathlib
 import re
 from operator import attrgetter
 from typing import TYPE_CHECKING, Any, Iterator, Optional, Union, cast
@@ -108,18 +109,58 @@ def __init__(self, providers: ProvidersDict) -> None:
                         "Check that the plugin module (%s) is importable",
                         entry_point.name,
                     )
-                if entry_point.dist and entry_point.dist.name != "eodag":
-                    # use plugin providers if any
-                    name = entry_point.dist.name
-                    dist = entry_point.dist
-                    plugin_providers_config_path = [
-                        str(x) for x in dist.locate_file(name).rglob("providers.yml")
-                    ]
-                    if plugin_providers_config_path:
-                        plugin_configs = load_config(plugin_providers_config_path[0])
-                        self.providers.update_from_configs(plugin_configs)
+                plugin_config_paths = self._get_external_provider_config_paths(
+                    entry_point
+                )
+                if plugin_config_paths:
+                    plugin_configs: dict[str, ProviderConfig] = {}
+                    for path in plugin_config_paths:
+                        plugin_configs.update(load_config(path.as_posix()))
+                    self.providers.update_from_configs(plugin_configs)
         self.rebuild()
 
+    def _get_external_provider_config_paths(
+        self,
+        entry_point: importlib_metadata.EntryPoint,
+    ) -> list[pathlib.Path]:
+        """Return provider config paths exposed by an external plugin distribution."""
+        config_paths: list[pathlib.Path] = []
+        dist = entry_point.dist
+        dist_name = getattr(dist, "name", None)
+
+        if not dist or not isinstance(dist_name, str) or dist_name == "eodag":
+            return config_paths
+
+        providers_dir = pathlib.Path(
+            str(dist.locate_file(pathlib.Path("eodag/providers")))
+        )
+        if providers_dir.exists() and providers_dir.is_dir():
+            config_paths.extend(
+                path for path in providers_dir.iterdir() if path.is_file()
+            )
+
+        module_name = getattr(entry_point, "module", None)
+        if isinstance(module_name, str) and module_name:
+            module_path = pathlib.Path(module_name.replace(".", "/"))
+
+            # Check providers/ subdirectory within the module
+            module_providers_dir = pathlib.Path(
+                str(dist.locate_file(module_path / "providers"))
+            )
+            if module_providers_dir.exists() and module_providers_dir.is_dir():
+                config_paths.extend(
+                    path for path in module_providers_dir.iterdir() if path.is_file()
+                )
+
+            # Legacy: single providers.yml file at the module root
+            providers_yml = pathlib.Path(
+                str(dist.locate_file(module_path / "providers.yml"))
+            )
+            if providers_yml.exists() and providers_yml.is_file():
+                config_paths.append(providers_yml)
+
+        return config_paths
+
     def rebuild(self, providers: Optional[ProvidersDict] = None) -> None:
         """(Re)Build plugin manager mapping and cache"""
         if providers is not None: