feat: allow to download results from all pages

docs/notebooks/api_user_guide/7_download.ipynb

@@ -668,7 +668,11 @@
     "* `output_dir` (`str`): absolute path to a folder where the products should be saved\n",
     "* `extract` (`bool`): whether to automatically extract or not the downloaded product archive\n",
     "* `dl_url_params` (`dict`): additional parameters to pass over to the download url as an url parameter\n",
-    "* `delete_archive` (`bool`): whether to delete the downloaded archives"
+    "* `delete_archive` (`bool`): whether to delete the downloaded archives\n",
+    "\n",
+    "An additionnal kwarg is accepted with the [download_all()](../../api_reference/core.rst#eodag.api.core.EODataAccessGateway.download_all) method:\n",
+    "\n",
+    "* `exhaust` (`bool`): whether to download products from all pages of the search request used to get the results in arguments."
    ]
   },
   {
@@ -1011,6 +1015,13 @@
     "dag.download_all(products_to_download)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If the [SearchResult](../../api_reference/searchresult.rst#eodag.api.search_result.SearchResult) argument does not come from the [search_all()](../../api_reference/core.rst#eodag.api.core.EODataAccessGateway.search_all) method, the `exhaust` argument can be set to `True` (default to `False`) to download products from all pages of the search request used to get the results in arguments by doing a new search on their provider with their search kwargs."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

eodag/api/core.py

@@ -217,6 +217,9 @@ def __init__(
         self.set_locations_conf(locations_conf_path)
         self.search_errors: Set = set()
 
+        # record search kwargs of input products for download_all() method used with "exhaust" parameter set to True
+        self.search_kwargs_for_exhaust: Optional[dict[str, Any]] = None
+
     def get_version(self) -> str:
         """Get eodag package version"""
         return pkg_resources.get_distribution("eodag").version
@@ -1242,12 +1245,12 @@ def search_iter_page_plugin(
         self,
         search_plugin: Union[Search, Api],
         items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
-        **kwargs: Any,
+        **search_kwargs: Any,
     ) -> Iterator[SearchResult]:
         """Iterate over the pages of a products search using a given search plugin.
 
         :param items_per_page: (optional) The number of results requested per page
-        :param kwargs: Some other criteria that will be used to do the search,
+        :param search_kwargs: Some other criteria that will be used to do the search,
                        using parameters compatibles with the provider
         :param search_plugin: search plugin to be used
         :returns: An iterator that yields page per page a collection of EO products
@@ -1263,7 +1266,7 @@ def search_iter_page_plugin(
         prev_next_page_query_obj = pagination_config.get("next_page_query_obj", None)
         # Page has to be set to a value even if use_next is True, this is required
         # internally by the search plugin (see collect_search_urls)
-        kwargs.update(
+        search_kwargs.update(
             page=1,
             items_per_page=items_per_page,
         )
@@ -1276,79 +1279,87 @@ def search_iter_page_plugin(
             if iteration > 1 and next_page_query_obj:
                 pagination_config["next_page_query_obj"] = next_page_query_obj
             logger.info("Iterate search over multiple pages: page #%s", iteration)
-            try:
-                # remove unwanted kwargs for _do_search
-                kwargs.pop("count", None)
-                kwargs.pop("raise_errors", None)
-                search_result = self._do_search(
-                    search_plugin, count=False, raise_errors=True, **kwargs
-                )
-            except Exception:
-                logger.warning(
-                    "error at retrieval of data from %s, for params: %s",
-                    search_plugin.provider,
-                    str(kwargs),
+            if search_kwargs == self.search_kwargs_for_exhaust:
+                logger.info(
+                    "Search on page #%s skipped since it was already requested",
+                    iteration,
                 )
-                raise
-            finally:
-                # we don't want that next(search_iter_page(...)) modifies the plugin
-                # indefinitely. So we reset after each request, but before the generator
-                # yields, the attr next_page_url (to None) and
-                # config.pagination["next_page_url_tpl"] (to its original value).
-                next_page_url = getattr(search_plugin, "next_page_url", None)
-                next_page_query_obj = getattr(search_plugin, "next_page_query_obj", {})
-                next_page_merge = getattr(search_plugin, "next_page_merge", None)
-
-                if next_page_url:
-                    search_plugin.next_page_url = None
-                    if prev_next_page_url_tpl:
-                        search_plugin.config.pagination[
-                            "next_page_url_tpl"
-                        ] = prev_next_page_url_tpl
-                if next_page_query_obj:
-                    if prev_next_page_query_obj:
-                        search_plugin.config.pagination[
-                            "next_page_query_obj"
-                        ] = prev_next_page_query_obj
-                    # Update next_page_query_obj for next page req
-                    if next_page_merge:
-                        search_plugin.next_page_query_obj = dict(
-                            getattr(search_plugin, "query_params", {}),
-                            **next_page_query_obj,
-                        )
-                    else:
-                        search_plugin.next_page_query_obj = next_page_query_obj
-
-            if len(search_result) > 0:
-                # The first products between two iterations are compared. If they
-                # are actually the same product, it means the iteration failed at
-                # progressing for some reason. This is implemented as a workaround
-                # to some search plugins/providers not handling pagination.
-                product = search_result[0]
-                if (
-                    prev_product
-                    and product.properties["id"] == prev_product.properties["id"]
-                    and product.provider == prev_product.provider
-                ):
+            else:
+                try:
+                    # remove unwanted kwargs for _do_search
+                    search_kwargs.pop("count", None)
+                    search_kwargs.pop("raise_errors", None)
+                    search_result = self._do_search(
+                        search_plugin, count=False, raise_errors=True, **search_kwargs
+                    )
+                except Exception:
                     logger.warning(
-                        "Iterate over pages: stop iterating since the next page "
-                        "appears to have the same products as in the previous one. "
-                        "This provider may not implement pagination.",
+                        "error at retrieval of data from %s, for params: %s",
+                        search_plugin.provider,
+                        str(search_kwargs),
                     )
+                    raise
+                finally:
+                    # we don't want that next(search_iter_page(...)) modifies the plugin
+                    # indefinitely. So we reset after each request, but before the generator
+                    # yields, the attr next_page_url (to None) and
+                    # config.pagination["next_page_url_tpl"] (to its original value).
+                    next_page_url = getattr(search_plugin, "next_page_url", None)
+                    next_page_query_obj = getattr(
+                        search_plugin, "next_page_query_obj", {}
+                    )
+                    next_page_merge = getattr(search_plugin, "next_page_merge", None)
+
+                    if next_page_url:
+                        search_plugin.next_page_url = None
+                        if prev_next_page_url_tpl:
+                            search_plugin.config.pagination[
+                                "next_page_url_tpl"
+                            ] = prev_next_page_url_tpl
+                    if next_page_query_obj:
+                        if prev_next_page_query_obj:
+                            search_plugin.config.pagination[
+                                "next_page_query_obj"
+                            ] = prev_next_page_query_obj
+                        # Update next_page_query_obj for next page req
+                        if next_page_merge:
+                            search_plugin.next_page_query_obj = dict(
+                                getattr(search_plugin, "query_params", {}),
+                                **next_page_query_obj,
+                            )
+                        else:
+                            search_plugin.next_page_query_obj = next_page_query_obj
+
+                if len(search_result) > 0:
+                    # The first products between two iterations are compared. If they
+                    # are actually the same product, it means the iteration failed at
+                    # progressing for some reason. This is implemented as a workaround
+                    # to some search plugins/providers not handling pagination.
+                    product = search_result[0]
+                    if (
+                        prev_product
+                        and product.properties["id"] == prev_product.properties["id"]
+                        and product.provider == prev_product.provider
+                    ):
+                        logger.warning(
+                            "error at retrieval of data from %s, for params: %s",
+                            search_plugin.provider,
+                            str(search_kwargs),
+                        )
+                        last_page_with_products = iteration - 1
+                        break
+                    yield search_result
+                    prev_product = product
+                    # Prevent a last search if the current one returned less than the
+                    # maximum number of items asked for.
+                    if len(search_result) < items_per_page:
+                        last_page_with_products = iteration
+                        break
+                else:
                     last_page_with_products = iteration - 1
                     break
-                yield search_result
-                prev_product = product
-                # Prevent a last search if the current one returned less than the
-                # maximum number of items asked for.
-                if len(search_result) < items_per_page:
-                    last_page_with_products = iteration
-                    break
-            else:
-                last_page_with_products = iteration - 1
-                break
             iteration += 1
-            kwargs["page"] = iteration
+            search_kwargs["page"] = iteration
         logger.debug(
             "Iterate over pages: last products found on page %s",
             last_page_with_products,
@@ -1928,7 +1939,11 @@ def _do_search(
                     search_plugin.provider,
                 )
                 self.search_errors.add((search_plugin.provider, e))
-        return SearchResult(results, total_results)
+        normalized_results = SearchResult(results, total_results)
+        normalized_results.search_kwargs = dict(
+            {"page": prep.page, "items_per_page": prep.items_per_page}, **kwargs
+        )
+        return normalized_results
 
     def crunch(self, results: SearchResult, **kwargs: Any) -> SearchResult:
         """Apply the filters given through the keyword arguments to the results
@@ -2009,14 +2024,16 @@ def download_all(
         """
         paths = []
         if search_result:
-            logger.info("Downloading %s products", len(search_result))
+            if not kwargs.get("exhaust", False):
+                logger.info(f"Downloading {len(search_result)} product(s)")
             # Get download plugin using first product assuming product from several provider
             # aren't mixed into a search result
             download_plugin = self._plugins_manager.get_download_plugin(
                 search_result[0]
             )
             paths = download_plugin.download_all(
                 search_result,
+                self,
                 downloaded_callback=downloaded_callback,
                 progress_callback=progress_callback,
                 wait=wait,

eodag/api/search_result.py

@@ -17,8 +17,9 @@
 # limitations under the License.
 from __future__ import annotations
 
+import logging
 from collections import UserList
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union
 
 from shapely.geometry import GeometryCollection, shape
 
@@ -34,6 +35,8 @@
 
     from eodag.plugins.crunch.base import Crunch
 
+logger = logging.getLogger("eodag.search_result")
+
 
 class SearchResult(UserList):
     """An object representing a collection of :class:`~eodag.api.product._product.EOProduct` resulting from a search.
@@ -42,7 +45,13 @@ class SearchResult(UserList):
     :param number_matched: (optional) the estimated total number of matching results
 
     :cvar data: List of products
+    :vartype data: List[EOProduct]
     :ivar number_matched: Estimated total number of matching results
+    :vartype number_matched: Optional[int]
+    :ivar search_kwargs: (optional) The search kwargs used by eodag to search for the product
+    :vartype search_kwargs: Optional[Dict[str, Any]]
+    :ivar crunchers: The list of crunchers used to filter these products
+    :vartype crunchers: Set[Crunch]
     """
 
     data: List[EOProduct]
@@ -51,7 +60,16 @@ def __init__(
         self, products: List[EOProduct], number_matched: Optional[int] = None
     ) -> None:
         super(SearchResult, self).__init__(products)
-        self.number_matched = number_matched
+        self.number_matched: Optional[int] = number_matched
+        self.search_kwargs: Optional[Dict[str, Any]] = None
+        self.crunchers: Set[Crunch] = set()
+
+    def clear(self) -> None:
+        """Clear search result"""
+        super().clear()
+        self.number_matched = None
+        self.search_kwargs = None
+        self.crunchers.clear()
 
     def crunch(self, cruncher: Crunch, **search_params: Any) -> SearchResult:
         """Do some crunching with the underlying EO products.
@@ -60,8 +78,12 @@ def crunch(self, cruncher: Crunch, **search_params: Any) -> SearchResult:
         :param search_params: The criteria that have been used to produce this result
         :returns: The result of the application of the crunching method to the EO products
         """
-        crunched_results = cruncher.proceed(self.data, **search_params)
-        return SearchResult(crunched_results)
+        crunched_results_list = cruncher.proceed(self.data, **search_params)
+        crunched_results = SearchResult(crunched_results_list)
+        crunched_results.search_kwargs = self.search_kwargs
+        self.crunchers.add(cruncher)
+        crunched_results.crunchers = self.crunchers
+        return crunched_results
 
     def filter_date(
         self, start: Optional[str] = None, end: Optional[str] = None

eodag/plugins/apis/base.py

@@ -35,6 +35,8 @@ class Api(Search, Download):
 
     - download data in the ``output_dir`` folder defined in the plugin's
       configuration or passed through kwargs
+    - download data from all pages of the search of results given in parameters
+      if ``exhaust`` is set to True (False by default) with ``download_all``
     - extract products from their archive (if relevant) if ``extract`` is set to True
       (True by default)
     - save a product in an archive/directory (in ``output_dir``) whose name must be

eodag/plugins/apis/ecmwf.py

@@ -47,6 +47,7 @@
 
     from requests.auth import AuthBase
 
+    from eodag.api.core import EODataAccessGateway
     from eodag.api.product import EOProduct
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
@@ -244,6 +245,7 @@ def download(
     def download_all(
         self,
         products: SearchResult,
+        dag: EODataAccessGateway,
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
@@ -256,6 +258,7 @@ def download_all(
         """
         return super(EcmwfApi, self).download_all(
             products,
+            dag,
             auth=auth,
             downloaded_callback=downloaded_callback,
             progress_callback=progress_callback,

eodag/plugins/apis/usgs.py

@@ -59,6 +59,7 @@
 if TYPE_CHECKING:
     from requests.auth import AuthBase
 
+    from eodag.api.core import EODataAccessGateway
     from eodag.api.search_result import SearchResult
     from eodag.config import PluginConfig
     from eodag.types.download_args import DownloadConf
@@ -441,6 +442,7 @@ def download_request(
     def download_all(
         self,
         products: SearchResult,
+        dag: EODataAccessGateway,
         auth: Optional[Union[AuthBase, Dict[str, str]]] = None,
         downloaded_callback: Optional[DownloadedCallback] = None,
         progress_callback: Optional[ProgressCallback] = None,
@@ -453,6 +455,7 @@ def download_all(
         """
         return super(UsgsApi, self).download_all(
             products,
+            dag,
             auth=auth,
             downloaded_callback=downloaded_callback,
             progress_callback=progress_callback,

eodag/plugins/base.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Dict, List, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from eodag.utils.exceptions import PluginNotFoundError
 
@@ -60,7 +60,7 @@ def get_plugin_by_class_name(cls, name: str) -> EODAGPluginMount:
 class PluginTopic(metaclass=EODAGPluginMount):
     """Base of all plugin topics in eodag"""
 
-    def __init__(self, provider: str, config: PluginConfig) -> None:
+    def __init__(self, provider: Optional[str], config: PluginConfig) -> None:
         self.config = config
         self.provider = provider