feat: allow to download results from all pages

Author: anesson-cs

eodag/api/core.py

@@ -170,6 +170,9 @@ def __init__(self, user_conf_file_path=None, locations_conf_path=None):
                     )
         self.set_locations_conf(locations_conf_path)
 
+        # record searchs done
+        self.search_kwargs_list = []
+
     def get_version(self):
         """Get eodag package version"""
         return pkg_resources.get_distribution("eodag").version
@@ -976,69 +979,77 @@ def search_iter_page(
             if iteration > 1 and next_page_query_obj:
                 pagination_config["next_page_query_obj"] = next_page_query_obj
             logger.info("Iterate search over multiple pages: page #%s", iteration)
-            try:
-                products, _ = self._do_search(
-                    search_plugin, count=False, raise_errors=True, **search_kwargs
+            if search_kwargs in self.search_kwargs_list:
+                logger.info(
+                    "Search on page #%s skipped since it was already requested",
+                    iteration,
                 )
-            except Exception:
-                products = SearchResult([])
-            finally:
-                # we don't want that next(search_iter_page(...)) modifies the plugin
-                # indefinitely. So we reset after each request, but before the generator
-                # yields, the attr next_page_url (to None) and
-                # config.pagination["next_page_url_tpl"] (to its original value).
-                next_page_url = getattr(search_plugin, "next_page_url", None)
-                next_page_query_obj = getattr(search_plugin, "next_page_query_obj", {})
-                next_page_merge = getattr(search_plugin, "next_page_merge", None)
-
-                if next_page_url:
-                    search_plugin.next_page_url = None
-                    if prev_next_page_url_tpl:
-                        search_plugin.config.pagination[
-                            "next_page_url_tpl"
-                        ] = prev_next_page_url_tpl
-                if next_page_query_obj:
-                    if prev_next_page_query_obj:
-                        search_plugin.config.pagination[
-                            "next_page_query_obj"
-                        ] = prev_next_page_query_obj
-                    # Update next_page_query_obj for next page req
-                    if next_page_merge:
-                        search_plugin.next_page_query_obj = dict(
-                            getattr(search_plugin, "query_params", {}),
-                            **next_page_query_obj,
-                        )
-                    else:
-                        search_plugin.next_page_query_obj = next_page_query_obj
-
-            if len(products) > 0:
-                # The first products between two iterations are compared. If they
-                # are actually the same product, it means the iteration failed at
-                # progressing for some reason. This is implemented as a workaround
-                # to some search plugins/providers not handling pagination.
-                product = products[0]
-                if (
-                    prev_product
-                    and product.properties["id"] == prev_product.properties["id"]
-                    and product.provider == prev_product.provider
-                ):
-                    logger.warning(
-                        "Iterate over pages: stop iterating since the next page "
-                        "appears to have the same products as in the previous one. "
-                        "This provider may not implement pagination.",
+            else:
+                try:
+                    products, _ = self._do_search(
+                        search_plugin, count=False, raise_errors=True, **search_kwargs
+                    )
+                except Exception:
+                    products = SearchResult([])
+                finally:
+                    # we don't want that next(search_iter_page(...)) modifies the plugin
+                    # indefinitely. So we reset after each request, but before the generator
+                    # yields, the attr next_page_url (to None) and
+                    # config.pagination["next_page_url_tpl"] (to its original value).
+                    next_page_url = getattr(search_plugin, "next_page_url", None)
+                    next_page_query_obj = getattr(
+                        search_plugin, "next_page_query_obj", {}
                     )
+                    next_page_merge = getattr(search_plugin, "next_page_merge", None)
+
+                    if next_page_url:
+                        search_plugin.next_page_url = None
+                        if prev_next_page_url_tpl:
+                            search_plugin.config.pagination[
+                                "next_page_url_tpl"
+                            ] = prev_next_page_url_tpl
+                    if next_page_query_obj:
+                        if prev_next_page_query_obj:
+                            search_plugin.config.pagination[
+                                "next_page_query_obj"
+                            ] = prev_next_page_query_obj
+                        # Update next_page_query_obj for next page req
+                        if next_page_merge:
+                            search_plugin.next_page_query_obj = dict(
+                                getattr(search_plugin, "query_params", {}),
+                                **next_page_query_obj,
+                            )
+                        else:
+                            search_plugin.next_page_query_obj = next_page_query_obj
+
+                if len(products) > 0:
+                    # The first products between two iterations are compared. If they
+                    # are actually the same product, it means the iteration failed at
+                    # progressing for some reason. This is implemented as a workaround
+                    # to some search plugins/providers not handling pagination.
+                    product = products[0]
+                    if (
+                        prev_product
+                        and product.properties["id"] == prev_product.properties["id"]
+                        and product.provider == prev_product.provider
+                    ):
+                        logger.warning(
+                            "Iterate over pages: stop iterating since the next page "
+                            "appears to have the same products as in the previous one. "
+                            "This provider may not implement pagination.",
+                        )
+                        last_page_with_products = iteration - 1
+                        break
+                    yield products
+                    prev_product = product
+                    # Prevent a last search if the current one returned less than the
+                    # maximum number of items asked for.
+                    if len(products) < items_per_page:
+                        last_page_with_products = iteration
+                        break
+                else:
                     last_page_with_products = iteration - 1
                     break
-                yield products
-                prev_product = product
-                # Prevent a last search if the current one returned less than the
-                # maximum number of items asked for.
-                if len(products) < items_per_page:
-                    last_page_with_products = iteration
-                    break
-            else:
-                last_page_with_products = iteration - 1
-                break
             iteration += 1
             search_kwargs["page"] = iteration
         logger.debug(
@@ -1360,7 +1371,10 @@ def _prepare_search(
         logger.debug(
             "Using plugin class for search: %s", search_plugin.__class__.__name__
         )
-        auth_plugin = self._plugins_manager.get_auth_plugin(search_plugin.provider)
+        # get auth plugin by preventing possible multiple "auth" key in the return value
+        auth_plugin = kwargs.pop(
+            "auth", self._plugins_manager.get_auth_plugin(search_plugin.provider)
+        )
 
         # append auth to search plugin if needed
         if getattr(search_plugin.config, "need_auth", False) and callable(
@@ -1529,7 +1543,10 @@ def _do_search(self, search_plugin, count=True, raise_errors=False, **kwargs):
                     "Error while searching on provider %s (ignored):",
                     search_plugin.provider,
                 )
-        return SearchResult(results), total_results
+        results.search_kwargs = kwargs
+        if kwargs not in self.search_kwargs_list:
+            self.search_kwargs_list.append(kwargs)
+        return results, total_results
 
     def crunch(self, results, **kwargs):
         """Apply the filters given through the keyword arguments to the results
@@ -1604,24 +1621,26 @@ def download_all(
         :param timeout: (optional) If download fails, maximum time in minutes
                         before stop retrying to download
         :type timeout: int
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
-                        and `dl_url_params` (dict) can be provided as additional kwargs
-                        and will override any other values defined in a configuration
-                        file or with environment variables.
+        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool),
+                        `dl_url_params` (dict) and `exhaust` (bool) can be provided as
+                        additional kwargs and will override any other values defined in a
+                        configuration file or with environment variables.
         :type kwargs: Union[str, bool, dict]
         :returns: A collection of the absolute paths to the downloaded products
         :rtype: list
         """
         paths = []
         if search_result:
-            logger.info("Downloading %s products", len(search_result))
+            if not kwargs.get("exhaust", False):
+                logger.info("Downloading %s products", len(search_result))
             # Get download plugin using first product assuming product from several provider
             # aren't mixed into a search result
             download_plugin = self._plugins_manager.get_download_plugin(
                 search_result[0]
             )
             paths = download_plugin.download_all(
                 search_result,
+                self,
                 downloaded_callback=downloaded_callback,
                 progress_callback=progress_callback,
                 wait=wait,

eodag/api/search_result.py

@@ -15,6 +15,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 from collections import UserList
 
 from shapely.geometry import GeometryCollection, shape
@@ -26,16 +27,24 @@
 from eodag.plugins.crunch.filter_overlap import FilterOverlap
 from eodag.plugins.crunch.filter_property import FilterProperty
 
+logger = logging.getLogger("eodag.search_result")
+
 
 class SearchResult(UserList):
     """An object representing a collection of :class:`~eodag.api.product._product.EOProduct` resulting from a search.
 
     :param products: A list of products resulting from a search
     :type products: list(:class:`~eodag.api.product._product.EOProduct`)
+    :ivar search_kwargs: The search kwargs used by eodag to search for the product
+    :vartype search_kwargs: Any
+    :ivar crunchers: The list of crunchers used to filter these products
+    :vartype crunchers: list(tuple(subclass of :class:`~eodag.plugins.crunch.base.Crunch`, dict))
     """
 
     def __init__(self, products):
         super(SearchResult, self).__init__(products)
+        self.search_kwargs = None
+        self.crunchers = []
 
     def crunch(self, cruncher, **search_params):
         """Do some crunching with the underlying EO products.
@@ -47,8 +56,24 @@ def crunch(self, cruncher, **search_params):
         :returns: The result of the application of the crunching method to the EO products
         :rtype: :class:`~eodag.api.search_result.SearchResult`
         """
-        crunched_results = cruncher.proceed(self, **search_params)
-        return SearchResult(crunched_results)
+        for results_cruncher in self.crunchers:
+            if (
+                cruncher.__class__.__name__ == results_cruncher.__class__.__name__
+                and cruncher.config == results_cruncher.config
+            ):
+                logger.info(
+                    (
+                        "The cruncher '{}' has already been used for these search results with "
+                        "the following parameter(s): {}. Please change parameters or use an other cruncher"
+                    ).format(cruncher.__class__.__name__, cruncher.config)
+                )
+                return self
+        crunched_results_list = cruncher.proceed(self, **search_params)
+        crunched_results = SearchResult(crunched_results_list)
+        crunched_results.search_kwargs = self.search_kwargs
+        self.crunchers.append(cruncher)
+        crunched_results.crunchers = self.crunchers
+        return crunched_results
 
     def filter_date(self, start=None, end=None):
         """

eodag/plugins/apis/base.py

@@ -108,6 +108,7 @@ def download(
     def download_all(
         self,
         products,
+        dag,
         auth=None,
         downloaded_callback=None,
         progress_callback=None,
@@ -120,6 +121,8 @@ def download_all(
 
         :param products: Products to download
         :type products: :class:`~eodag.api.search_result.SearchResult`
+        :param dag: The gateway to download products
+        :type dag: :class:`~eodag.api.core.EODataAccessGateway`
         :param auth: (optional) The configuration of a plugin of type Authentication
         :type auth: :class:`~eodag.config.PluginConfig`
         :param downloaded_callback: (optional) A method or a callable object which takes
@@ -136,10 +139,10 @@ def download_all(
         :param timeout: (optional) If download fails, maximum time in minutes before stop retrying
                         to download
         :type timeout: int
-        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool)
-                        and `dl_url_params` (dict) can be provided as additional kwargs
-                        and will override any other values defined in a configuration
-                        file or with environment variables.
+        :param kwargs: `outputs_prefix` (str), `extract` (bool), `delete_archive` (bool),
+                        `dl_url_params` (dict) and `exhaust` (bool) can be provided as
+                        additional kwargs and will override any other values defined in a
+                        configuration file or with environment variables.
         :type kwargs: Union[str, bool, dict]
         :returns: List of absolute paths to the downloaded products in the local
             filesystem (e.g. ``['/tmp/product.zip']`` on Linux or

eodag/plugins/apis/cds.py

@@ -230,6 +230,7 @@ def download(self, product, auth=None, progress_callback=None, **kwargs):
     def download_all(
         self,
         products,
+        dag,
         auth=None,
         downloaded_callback=None,
         progress_callback=None,
@@ -242,6 +243,7 @@ def download_all(
         """
         return super(CdsApi, self).download_all(
             products,
+            dag,
             auth=auth,
             downloaded_callback=downloaded_callback,
             progress_callback=progress_callback,

eodag/plugins/apis/ecmwf.py

@@ -218,6 +218,7 @@ def download(self, product, auth=None, progress_callback=None, **kwargs):
     def download_all(
         self,
         products,
+        dag,
         auth=None,
         downloaded_callback=None,
         progress_callback=None,
@@ -230,6 +231,7 @@ def download_all(
         """
         return super(EcmwfApi, self).download_all(
             products,
+            dag,
             auth=auth,
             downloaded_callback=downloaded_callback,
             progress_callback=progress_callback,

eodag/plugins/apis/usgs.py

@@ -361,6 +361,7 @@ def download_request(product, fs_path, progress_callback, **kwargs):
     def download_all(
         self,
         products,
+        dag,
         auth=None,
         downloaded_callback=None,
         progress_callback=None,
@@ -373,6 +374,7 @@ def download_all(
         """
         return super(UsgsApi, self).download_all(
             products,
+            dag,
             auth=auth,
             downloaded_callback=downloaded_callback,
             progress_callback=progress_callback,

eodag/plugins/download/aws.py

@@ -934,6 +934,7 @@ def get_chunk_dest_path(self, product, chunk, dir_prefix=None, build_safe=False)
     def download_all(
         self,
         products,
+        dag,
         auth=None,
         downloaded_callback=None,
         progress_callback=None,
@@ -944,6 +945,7 @@ def download_all(
         """
         return super(AwsDownload, self).download_all(
             products,
+            dag,
             auth=auth,
             downloaded_callback=downloaded_callback,
             progress_callback=progress_callback,