Merge pull request #8 from CMCC-Foundation/boto3_config_with_retries

Boto3 config with retries

Author: aded

environment.yml

@@ -4,6 +4,7 @@ channels:
   - defaults
 dependencies:
   - python >= 3.9
-  - copernicusmarine >2.1
+  - copernicusmarine >= 2.2.2
+  - zarr >= 3.0
   - boto3 >=1.37.4
   - click >=8.1.8

mds/core/s3_singleton.py

@@ -67,7 +67,12 @@ def __new__(cls, *args, **kwargs):
     def __init__(self, s3_endpoint=S3_ENDPOINT):
         s3_endpoint = s3_endpoint
         self.__s3 = boto3.client(
-            "s3", endpoint_url=s3_endpoint, config=Config(signature_version=UNSIGNED)
+            "s3",
+            endpoint_url=s3_endpoint,
+            config=Config(
+                signature_version=UNSIGNED,
+                retries={"max_attempts": 10, "mode": "adaptive"},
+            ),
         )
         self.__paginator = self.__s3.get_paginator("list_objects_v2")
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "mds-toolbox"
-version = "2.0.1"
+version = "2.0.2"
 description = "A custom CMCC library to list and download data from the Marine Data Store"
 authors = ["Antonio Mariani <[email protected]>"]
 readme = ["README.md"]
@@ -13,8 +13,9 @@ keywords = ["copernicus", "copernicusmarine", "mds", "marine data store"]
 [tool.poetry.dependencies]
 python = ">=3.9,<3.13"
 boto3 = "^1.37.4"
-copernicusmarine = ">=2.1.1"
+copernicusmarine = ">=2.2.2"
 click = "^8.1.8"
+zarr = ">= 3.0"
 
 [build-system]
 requires = ["poetry-core"]