Merge pull request #46 from rodekruis/fix/various

Fix/various

Author: p-phung

retrievalpipeline/config/somalia.py

@@ -97,7 +97,7 @@ def extreme_heat_data_request(self) -> EcmwfDataRequest:
                 "year": [
                     f"{year}"
                     for year in range(
-                        self.datetime_config.recent_start_year, self.datetime_config.baseline_end_year + 1
+                        self.datetime_config.recent_start_year, self.datetime_config.recent_end_year + 1
                     )
                 ],
                 "month": [f"{m:02d}" for m in range(1, 13)],

retrievalpipeline/extract/extractors/api_json_extractor.py

@@ -1,3 +1,4 @@
+import time
 from collections.abc import Iterable, Iterator
 from typing import Any
 
@@ -46,13 +47,28 @@ def get_data(
         Returns:
             list of pydantic models (of type specified in the initiation).
         """
-        try:
-            response = self.get_response(api_url, params, headers)
-            json_data = self.get_data_from_response(response)
-        except ApiResponseError as err:
-            logger.warning(err)
+        max_retries = 5
+        sleep_seconds = 10
+
+        for attempt in range(1, max_retries + 1):
+            try:
+                response = self.get_response(api_url, params, headers)
+                json_data = self.get_data_from_response(response)
+            except ApiResponseError as err:
+                if attempt < max_retries:
+                    logger.warning(
+                        f"""Attempt {attempt}/{max_retries} failed for {api_url}: {err}.
+                        Retrying in {sleep_seconds} seconds""",
+                    )
+                    time.sleep(sleep_seconds)
+                    continue
+                else:
+                    logger.warning(f"All {max_retries} attempts failed for {api_url}: {err}")
+
+        if json_data:
+            return list(self.validate_and_parse(json_data))
+        else:
             return []
-        return list(self.validate_and_parse(json_data))
 
     def get_response(
         self, api_url: str, params: dict[str, str] | None = None, headers: dict[str, str] | None = None

retrievalpipeline/extract/extractors/ipc.py

@@ -188,12 +188,11 @@ def get_available_data(self, data_type: str) -> gpd.GeoDataFrame:
     def get_metadata(self, anl_id: str) -> pd.DataFrame:
         """Collect metadata from IPC based on analysis ID."""
         metadata_frames = []
-        for iso2 in self.country_codes_iso2:
-            try:
-                metadata = self.get_available_metadata_iso2(iso2, self.types, [anl_id])
-                metadata_frames.append(metadata)
-            except Exception as e:
-                logger.warning(f"Failed to get metadata for {iso2}: {e}")
+        try:
+            metadata = self.get_available_metadata_iso2(self.country_codes_iso2, self.types, [anl_id])
+            metadata_frames.append(metadata)
+        except Exception as e:
+            logger.warning(f"Failed to get metadata for {self.country_codes_iso2}: {e}")
 
         return pd.concat(metadata_frames, ignore_index=True)