Merge pull request #13 from IFRCGo/feature/add-transform-support-for-json

Add support for json in emdat transformation.

Author: emmanuelmathot

pystac_monty/sources/emdat.py

@@ -3,6 +3,7 @@
 from datetime import datetime
 from typing import List, Optional, Union
 
+import numpy as np
 import pandas as pd
 import pytz
 import requests
@@ -20,6 +21,7 @@
 from pystac_monty.geocoding import MontyGeoCoder
 from pystac_monty.hazard_profiles import MontyHazardProfiles
 from pystac_monty.sources.common import MontyDataSource
+from pystac_monty.utils import rename_columns
 
 STAC_EVENT_ID_PREFIX = "emdat-event-"
 STAC_HAZARD_ID_PREFIX = "emdat-hazard-"
@@ -39,8 +41,13 @@ def __init__(self, source_url: str, data: Union[str, pd.DataFrame]):
             self.df = pd.read_excel(data)
         elif isinstance(data, pd.DataFrame):
             self.df = data
+        elif isinstance(data, dict):
+            # If data is a dict, assume it's Json content
+            data = data["data"]["public_emdat"]["data"]
+            df = pd.DataFrame(data)
+            self.df = rename_columns(df)
         else:
-            raise ValueError("Data must be either Excel content (str) or pandas DataFrame")
+            raise ValueError("Data must be either Excel content (str) or pandas DataFrame or Json")
 
     def get_data(self) -> pd.DataFrame:
         return self.df
@@ -125,7 +132,7 @@ def _create_event_item_from_row(self, row: pd.Series) -> Optional[Item]:
         bbox = None
 
         # 1. Try admin units first if geocoder is available
-        if self.geocoder and not pd.isna(row.get("Admin Units")):
+        if self.geocoder and np.any(pd.notna(row.get("Admin Units"))):
             geom_data = self.geocoder.get_geometry_from_admin_units(row.get("Admin Units"))
             if geom_data:
                 geometry = geom_data["geometry"]

pystac_monty/sources/idu.py

@@ -46,7 +46,7 @@ class IDUTransformer:
     """Transform the source data into the STAC items"""
 
     idu_events_collection_id = "idu-events"
-    idu_events_collection_url = "https://raw.githubusercontent.com/IFRCGo/monty-stac-extension/refs/heads/feature/update-idu-documentation/examples/idu-events/idu-events.json"  # noqa
+    idu_events_collection_url = "https://raw.githubusercontent.com/IFRCGo/monty-stac-extension/refs/heads/main/examples/idmc-idu-events/idmc-idu-events.json"  # noqa
     idu_impacts_collection_id = "idu-impacts"
     idu_impacts_collection_url = "https://raw.githubusercontent.com/IFRCGo/monty-stac-extension/refs/heads/feature/update-idu-documentation/examples/idu-impacts/idu-impacts.json"  # noqa
 

pystac_monty/utils.py

@@ -0,0 +1,31 @@
+def rename_columns(df):
+    """Rename columns of a pandas DataFrame"""
+    return df.rename(
+        columns={
+            "disno": "DisNo.",
+            "admin_units": "Admin Units",
+            "latitude": "Latitude",
+            "longitude": "Longitude",
+            "country": "Country",
+            "classif_key": "Classification Key",
+            "iso": "ISO",
+            "total_deaths": "Total Deaths",
+            "no_injured": "No Injured",
+            "no_affected": "No Affected",
+            "no_homeless": "No Homeless",
+            "total_affected": "Total Affected",
+            "total_dam": "Total Damages ('000 US$)",
+            "start_year": "Start Year",
+            "start_month": "Start Month",
+            "start_day": "Start Day",
+            "end_year": "End Year",
+            "end_month": "End Month",
+            "end_day": "End Day",
+            "magnitude": "Magnitude",
+            "magnitude_scale": "Magnitude Sacle",
+            "name": "Event Name",
+            "type": "Disaster Type",
+            "subtype": "Disaster Subtype",
+            "location": "Location",
+        }
+    )