🐛 Nest IceNetSIC downloads under the dataset name to avoid overwriting data when multiple datasets use the same year

Author: jemrobinson

ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2003_2003_24h_v1.yaml

@@ -16,4 +16,4 @@ nbsamp-sicnorth-osisaf-25k-2003-2003-24h-v1:
 
   input:
     netcdf:
-      path: ${base_path}/data/preprocessing/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2003.nc
+      path: ${base_path}/data/preprocessing/${...name}/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2003.nc

ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2010_2010_24h_v1.yaml

@@ -16,4 +16,4 @@ nbsamp-sicnorth-osisaf-25k-2010-2010-24h-v1:
 
   input:
     netcdf:
-      path: ${base_path}/data/preprocessing/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2010.nc
+      path: ${base_path}/data/preprocessing/${...name}/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2010.nc

ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2018_2018_24h_v1.yaml

@@ -16,4 +16,4 @@ nbsamp-sicnorth-osisaf-25k-2018-2018-24h-v1:
 
   input:
     netcdf:
-      path: ${base_path}/data/preprocessing/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2018.nc
+      path: ${base_path}/data/preprocessing/${...name}/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2018.nc

ice_station_zebra/config/datasets/nbsamp_weathersouth_era5_0p5_2020_2020_24h_v1.yaml

@@ -13,5 +13,5 @@ nbsamp-weathersouth-era5-0p5-2020-2020-24h-v1:
   input:
     anemoi-dataset:
       join:
-      - dataset: ${base_path}/data/anemoi/samp-era5-0p5-2020-2020-24h-v1-01-04-south.zarr
+      - dataset: ${base_path}/data/anemoi/samp-weathersouth-era5-0p5-2020-2020-24h-v1.zarr
         select: [ 2t, z_250, z_500 , 10u, 10v, cos_julian_day, sin_julian_day ]

ice_station_zebra/config/datasets/samp_sicsouth_osisaf_25k_2020_2020_24h_v1.yaml

@@ -16,4 +16,4 @@ samp-sicsouth-osisaf-25k-2020-2020-24h-v1:
 
   input:
     netcdf:
-      path: ${base_path}/data/preprocessing/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2020.nc
+      path: ${base_path}/data/preprocessing/${...name}/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2020.nc

ice_station_zebra/data_processors/preprocessors/base.py

@@ -1,13 +1,16 @@
+from collections.abc import ABC, abstractmethod
 from pathlib import Path
 
 from omegaconf import DictConfig
 
 
-class IPreprocessor:
+class IPreprocessor(ABC):
     def __init__(self, config: DictConfig) -> None:
         """Initialise the IPreprocessor base class."""
-        self.name = str(config.get("preprocessor", {}).get("type", "None"))
+        self.cls_name = str(config.get("preprocessor", {}).get("type", "None"))
+        self.dataset_name = str(config.get("name", "None"))
 
+    @abstractmethod
     def download(self, preprocessor_path: Path) -> None:
         """Download data to the specified preprocessor path."""
 

ice_station_zebra/data_processors/preprocessors/icenet_sic.py

@@ -30,7 +30,7 @@ def is_south(self) -> bool:
     def download(self, preprocessor_path: Path) -> None:
         """Download data to the specified preprocessor path."""
         # Change to the output directory before downloading
-        icenet_path = preprocessor_path / self.name
+        icenet_path = preprocessor_path / self.dataset_name / self.cls_name
         icenet_path.mkdir(parents=True, exist_ok=True)
         current_directory = Path.cwd()
         os.chdir(icenet_path)
@@ -41,7 +41,7 @@ def download(self, preprocessor_path: Path) -> None:
         logger.info("Generating polar masks for %s.", mask_year)
         masks.generate(year=mask_year)
 
-        logger.info("Downloading sea ice concentration data.")
+        logger.info("Downloading sea ice concentration data to %s.", icenet_path)
         sic = SICDownloader(
             dates=[
                 pd.to_datetime(date).date() for date in self.date_range