Adding configs for downloading a sample dataset (#69)

* 🔧 First pass at adding sample config

* 🔧 Add SIC sample download and edit default data

* 🚨 Linting

* 🔧 Updating split dates for test / train / validate

* 🔧 Updating names to follow anemoi convention

* Adapting nbsamp to process predownloaded data

* ✨ Implementing a test filter, and adding forcings

* 🚨 Fixing linter errors

* 🚨 Fixing linter errors

* ✅ Update code to past tests

* ✅ Update code to past tests

* 🐛 Update name of example_filter in all files

* 🔧 Testing adding filters to local dataset

* 🔧 updating files to default download 3 months of data

* 🔧 Updating the dates for the era5 config

* 🚨 Fixing linter warnings

* 🔧 Adding in group_as as it's required for training

* 🔧 Updating the test period so it has enough dates to run

* 🚚 Renamed config files to match Anemoi pattern

* 🔧 Update dataset names inside config files

* 🔧 Update file settings to point to new dataset names

* 🐛 Nest IceNetSIC downloads under the dataset name to avoid overwriting data when multiple datasets use the same year

* ♻️ Separate preprocessors

* 🎨 Move filter registration to CLI and rename ExampleFilter to DoublingFilter

---------

Co-authored-by: James Robinson <james.em.robinson@gmail.com>

Author: IFenton

ice_station_zebra/config/base.yaml

@@ -1,9 +1,7 @@
 defaults:
   - datasets:
-    - era5-0d5-south-2019-12-24h-v1
-    - era5-0d5-south-2020-01-24h-v1
-    - osisaf-sic-south-2019-12-24h-v1
-    - osisaf-sic-south-2020-01-24h-v1
+    - samp_sicsouth_osisaf_25k_2020_2020_24h_v1
+    - samp_weathersouth_era5_0p5_2020_2020_24h_v1
   - evaluate: default
   - loggers:
     - wandb

ice_station_zebra/config/datasets/era5-0d5-south-2019-12-24h-v1.yaml

@@ -0,0 +1,26 @@
+We use the Anemoi naming convention from https://anemoi.readthedocs.io/projects/registry/en/stable/naming-conventions.html.
+
+purpose-content-source-resolution-start-year-end-year-frequency-version
+
+purpose:
+  - nbsamp: Test datasets for use in notebooks
+  - samp: Test datasets for use on the command line
+content:
+  - sicnorth: sea ice concentration (northern hemisphere)
+  - sicsouth: sea ice concentration (southern hemisphere)
+  - weathernorth: weather data (northern hemisphere)
+  - weathersouth: weather data (southern hemisphere)
+source:
+  - era5: ERA5 weather data
+  - osisaf: OSISAF sea ice data
+resolution:
+  - 0p5: 0.5 degree
+  - 25k: 25km grid
+- start-year:
+  - YYYY: year code
+- end-year:
+  - YYYY: year code
+- frequency:
+  - 24h: every 24 hours
+- version:
+  - vX: where X increments each time changes are made

ice_station_zebra/config/datasets/naming_convention.txt

@@ -1,5 +1,5 @@
-osisaf-sic-north-2003-08-24h-v1:
-  name: osisaf-sic-north-2003-08-24h-v1
+nbsamp-sicnorth-osisaf-25k-2003-2003-24h-v1:
+  name: nbsamp-sicnorth-osisaf-25k-2003-2003-24h-v1
   group_as: osisaf-north
   description: OSISAF sea ice concentration for northern hemisphere - Summer 2003 (August-September)
   attribution: EUMETSAT
@@ -16,4 +16,4 @@ osisaf-sic-north-2003-08-24h-v1:
 
   input:
     netcdf:
-      path: ${base_path}/data/preprocessing/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2003.nc
+      path: ${base_path}/data/preprocessing/${...name}/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2003.nc

…ets/osisaf-sic-north-2003-08-24h-v1.yaml …icnorth_osisaf_25k_2003_2003_24h_v1.yamlice_station_zebra/config/datasets/osisaf-sic-north-2003-08-24h-v1.yaml renamed to ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2003_2003_24h_v1.yaml ice_station_zebra/config/datasets/osisaf-sic-north-2003-08-24h-v1.yaml renamed to ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2003_2003_24h_v1.yaml

@@ -1,5 +1,5 @@
-osisaf-sic-north-2010-08-24h-v1:
-  name: osisaf-sic-north-2010-08-24h-v1
+nbsamp-sicnorth-osisaf-25k-2010-2010-24h-v1:
+  name: nbsamp-sicnorth-osisaf-25k-2010-2010-24h-v1
   group_as: osisaf-north
   description: OSISAF sea ice concentration for northern hemisphere - Summer 2010 (August-September)
   attribution: EUMETSAT
@@ -16,4 +16,4 @@ osisaf-sic-north-2010-08-24h-v1:
 
   input:
     netcdf:
-      path: ${base_path}/data/preprocessing/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2010.nc
+      path: ${base_path}/data/preprocessing/${...name}/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2010.nc

…ets/osisaf-sic-north-2010-08-24h-v1.yaml …icnorth_osisaf_25k_2010_2010_24h_v1.yamlice_station_zebra/config/datasets/osisaf-sic-north-2010-08-24h-v1.yaml renamed to ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2010_2010_24h_v1.yaml ice_station_zebra/config/datasets/osisaf-sic-north-2010-08-24h-v1.yaml renamed to ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2010_2010_24h_v1.yaml

@@ -1,5 +1,5 @@
-osisaf-sic-north-2018-08-24h-v1:
-  name: osisaf-sic-north-2018-08-24h-v1
+nbsamp-sicnorth-osisaf-25k-2018-2018-24h-v1:
+  name: nbsamp-sicnorth-osisaf-25k-2018-2018-24h-v1
   group_as: osisaf-north
   description: OSISAF sea ice concentration for northern hemisphere - Summer 2018 (August-September)
   attribution: EUMETSAT
@@ -16,4 +16,4 @@ osisaf-sic-north-2018-08-24h-v1:
 
   input:
     netcdf:
-      path: ${base_path}/data/preprocessing/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2018.nc
+      path: ${base_path}/data/preprocessing/${...name}/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2018.nc

…ets/osisaf-sic-north-2018-08-24h-v1.yaml …icnorth_osisaf_25k_2018_2018_24h_v1.yamlice_station_zebra/config/datasets/osisaf-sic-north-2018-08-24h-v1.yaml renamed to ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2018_2018_24h_v1.yaml ice_station_zebra/config/datasets/osisaf-sic-north-2018-08-24h-v1.yaml renamed to ice_station_zebra/config/datasets/nbsamp_sicnorth_osisaf_25k_2018_2018_24h_v1.yaml

@@ -0,0 +1,21 @@
+nbsamp-weathersouth-era5-0p5-2020-2020-24h-v1:
+  name: nbsamp-weathersouth-era5-0p5-2020-2020-24h-v1
+  group_as: era5
+  description: Small ERA5 dataset for testing (0.5 deg, south)
+  attribution: ECMWF/C3S
+  licence: CC-BY-4.0
+
+  dates:
+    start: '2020-01-01T00:00:00'
+    end: '2020-04-30T23:00:00'
+    frequency: 24h
+
+  input:
+    pipe:
+      - anemoi-dataset:
+          join:
+          - dataset: ${base_path}/data/anemoi/samp-weathersouth-era5-0p5-2020-2020-24h-v1.zarr
+            select: [ 2t, z_250, z_500 , 10u, 10v, cos_julian_day, sin_julian_day ]
+      - doubling_filter:
+          input_field: 2t
+          output_field: doubled_2t

ice_station_zebra/config/datasets/nbsamp_weathersouth_era5_0p5_2020_2020_24h_v1.yaml

@@ -1,13 +1,13 @@
-osisaf-sic-south-2020-01-24h-v1:
-  name: osisaf-sic-south-2020-01-24h-v1
+samp-sicsouth-osisaf-25k-2020-2020-24h-v1:
+  name: samp-sicsouth-osisaf-25k-2020-2020-24h-v1
   group_as: osisaf-south
   description: OSISAF sea ice concentration for southern hemisphere
   attribution: EUMETSAT
   licence: CC-BY-4.0
 
   dates:
     start: '2020-01-01T00:00:00'
-    end: '2020-01-31T23:00:00'
+    end: '2020-04-30T23:00:00'
     frequency: 24h
 
   preprocessor:
@@ -16,4 +16,4 @@ osisaf-sic-south-2020-01-24h-v1:
 
   input:
     netcdf:
-      path: ${base_path}/data/preprocessing/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2020.nc
+      path: ${base_path}/data/preprocessing/${...name}/${...preprocessor.type}/data/osisaf/${...preprocessor.hemisphere}/siconca/2020.nc

ice_station_zebra/config/datasets/osisaf-sic-south-2019-12-24h-v1.yaml

@@ -1,13 +1,13 @@
-era5-0d5-south-2020-01-24h-v1:
-  name: era5-0d5-south-2020-01-24h-v1
-  group_as: era5-south
-  description: 0.5 degree resolution ERA5 for southern hemisphere
+samp-weathersouth-era5-0p5-2020-2020-24h-v1:
+  name: samp-weathersouth-era5-0p5-2020-2020-24h-v1
+  group_as: era5
+  description: Small ERA5 dataset for testing (0.5 deg, south)
   attribution: ECMWF/C3S
   licence: CC-BY-4.0
 
   dates:
     start: '2020-01-01T00:00:00'
-    end: '2020-01-31T23:00:00'
+    end: '2020-04-30T23:00:00'
     frequency: 24h
 
   input:
@@ -20,43 +20,31 @@ era5-0d5-south-2020-01-24h-v1:
         area: "0/-180/-90/180"  # southern hemisphere
         levtype: sfc
         param:
+        - 2t
+        - sp
         - 10u
         - 10v
-        - 2d
-        - 2t
-        - lsm
         - msl
-        - sdor
-        - skt
-        - slor
-        - sp
-        - tcw
-        - z
     - mars:
         use_cdsapi_dataset: "reanalysis-era5-complete"
         class: ea
         expver: "0001"
         grid: "0.5 / 0.5"
         area: "0/-180/-90/180"  # southern hemisphere
         level:
+        - 10
         - 250
         - 500
-        - 850
         - 1000
         levtype: pl
         param:
+        - z
+        - t
+        - q
         - u
         - v
-        - q
-        - t
-        - z
-    - accumulations:
-        use_cdsapi_dataset: "reanalysis-era5-complete"
-        accumulation_period: 6
-        class: ea
-        expver: "0001"
-        grid: "0.5 / 0.5"
-        area: "0/-180/-90/180"  # southern hemisphere
+    - forcings:
         param:
-        - cp
-        - tp
+        - cos_julian_day
+        - sin_julian_day
+        template: \${input.join.0.mars}