Aeronet format updates and diagnostic changes (#4131)

Author: nchawang

doc/sphinx/source/recipes/recipe_aod_aeronet_assess.rst

@@ -57,15 +57,15 @@ User settings in recipe
 
    *Required settings for script*
 
-   * wavel: The wavelength of interest for the evaluation, currently set up for 440nm
-   * min_days_per_mon: The minimum number of days used to calculate the AOD monthly mean
+   * wavel: The wavelength of interest for the evaluation. Defaults to 440nm.
+   * min_days_per_mon: The minimum number of days used to calculate the AOD monthly mean. Defaults to 1.
    * min_mon_per_seas: The minimum number of seasons used to calculate each
-     seasonal mean. This must be between 1 and 3.
+     seasonal mean. This must be between 1 and 3. Defaults to 3.
    * min_seas_per_year: The minimum number of seasonal means in each year. This
-     must be between 1 and 4.
+     must be between 1 and 4. Defaults to 4.
    * min_seas_per_clim: The minimum number of seasonal means used to calculate
      the multiannual seasonal mean. This must be btween 1 and the number of years
-     of available AeroNET data.
+     of available AeroNET data. Defaults to 5.
 
    *Optional settings for script*
 
@@ -96,6 +96,9 @@ Variables
 ---------
 
 * od440aer (atmos, monthly mean, longitude latitude time)
+* od550aer (atmos, monthly mean, longitude latitude time)
+* od870aer (atmos, monthly mean, longitude latitude time)
+
 
 
 Observations and reformat scripts

esmvaltool/cmorizers/data/cmor_config/AERONET.yml

@@ -5,7 +5,7 @@ filename: 'AOD_Level20_Monthly_V3.tar.gz'
 # Common global attributes for Cmorizer output
 attributes:
   dataset_id: AERONET
-  version: 20240406
+  version: 20250726
   tier: 3
   modeling_realm: atmos
   project_id: OBS6

esmvaltool/cmorizers/data/datasets.yml

@@ -4,7 +4,7 @@ datasets:
   AERONET:
     tier: 3
     source: "https://aeronet.gsfc.nasa.gov/"
-    last_access: 2024-04-06
+    last_access: 2025-07-26
     info: |
       Aerosol Optical Depth information from a worldwide network of stations.
 

esmvaltool/cmorizers/data/formatters/datasets/aeronet.py

@@ -7,7 +7,7 @@
     https://aeronet.gsfc.nasa.gov/
 
 Last access
-    20240406
+    20250726
 
 Download and processing instructions
     Download the following file:
@@ -89,7 +89,7 @@ def parse_contact(contact):
     if match is None:
         raise RuntimeError(f"Could not parse contact line {contact}")
     names = match.group("names").replace("_", " ").split(" and ")
-    emails = match.group("emails").split("_and_")
+    emails = re.split(r"_and_| and ", match.group("emails"))
     mailboxes = ", ".join(
         [
             f'"{name}" <{email}>'
@@ -273,7 +273,7 @@ def assemble_cube(stations, idx, wavelengths=None):
         points=wavelength_points,
         standard_name="radiation_wavelength",
         long_name="Wavelength",
-        var_name="wl",
+        var_name="wavelength",
         units="nm",
     )
     times = date_index.to_pydatetime()

esmvaltool/diag_scripts/aerosols/aod_aeronet_assess.py

@@ -309,7 +309,7 @@ def aod_analyse(model_data, aeronet_obs_cube, clim_seas, wavel):
     return figures, fig_scatter
 
 
-def preprocess_aod_obs_dataset(obs_dataset):
+def preprocess_aod_obs_dataset(obs_dataset, thresholds):
     """Calculate a multiannual seasonal mean AOD climatology.
 
     Observational AOD timeseries data from AeroNET are used to generate a
@@ -324,6 +324,11 @@ def preprocess_aod_obs_dataset(obs_dataset):
     ----------
     obs_dataset : ESMValTool dictionary. Holds meta data for the observational
         AOD dataset.
+    thresholds : Minimum days,months, seasons and years threshold dictionary; with keys
+        - min_days_per_mon
+        - min_mon_per_seas
+        - min_seas_per_year
+        - min_seas_per_clim
 
     Returns
     -------
@@ -332,12 +337,6 @@ def preprocess_aod_obs_dataset(obs_dataset):
     """
     obs_cube = iris.load_cube(obs_dataset[0]["filename"])
 
-    # Set up thresholds for generating the multi annual seasonal mean
-    min_days_per_mon = 1
-    min_mon_per_seas = 3
-    min_seas_per_year = 4
-    min_seas_per_clim = 5
-
     # Add the clim_season and season_year coordinates.
     iris.coord_categorisation.add_year(obs_cube, "time", name="year")
 
@@ -352,7 +351,7 @@ def preprocess_aod_obs_dataset(obs_dataset):
     num_days_var = obs_cube.ancillary_variable("Number of days")
     masked_months_obs_cube = obs_cube.copy(
         data=ma.masked_where(
-            num_days_var.data < min_days_per_mon, obs_cube.data
+            num_days_var.data < thresholds["min_days_per_mon"], obs_cube.data
         )
     )
 
@@ -370,7 +369,7 @@ def preprocess_aod_obs_dataset(obs_dataset):
         function=lambda values: ~ma.getmask(values),
     )
     annual_seasonal_mean.data = ma.masked_where(
-        annual_seasonal_count.data < min_mon_per_seas,
+        annual_seasonal_count.data < thresholds["min_mon_per_seas"],
         annual_seasonal_mean.data,
     )
 
@@ -389,7 +388,7 @@ def preprocess_aod_obs_dataset(obs_dataset):
         function=lambda values: ~ma.getmask(values),
     )
     multi_annual_seasonal_mean.data = ma.masked_where(
-        clim_season_agg_count.data < min_seas_per_clim,
+        clim_season_agg_count.data < thresholds["min_seas_per_clim"],
         multi_annual_seasonal_mean.data,
     )
     year_agg_count = multi_annual_seasonal_mean.aggregated_by(
@@ -403,7 +402,7 @@ def preprocess_aod_obs_dataset(obs_dataset):
     )
     for iseas in counter:
         multi_annual_seasonal_mean.data[iseas, :] = ma.masked_where(
-            year_agg_count.data[0, :] < min_seas_per_year,
+            year_agg_count.data[0, :] < thresholds["min_seas_per_year"],
             multi_annual_seasonal_mean.data[iseas, :],
         )
 
@@ -425,11 +424,17 @@ def main(config):
     datasets = group_metadata(input_data.values(), "dataset")
 
     # Default wavelength
-    wavel = "440"
+    wavel = config.get("wavel", "440")
+    thresholds = {
+        "min_days_per_mon": int(config.get("min_days_per_mon", 1)),
+        "min_mon_per_seas": int(config.get("min_mon_per_seas", 3)),
+        "min_seas_per_year": int(config.get("min_seas_per_year", 4)),
+        "min_seas_per_clim": int(config.get("min_seas_per_clim", 5)),
+    }
 
     # Produce climatology for observational dataset
     obs_dataset = datasets.pop(config["observational_dataset"])
-    obs_cube = preprocess_aod_obs_dataset(obs_dataset)
+    obs_cube = preprocess_aod_obs_dataset(obs_dataset, thresholds)
 
     for model_dataset, group in datasets.items():
         # 'model_dataset' is the name of the model dataset.

esmvaltool/recipes/examples/recipe_check_obs.yml

@@ -1084,7 +1084,7 @@ diagnostics:
     variables:
       od440aer:
     additional_datasets:
-      - {dataset: AERONET, project: OBS6, mip: AERmon, tier: 3, type: atmos, version: 20240406}
+      - {dataset: AERONET, project: OBS6, mip: AERmon, tier: 3, type: atmos, version: 20250726}
     scripts: null
 
   ANUClimate:

esmvaltool/recipes/recipe_aod_aeronet_assess.yml

@@ -44,7 +44,7 @@ diagnostics:
         start_year: 1994
         end_year: 2014
         additional_datasets:
-          - {dataset: AERONET, project: OBS6, mip: AERmon, tier: 3, type: atmos, version: 20240406}
+          - {dataset: AERONET, project: OBS6, mip: AERmon, tier: 3, type: atmos, version: 20250726}
 
       od440aer_season:
         <<: *var_od440aer