Merge pull request #45 from NorESMhub/feature/new_timeseries

 global_mean_timeseries: obs support and optional full-range time series

Author: mvertens

CONFIG_REFERENCE.md

@@ -111,6 +111,7 @@ Settings shared by all runs.
 | `central_longitude` | optional | `180` | Center longitude for lat/lon maps. |
 | `num_procs` | optional | `1` | Processors for parallel steps. `"*"` = all on the node. Does **not** affect SE regrid (that runs serially). |
 | `redo_plot` | optional | `false` | `true` = remake plots even if they exist. |
+| `global_mean_ts_full_range` | optional | `false` | `true` = also generate time series over the **full available year range** (2-D variables only) for the `global_mean_timeseries` drift plot, so its model line(s) span the whole run rather than just the `start_year`–`end_year` climo window. Only takes effect when `global_mean_timeseries` is in `plotting_scripts`; applies to the test case(s) and the baseline (skipped for obs, which has no time series). Cases whose full range already equals the climo range are reused (no extra pass). The full-range series are written to a sibling `s<start>-e<end>` subdirectory of `cam_ts_loc`. |
 
 ---
 

config_noresm_template.yaml config_noresm_template_base.yamlconfig_noresm_template.yaml renamed to config_noresm_template_base.yaml

@@ -1,12 +1,13 @@
-user: '<user name>'
-case: '<case name>'
-nick: '<case nickname>'
-case_base: 'baseline case name>'
-nick_base: 'baseline case nickname>'
+user: 'mvertens'
+case: 'n1850.ne16pg3_tn14.noresm3_0_beta22.i486.2026-07-31'
+nick: 'testcase'
+case_base: 'n1850.ne16pg3_tn14.noresm3_0_beta22.bdmc2_1p2.20260730'
+nick_base: 'refcase'
 
 # baseic diagnostic info
 diag_basic_info:
     compare_obs: false
+    global_mean_ts_full_range: true 
     obs_source: "ERA5_1deg"      # match an obs_name; omit to use each variable's default
     create_html: true
     defaults_overlay_file: adf_variable_defaults_noresm.yaml
@@ -31,8 +32,8 @@ diag_cam_climo:
     case_nickname: ${nick}
     cam_hist_loc: /nird/datalake/NS9560K/noresm3/cases/${diag_cam_climo.cam_case_name}/atm/hist
     cam_climo_loc: /scratch/${user}/noresm3/ADF/${diag_cam_climo.cam_case_name}/atm/proc/climo
-    start_year: 960
-    end_year: 969
+    start_year: 1140
+    end_year: 1149
     cam_ts_done: false
     cam_ts_save: true
     cam_overwrite_ts: false
@@ -50,8 +51,8 @@ diag_cam_baseline_climo:
     case_nickname: ${nick_base}
     cam_hist_loc: /nird/datalake/NS9560K/noresm3/cases/${diag_cam_baseline_climo.cam_case_name}/atm/hist
     cam_climo_loc: /scratch/${user}/noresm3/ADF/${diag_cam_baseline_climo.cam_case_name}/atm/proc/climo
-    start_year: 960
-    end_year: 969
+    start_year: 1200
+    end_year: 1209
     cam_ts_done: false
     cam_ts_save: false
     cam_overwrite_ts: false

config_noresm_template_obs.yaml

@@ -0,0 +1,178 @@
+user: '<user name>'
+case: '<case name>'
+nick: '<case nickname>'
+
+# baseic diagnostic info
+diag_basic_info:
+    compare_obs: true
+    global_mean_ts_full_range: true
+    obs_source: "ERA5_1deg"      # match an obs_name; omit to use each variable's default
+    create_html: true
+    defaults_overlay_file: adf_variable_defaults_noresm.yaml
+    obs_data_loc: /nird/datalake/NS16000B/ADF-obs
+    cam_regrid_loc: /scratch/${user}/noresm3/${diag_cam_climo.cam_case_name}/atm/proc/tseries/regrid
+    cam_overwrite_regrid: false
+    cam_se_grid: ne16
+    cam_se_weight_file_ne16: /nird/datalake/NS9560K/diagnostics/land_xesmf_diag_data/map_ne16pg3_to_1.9x2.5_nomask_scripgrids_c250425.nc
+    cam_se_weight_file_ne30: /nird/datalake/NS9560K/diagnostics/land_xesmf_diag_data/map_ne30pg3_to_0.5x0.5_nomask_aave_da_c180515.nc
+    cam_diag_plot_loc: /nird/datalake/NS9560K/www/diagnostics/noresm/${user}/${case}/ADF
+    plot_press_levels: [200,500,850]
+    central_longitude: 180
+    num_procs: 8
+    redo_plot: true
+
+# info for target diagnostic case
+diag_cam_climo:
+    hist_str: cam.h0a
+    calc_cam_climo: true
+    cam_overwrite_climo: false
+    cam_case_name: ${case}
+    case_nickname: ${nick}
+    cam_hist_loc: /nird/datalake/NS9560K/noresm3/cases/${diag_cam_climo.cam_case_name}/atm/hist
+    cam_climo_loc: /scratch/${user}/noresm3/ADF/${diag_cam_climo.cam_case_name}/atm/proc/climo
+    start_year: 960
+    end_year: 969
+    cam_ts_done: false
+    cam_ts_save: true
+    cam_overwrite_ts: false
+    cam_ts_loc: /scratch/${user}/noresm3/ADF/${diag_cam_climo.cam_case_name}/atm/proc/tseries
+    tem_hist_str: cam.h4
+    cam_tem_loc: /scratch/${user}/noresm3/ADF/${diag_cam_climo.cam_case_name}/tem/
+    overwrite_tem: false
+
+# don't run cvdp
+diag_cvdp_info:
+    cvdp_run: false
+
+# don't run mdtf
+diag_mdtf_info:
+    mdtf_run: false
+
+# any of the scripts in scripts/averaging without the .py extension
+time_averaging_scripts:
+    - create_climo_files
+
+# any of the scripts in scripts/regridding without the .py extension
+regridding_scripts:
+    - regrid_and_vert_interp
+
+# any of the scripts in scripts/analysis without the .py extension
+analysis_scripts:
+    - amwg_table
+   #- ENSO_acrossRuns
+   #- aerosol_gas_tables
+
+# any of the scripts in scripts/plotting without the .py extension
+plotting_scripts:
+    - global_mean_timeseries
+    - global_latlon_map
+    - zonal_mean
+    - meridional_mean
+    - cam_taylor_diagram
+  # - global_latlon_vect_map
+  # - polar_map
+  # - qbo
+  # - ozone_diagnostics
+  # - enso_comparison_plots
+  # - tape_recorder
+  # - tem
+  # - regional_map_multicase #To use this please un-comment and fill-out
+                             #the "region_multicase" section below
+
+# variables to diagnose and plot
+diag_var_list:
+    - AODVIS
+    - mmr_BC
+    - mmr_DUST
+    - mmr_OM
+    - mmr_SALT
+    - mmr_SULFATE
+  # - D550_BC
+  # - D550_DU
+  # - D550_POM
+  # - D550_SO4
+  # - D550_SS
+    - cb_SULFATE
+    - cb_isoprene
+    - cb_monoterp
+    - cb_DUST
+    - cb_DMS
+    - cb_BC
+    - cb_OM
+    - cb_H2O2
+    - cb_H2SO4
+    - cb_SALT
+    - cb_SO2
+    - SFmonoterp
+    - SFisoprene
+    - SFSS
+    - SFSO4
+    - SFSO2_net
+    - SFOM
+    - SFBC
+    - SFDMS
+    - SFH2O2
+    - SFH2SO4
+    - CLDHGH
+    - CLDICE
+    - CLDLIQ
+    - CLDLOW
+    - CLDMED
+    - CLDTOT
+    - CLOUD
+    - RESTOM
+    - ALBEDOSURF
+    - TOTCF 
+    - FLNS
+    - FLNSC
+    - FLNT
+    - FLNTC
+    - FLUS
+    - FLDS
+    - FSDS
+    - FSDSC
+    - FSNS
+    - FSNT
+    - FSNTC
+    - FSUS
+    - LHFLX
+    - LWCF
+    - PBLH
+    - PRECT
+    - PS
+    - PSL
+    - QFLX
+    - RELHUM
+    - SHFLX
+    - SWCF
+    - T
+    - TAUX
+    - TAUY
+    - TGCLDIWP
+    - TGCLDLWP
+    - TMQ
+    - TREFHT
+    - TS
+    - U
+    - U10
+    - ICEFRAC
+    - OCNFRAC
+    - LANDFRAC
+    - ALBEDO
+    - ALBEDOC
+
+# <Add more variables here.>
+
+# Options for multi-case regional contour plots (./plotting/regional_map_multicase.py)
+# for region_time_option - if calendar, will look for specified years, if zeroanchor
+#  will use a nyears starting from year_offset from the beginning of timeseries
+# region_multicase:
+#     region_spec: [slat, nlat, wlon, elon]
+#     region_time_option: <calendar | zeroanchor>
+#     region_start_year:
+#     region_end_year:
+#     region_nyear:
+#     region_year_offset:
+#     region_month: <NULL means look for season>
+#     region_season: <NULL means use annual mean>
+#     region_variables: <list of variables to try to use; allows for a subset of the total diag variables>

lib/adf_dataset.py

@@ -127,16 +127,24 @@ def _hist_strs_for_reference(self):
     # Time series files
     #------------------
     # Test case(s)
-    def get_timeseries_file(self, case, field, hist_str=None):
+    def get_timeseries_file(self, case, field, hist_str=None, full_range=False):
         """Return list of test time series files.
 
         If hist_str is given, restrict the search to that history stream
         (time series files are named {case}.{hist_str}.{field}.*.nc).
+
+        If full_range is True, look in the full-range year-stamped subdirectory
+        (a sibling of the climo-range one) written by
+        create_time_series(full_range=True) -- used by the global-mean drift plot.
         """
         # list of paths (could be multiple cases)
         ts_locs = self.adf.get_cam_info("cam_ts_loc", required=True)
         caseindex = (self.case_names).index(case)
         ts_loc = Path(ts_locs[caseindex])
+        if full_range:
+            syr = self.adf.climo_yrs["syears_all"][caseindex]
+            eyr = self.adf.climo_yrs["eyears_all"][caseindex]
+            ts_loc = ts_loc.parent / f"s{syr}-e{eyr}"
         if hist_str:
             ts_filenames = f'{case}.{hist_str}.{field}.*nc'
         else:
@@ -145,16 +153,23 @@ def get_timeseries_file(self, case, field, hist_str=None):
         return ts_files
 
     # Reference case (baseline/obs)
-    def get_ref_timeseries_file(self, field, hist_str=None):
+    def get_ref_timeseries_file(self, field, hist_str=None, full_range=False):
         """Return list of reference time series files.
 
         If hist_str is given, restrict the search to that history stream.
+
+        If full_range is True, look in the full-range year-stamped subdirectory
+        written by create_time_series(baseline=True, full_range=True).
         """
         if self.adf.compare_obs:
             warnings.warn("\t    WARNING: ADF does not currently expect "
                           "observational time series files.")
             return None
         ts_loc = Path(self.adf.get_baseline_info("cam_ts_loc", required=True))
+        if full_range:
+            syr = self.adf.climo_yrs["syear_baseline_all"]
+            eyr = self.adf.climo_yrs["eyear_baseline_all"]
+            ts_loc = ts_loc.parent / f"s{syr}-e{eyr}"
         if hist_str:
             ts_filenames = f'{self.ref_case_label}.{hist_str}.{field}.*nc'
         else:

lib/adf_diag.py

@@ -323,9 +323,17 @@ def __function_caller(
 
     #########
 
-    def create_time_series(self, baseline=False):
+    def create_time_series(self, baseline=False, full_range=False):
         """
         Generate time series versions of the CAM history file data.
+
+        If ``full_range`` is True, generate time series over the *full available*
+        year range (``climo_yrs[..._all]``) into a separate year-stamped
+        directory, and only for 2-D variables. This is the optional pass used by
+        the ``global_mean_ts_full_range`` time series so the global-mean drift
+        plot spans the whole run rather than just the climo window (see
+        ``run_adf_diag``). Cases whose full range already equals the climo range
+        are skipped (the normal pass already produced those files).
         """
 
         #Notify user that script has started:
@@ -369,6 +377,24 @@ def call_ncrcat(cmd):
             hist_str_list = self.hist_string["test_hist_str"]
         # End if
 
+        # For the optional "full range" pass, override the years and output
+        # directories: use the full available range (climo_yrs[..._all]) and a
+        # separate year-stamped subdirectory (sibling of the climo-range one).
+        # The climo years are kept so cases whose full range already equals the
+        # climo range can be skipped below.
+        climo_syears = start_years
+        climo_eyears = end_years
+        if full_range:
+            if baseline:
+                start_years = [self.climo_yrs["syear_baseline_all"]]
+                end_years = [self.climo_yrs["eyear_baseline_all"]]
+            else:
+                start_years = self.climo_yrs["syears_all"]
+                end_years = self.climo_yrs["eyears_all"]
+            ts_dirs = [os.path.join(os.path.dirname(str(d)), f"s{sy}-e{ey}")
+                       for d, sy, ey in zip(ts_dirs, start_years, end_years)]
+        # End if
+
         # Read hist_str (component.hist_num) from the yaml file, or set to default
         dmsg = f"reading from {hist_str_list} files"
         self.debug_log(dmsg)
@@ -397,6 +423,16 @@ def call_ncrcat(cmd):
             start_year = start_years[case_idx]
             end_year = end_years[case_idx]
 
+            # Full-range reuse: if the full available range already equals the
+            # climo range for this case, the normal (climo) pass already wrote
+            # these time series -- skip to avoid duplicating them.
+            if full_range and (str(climo_syears[case_idx]) == str(start_year)
+                               and str(climo_eyears[case_idx]) == str(end_year)):
+                print(f"\tNOTE: full range for '{case_name}' equals the climo range; "
+                      "reusing the existing time series.")
+                continue
+            # End if
+
             # Create path object for the CAM history file(s) location:
             starting_location = Path(cam_hist_locs[case_idx])
 
@@ -534,6 +570,16 @@ def call_ncrcat(cmd):
                 # Loop over CAM history variables:
                 ts_output_files = []  # native SE time-series files to regrid to lat/lon
                 for var in diag_var_list:
+                    # In the full-range pass, only 2-D variables are needed (the
+                    # global-mean time series plot skips 3-D fields), so skip any
+                    # variable that carries a vertical dimension.
+                    if full_range and var in hist_file_ds.variables:
+                        _vdims = hist_file_ds[var].dims
+                        if ("lev" in _vdims) or ("ilev" in _vdims):
+                            continue
+                        #End if
+                    #End if
+
                     # Notify user of new time series file:
                     print(f"\t - time series for {var}")