[Bug]: `pcmdi_diags_variability_modes_atm` fails with `assignment destination is read-only`

[forsyth2]

What happened?

When testing zppy (first with the test automation branch and then for the standard tests), the pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994 job failed with ValueError: assignment destination is read-only

What machine were you running on?

Chrysalis

Environment

source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate test-zi-main-20260130

What command did you run?

zppy -c tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg

Copy your cfg file

[default]
case = "v3.LR.historical_0051"
constraint = ""
dry_run = "False"
environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"
fail_on_dependency_skip = True
infer_path_parameters = False
infer_section_parameters = False
input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051
input_subdir = archive/atm/hist
mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051"
partition = "debug"
qos = "regular"
www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/zppy_main_branch_test_20260130"
years = "1985:1989:2",

[climo]
active = True
walltime = "00:30:00"

  [[ atm_monthly_180x360_aave ]]
  frequency = "monthly"
  input_files = "eam.h0"
  input_subdir = "archive/atm/hist"
  vars = ""

  [[ atm_monthly_diurnal_8xdaily_180x360_aave ]]
  active = True
  frequency = "diurnal_8xdaily"
  input_files = "eam.h3"
  input_subdir = "archive/atm/hist"
  vars = "PRECT"

  [[ land_monthly_climo ]]
  active = True
  frequency = "monthly"
  input_files = "elm.h0"
  input_subdir = "archive/lnd/hist"
  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
  vars = ""

[ts]
active = True
walltime = "00:30:00"

  [[ atm_monthly_180x360_aave ]]
  frequency = "monthly"
  input_files = "eam.h0"
  input_subdir = "archive/atm/hist"
  years = "1985:1995:2", # Need 10 years for pcmdi_diags task

  [[ atm_daily_180x360_aave ]]
  active = True
  frequency = "daily"
  input_files = "eam.h1"
  input_subdir = "archive/atm/hist"
  vars = "PRECT"

  [[ rof_monthly ]]
  active = True
  extra_vars = 'areatotal2'
  frequency = "monthly"
  input_files = "mosart.h0"
  input_subdir = "archive/rof/hist"
  mapping_file = ""
  vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"

  [[ atm_monthly_glb ]]
  # Note global average won't work for 3D variables.
  active = True
  frequency = "monthly"
  input_files = "eam.h0"
  input_subdir = "archive/atm/hist"
  mapping_file = "glb"
  years = "1985:1995:5",

  [[ lnd_monthly_glb ]]
  active = True
  frequency = "monthly"
  input_files = "elm.h0"
  input_subdir = "archive/lnd/hist"
  mapping_file = "glb"
  vars = ""
  years = "1985:1995:5",

  [[ land_monthly ]]
  extra_vars = "landfrac"
  frequency = "monthly"
  input_files = "elm.h0"
  input_subdir = "archive/lnd/hist"
  mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
  vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[e3sm_to_cmip]
active = True
frequency = "monthly"
ts_num_years=2
walltime = "00:30:00"

  [[ atm_monthly_180x360_aave ]]
  cmip_plevdata = "/lcrc/group/e3sm/diagnostics/e3sm_to_cmip_data/maps/vrt_remap_plev19.nc"
  cmip_vars = "ua, va, ta, wa, zg, hur, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlus, rsds, rsus, hfss, clivi, clwvi, rlut, rsdt, rsuscs, rsut, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin"
  input_files = "eam.h0"
  ts_subsection = "atm_monthly_180x360_aave"
  vars = "ICEFRAC,LANDFRAC,OCNFRAC,PSL,FSNTC,FSNTOAC,SWCF,LWCF,FLUT,FSNT,FSNTOA,FLNT,FLNTC,FSNS,FLNS,FSNS,SHFLX,QFLX,LHFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,U10,QREFHT,TMQ,CLDTOT,CLDHGH,CLDMED,CLDLOW,FLDS,FSDS,TGCLDIWP,TGCLDCWP,TGCLDLWP,FLNSC,FLUTC,FSDSC,SOLIN,FSNSC,AODABS,AODVIS,AODDUST,AREL,TREFMNAV,TREFMXAV,PS,PHIS,U,V,T,Z3"
  years = "1985:1995:2", # Need 10 years for pcmdi_diags task

  [[ land_monthly ]]
  input_files = "elm.h0"
  ts_subsection = "land_monthly"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
# [tc_analysis]
# active = True
# walltime = "00:30:00"

[e3sm_diags]
active = True
climo_diurnal_frequency = "diurnal_8xdaily"
climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
climo_subsection = "atm_monthly_180x360_aave"
environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate test-diags-main-20260130"
grid = '180x360_aave'
multiprocessing = True
num_workers = 8
partition = "compute"
qos = "regular"
ref_final_yr = 1986
ref_start_yr = 1985
ref_years = "1985-1986",
# Include all sets
# min_case_e3sm_diags_depend_on_climo: "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","annual_cycle_zonal_mean","zonal_mean_2d_stratosphere","aerosol_aeronet","aerosol_budget",
# min_case_e3sm_diags_depend_on_ts: "enso_diags","qbo",
# min_case_e3sm_diags_diurnal_cycle: "diurnal_cycle",
# min_case_e3sm_diags_streamflow: "streamflow",
# min_case_e3sm_diags_tc_analysis: "tc_analysis",
# min_case_e3sm_diags_tropical_subseasonal: "tropical_subseasonal",
# TODO: Add "tc_analysis" back in after empty dat is resolved.
# TODO: Add "aerosol_budget" back in once that's working for v3.
sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","annual_cycle_zonal_mean","zonal_mean_2d_stratosphere","enso_diags","qbo","diurnal_cycle","streamflow","tropical_subseasonal","aerosol_aeronet",
short_name = "v3.LR.historical_0051"
ts_daily_subsection = "atm_daily_180x360_aave"
ts_num_years = 2
ts_subsection = "atm_monthly_180x360_aave"
walltime = "5:00:00"
years = "1987:1989:2"
# Reference paths
# Used for mvo and mvm, if ts_num_years is set
obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/"
# mvo & mvm tc_analysis only
tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"

  [[ atm_monthly_180x360_aave ]]
  # Reference paths
  reference_data_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/climatology/"
  # mvo diurnal_cycle only
  dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
  # mvo streamflow only
  streamflow_obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/"

  [[ atm_monthly_180x360_aave_mvm ]]
  # Test model-vs-model using the same files as the reference
  diff_title = "Difference"
  partition = "compute"
  qos = "regular"
  ref_name = "v3.LR.historical_0051"
  run_type = "model_vs_model"
  short_ref_name = "same simulation"
  swap_test_ref = False
  tag = "model_vs_model"
  ts_num_years_ref = 2
  # Reference paths
  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/atm/180x360_aave/clim"
  # mvm streamflow only
  gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv"
  reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/rof/native/ts/monthly"
  # mvm diurnal_cycle only
  reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily"
  # mvm "enso_diags", "qbo", "area_mean_time_series"
  reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly"
  # mvm tropical_subseasonal only
  reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily"

  [[ lnd_monthly_mvm_lnd ]]
  # Test model-vs-model using the same files as the reference
  climo_subsection = "land_monthly_climo"
  diff_title = "Difference"
  partition = "compute"
  qos = "regular"
  ref_name = "v3.LR.historical_0051"
  run_type = "model_vs_model"
  sets = "lat_lon_land",
  short_ref_name = "same simulation"
  swap_test_ref = False
  tag = "model_vs_model"
  ts_num_years_ref = 2
  # Reference paths
  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/lnd/180x360_aave/clim"

[mpas_analysis]
active = True
anomalyRefYear = 1985
climo_years = "1985-1989", "1990-1995",
enso_years = "1985-1989", "1990-1995",
environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"
mesh = "IcoswISC30E3r5"
parallelTaskCount = 6
partition = "compute"
qos = "regular"
shortTermArchive = True
ts_years = "1985-1989", "1985-1995",
walltime = "02:00:00"

[global_time_series]
active = True
climo_years = "1985-1989", "1990-1995",
environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate test-zi-main-20260130"
experiment_name = "v3.LR.historical_0051"
figstr = "v3.LR.historical_0051"
#moc_file=mocTimeSeries_1985-1995.nc
# plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
ts_num_years = 5
ts_years = "1985-1989", "1985-1995",
walltime = "00:30:00"
years = "1985-1995",

  # Important parameter combinations ##########################################

  [[ viewer_both ]]
  # 1. make_viewer = True, plots_original set, >= 1 plots_<component> set
  # NOTE: This is the case displayed in examples/post.v3.LR.historical_zppy_v3.cfg
  make_viewer = True
  moc_file = "mocTimeSeries_1985-1995.nc"
  plots_atm = "TREFHT"
  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",

  [[ viewer_original ]]
  # 2. make_viewer = True, plots_original set, 0 plots_<component> set
  make_viewer = True
  moc_file = "mocTimeSeries_1985-1995.nc"
  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",

  [[ viewer_component ]]
  # 3. make_viewer = True, plots_original set, >= 1 plots_<component> set
  make_viewer = True
  plots_atm = "TREFHT"
  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
  plots_original = ""

  # We can ignore this case for image checking.
  # [[ viewer_none ]]
  # # 4. make_viewer = True, plots_original not set, 0 plots_<component> set
  # make_viewer = True
  # plots_original = ""

  [[ classic_pdf_both ]]
  # 5. make_viewer = False, plots_original set, >= 1 plots_<component> set
  # Default make_viewer = False
  moc_file = "mocTimeSeries_1985-1995.nc"
  plots_atm = "TREFHT"
  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",

  [[ classic_pdf_original ]]
  # 6. make_viewer = False, plots_original set, 0 plots_<component> set
  # NOTE: This is the default case -- where all of the above parameters are at their default values.
  # Default make_viewer = False
  moc_file = "mocTimeSeries_1985-1995.nc"
  # Default plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,net_atm_water_imbalance",

  [[ classic_pdf_component ]]
  # 7. make_viewer = False, plots_original set, >= 1 plots_<component> set
  # Default make_viewer = False
  plots_atm = "TREFHT"
  plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
  plots_original = ""

  # We can ignore this case for image checking.
  # [[ classic_pdf_none ]]
  # # 8. make_viewer = False, plots_original not set, 0 plots_<component> set
  # # Default make_viewer = False
  # plots_original = ""

  # Special cases #############################################################

  [[ all_lnd_var_viewer ]]
  # 1. plot ALL land variables
  make_viewer = True
  partition = "compute"
  plots_lnd = "all"
  plots_original = ""
  walltime = "03:00:00"

  [[ classic_original_8_no_ocn ]]
  # 2. exclude ocean from original plots
  plots_original = "net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"


[ilamb]
active = True
e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave"
e3sm_to_cmip_land_subsection = "land_monthly"
ilamb_obs = "/lcrc/group/e3sm/diagnostics/ilamb_data"
nodes = 8
partition = "debug"
short_name = "v3.LR.historical_0051"
ts_atm_subsection = "atm_monthly_180x360_aave"
ts_land_subsection = "land_monthly"
ts_num_years = 2
walltime = "2:00:00"
years = "1985:1989:4"

[pcmdi_diags]
active = True
environment_commands_secondary = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate test-zi-main-20260130"
walltime = "2:00:00"
model_name = 'e3sm.historical.v3-LR.0051'
model_tableID = 'Amon'
e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave" # Can be inferred if needed
multiprocessing = True
num_workers = 24
obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
pcmdi_debug = False
ts_num_years = 2
ts_years = "1985-1994",

  [[ mean_climate ]]
  current_set = "mean_climate" # Can be inferred if needed
  obs_sets = "default"
  ref_final_yr = 1994
  ref_start_yr = 1985
  ref_years = "1985-1994",
  cmip_vars = "pr,psl,rlut,rsut,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg"
  target_grid = "2.5x2.5"
  target_grid_string = "2p5x2p5"
  clim_vars = "pr,psl,rsus,rsuscs,rlus,rlut,rsds,rsdscs,rsdt,rsut,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850"
  clim_regions = "global,ocean,land,Tropics,NHEX,SHEX"

  [[ variability_modes_cpl ]]
  current_set = "variability_modes_cpl" # Can be inferred if needed
  obs_sets = "alternate1"
  ref_final_yr = 1994
  ref_start_yr = 1985
  ref_years = "1985-1994",
  frequency = 'mo'
  landmask = True
  ModUnitsAdjust = '(True,"subtract",273.15)'
  ObsUnitsAdjust = '(True,"subtract",273.15)'
  seasons = "yearly,monthly"
  movc_vars = "ts"
  movc_modes = "PDO,NPGO,AMO"

  [[ variability_modes_atm ]]
  current_set = "variability_modes_atm" # Can be inferred if needed
  obs_sets = "alternate2"
  ref_final_yr = 1994
  ref_start_yr = 1985
  ref_years = "1985-1994",
  frequency = 'mo'
  landmask = False
  ModUnitsAdjust = '(True,"divide",100.0)'
  ObsUnitsAdjust = '(True,"divide",100.0)'
  seasons = "DJF,MAM,JJA,SON,yearly,monthly"
  mova_vars = "psl"
  mova_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2"

  [[ synthetic_plots ]]
  current_set = "synthetic_plots" # Can be inferred if needed
  clim_vars = "pr,psl,ua-200"
  clim_regions = "global,ocean,land"
  mova_modes = "NAM,NAO,PNA,NPO,SAM"
  movc_modes = "PDO,NPGO"
  clim_viewer = True
  clim_years = "1985-1994"
  mova_viewer = True
  mova_years = "1985-1994"
  movc_viewer = True
  movc_years = "1985-1994"
  enso_viewer = False
  enso_years = ""
  cmip_clim_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate
  cmip_movs_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes

What jobs are failing?

pcmdi_diags_synthetic_plots_model_vs_obs.status:WAITING 1140312
pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.status:ERROR (14)

What stack trace are you encountering?

create_links_ts: linking from /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/atm/180x360_aave/cmip_ts/monthly to ts
ncap2: WARNING ncap_def_dim(): dim "bnds" - already exists in input/output.
Linking observational data into obs_link...
Linking observational data using SLURM...
Running a zi-pcmdi command: zi-pcmdi-link-observation --model_name_ref obs.historical.put_model_here.00 --tableID_ref Amon --vars=psl --obs_sets alternate2 --obs_ts /lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series --obstmp_dir obs_link --debug false
[Info] Linked psl → obs_link/obs.historical.NOAA-20C.00.Amon.psl.183601-201512.nc

real	0m10.073s
user	0m15.441s
sys	0m0.514s
create_links_ts_obs: linking from /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/scripts/tmp.pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.1140311.lAyF/obs_link to ts_ref
create_links_ts_obs: checking if nc file: obs.historical.NOAA-20C.00.Amon.psl.183601-201512.nc
create_links_ts_obs: processing /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/scripts/tmp.pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.1140311.lAyF/obs_link/obs.historical.NOAA-20C.00.Amon.psl.183601-201512.nc
ncrcat successful
Adding missing calendar attribute to time...
ncatted successful
ncap2: WARNING ncap_def_dim(): dim "bnds" - already exists in input/output.
ncap2 successful
About to create parameterfile.py, which will be passed in with -p
The current directory is: /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/scripts/tmp.pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.1140311.lAyF

===== RUN PCMDI DIAGS =====

About to set up a zi-pcmdi command
The current directory is: /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/scripts/tmp.pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.1140311.lAyF
Running a zi-pcmdi command: zi-pcmdi-variability-modes --num_workers 24 --multiprocessing True --subsection variability_modes_atm --climo_ts_dir_primary ts --climo_ts_dir_ref ts_ref --model_name e3sm.historical.v3-LR.0051 --model_tableID Amon --figure_format png  --run_type model_vs_obs --obs_sets alternate2 --model_name_ref obs.historical.put_model_here.00 --vars psl --tableID_ref Amon --generate_sftlf True --case_id v20260130 --results_dir model_vs_obs --debug false --var_modes NAM,NAO,PNA,NPO,SAM,PSA1,PSA2
The current directory is: /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/scripts/tmp.pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.1140311.lAyF
2026-01-30 18:48:56,166 [INFO]: pcmdi_setup.py(build_catalogues:76) >> DataCatalogueBuilder's vars: ['psl']
2026-01-30 18:48:56,167 [INFO]: pcmdi_setup.py(build_catalogues:78) >> Building catalogue for psl
2026-01-30 18:48:56,167 [INFO]: pcmdi_setup.py(build_catalogues:80) >> Looking for psl, the base var name of psl
2026-01-30 18:48:56,167 [INFO]: pcmdi_setup.py(build_catalogues:81) >> Finding test files in ts
2026-01-30 18:48:56,167 [INFO]: pcmdi_setup.py(build_catalogues:85) >> Finding ref files in ts_ref
2026-01-30 18:48:56,169 [INFO]: pcmdi_setup.py(build_catalogues:96) >> Extracting & assigining metadata for psl, the base var name of psl
2026-01-30 18:48:56,169 [INFO]: pcmdi_setup.py(_extract_metadata:135) >> Extracting metadata from e3sm.historical.v3-LR.0051.Amon.psl.198501-199412.nc, dervied from ts/e3sm.historical.v3-LR.0051.Amon.psl.198501-199412.nc
2026-01-30 18:48:56,169 [INFO]: pcmdi_setup.py(_extract_metadata:148) >> Parsing e3sm.historical.v3-LR.0051.Amon.psl.198501-199412.nc, determined mip=e3sm, exp=historical, model=v3-LR, realization=0051, tableID=Amon, yymms=198501, yymme=199412
2026-01-30 18:48:56,172 [INFO]: pcmdi_setup.py(_extract_metadata:135) >> Extracting metadata from obs.historical.NOAA-20C.00.Amon.psl.198501-199412.nc, dervied from ts_ref/obs.historical.NOAA-20C.00.Amon.psl.198501-199412.nc
2026-01-30 18:48:56,172 [INFO]: pcmdi_setup.py(_extract_metadata:148) >> Parsing obs.historical.NOAA-20C.00.Amon.psl.198501-199412.nc, determined mip=obs, exp=historical, model=NOAA-20C, realization=00, tableID=Amon, yymms=198501, yymme=199412
2026-01-30 18:48:56,172 [INFO]: pcmdi_setup.py(build_catalogues:111) >> test_files=['ts/e3sm.historical.v3-LR.0051.Amon.psl.198501-199412.nc']
2026-01-30 18:48:56,172 [INFO]: pcmdi_setup.py(build_catalogues:112) >> ref_files=['ts_ref/obs.historical.NOAA-20C.00.Amon.psl.198501-199412.nc']
2026-01-30 18:48:56,172 [INFO]: pcmdi_setup.py(build_catalogues:114) >> test_files[0]=ts/e3sm.historical.v3-LR.0051.Amon.psl.198501-199412.nc
2026-01-30 18:48:56,172 [INFO]: pcmdi_setup.py(build_catalogues:116) >> ref_files[0]=ts_ref/obs.historical.NOAA-20C.00.Amon.psl.198501-199412.nc
2026-01-30 18:48:56,172 [INFO]: pcmdi_setup.py(_save_catalogue:184) >> Saving catalogue pcmdi_diags/ts_variability_modes_atm_catalogue.json, absolute path /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/scripts/tmp.pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.1140311.lAyF/pcmdi_diags/ts_variability_modes_atm_catalogue.json
2026-01-30 18:48:56,173 [INFO]: pcmdi_setup.py(_save_catalogue:184) >> Saving catalogue pcmdi_diags/ts_ref_variability_modes_atm_catalogue.json, absolute path /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/scripts/tmp.pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.1140311.lAyF/pcmdi_diags/ts_ref_variability_modes_atm_catalogue.json
2026-01-30 18:48:57,452 [WARNING]: bounds.py(create_bounds:983) >> The 'lat' coordinate variable is missing a 'units' attribute. Assuming 'units' is 'degrees_north'.
2026-01-30 18:48:57,631 [WARNING]: bounds.py(create_bounds:983) >> The 'lat' coordinate variable is missing a 'units' attribute. Assuming 'units' is 'degrees_north'.
2026-01-30 18:48:57,642 [INFO]: pcmdi_setup.py(set_up:354) >> out_path=model_vs_obs/%(group_type)
2026-01-30 18:48:57,666 [INFO]: utils.py(run_parallel_jobs:61) >> Running 7 subprocesses...
2026-01-30 18:49:10,613 [ERROR]: utils.py(run_parallel_jobs:67) >> ERROR: Process failed: 'variability_modes_driver.py -p parameterfile.py --variability_mode NAM --eofn_mod 1 --eofn_obs 1 --varOBS psl --osyear 1985 --oeyear 1994 --reference_data_name NOAA-20C --reference_data_path ts_ref/obs.historical.NOAA-20C.00.Amon.psl.198501-199412.nc --case_id v20260130'
Error: /gpfs/fs1/home/ac.forsyth2/miniforge3/envs/test-zi-main-20260130/lib/python3.13/site-packages/xcdat/temporal.py:2223: UserWarning: The 'unit' keyword is only used when the Timedelta input is an integer or float, not timedelta64. To specify the storage unit of the output use `td.as_unit(unit)`
  median_delta = pd.to_timedelta(np.median(time_deltas), unit="ns")
Traceback (most recent call last):
  File "/gpfs/fs1/home/ac.forsyth2/miniforge3/envs/test-zi-main-20260130/bin/variability_modes_driver.py", line 338, in <module>
    obs_timeseries_season = adjust_timeseries(
        obs_timeseries, obs_var, mode, season, regions_specs, RmDomainMean
    )
  File "/gpfs/fs1/home/ac.forsyth2/miniforge3/envs/test-zi-main-20260130/lib/python3.13/site-packages/pcmdi_metrics/variability_mode/lib/adjust_timeseries.py", line 36, in adjust_timeseries
    ds_anomaly = get_anomaly_timeseries(ds, data_var, season)
  File "/gpfs/fs1/home/ac.forsyth2/miniforge3/envs/test-zi-main-20260130/lib/python3.13/site-packages/pcmdi_metrics/variability_mode/lib/adjust_timeseries.py", line 75, in get_anomaly_timeseries
    ds_anomaly_all_seasons = ds_anomaly.temporal.departures(
        data_var,
    ...<2 lines>...
        season_config={"dec_mode": "DJF", "drop_incomplete_djf": True},
    )
  File "/gpfs/fs1/home/ac.forsyth2/miniforge3/envs/test-zi-main-20260130/lib/python3.13/site-packages/xcdat/temporal.py", line 869, in departures
    ds = self._preprocess_dataset(ds)
  File "/gpfs/fs1/home/ac.forsyth2/miniforge3/envs/test-zi-main-20260130/lib/python3.13/site-packages/xcdat/temporal.py", line 1219, in _preprocess_dataset
    ds = self._shift_djf_decembers(ds)
  File "/gpfs/fs1/home/ac.forsyth2/miniforge3/envs/test-zi-main-20260130/lib/python3.13/site-packages/xcdat/temporal.py", line 1364, in _shift_djf_decembers
    time_coords.values[dec_indexes] = [
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^
ValueError: assignment destination is read-only
Land mask estimated using regionmask method.
Land mask estimated using regionmask method.
Execution failed: Subprocess failed: variability_modes_driver.py -p parameterfile.py --variability_mode NAM --eofn_mod 1 --eofn_obs 1 --varOBS psl --osyear 1985 --oeyear 1994 --reference_data_name NOAA-20C --reference_data_path ts_ref/obs.historical.NOAA-20C.00.Amon.psl.198501-199412.nc --case_id v20260130
successfully finish all jobs....

real	0m28.383s
user	1m10.499s
sys	0m7.048s

===== COPY FILES TO WEB SERVER =====

rsync: [sender] link_stat "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/zppy_main_branch_test_20260130/v3.LR.historical_0051/post/scripts/tmp.pcmdi_diags_variability_modes_atm_model_vs_obs_1985-1994.1140311.lAyF/model_vs_obs" failed: No such file or directory (2)
rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1338) [sender=3.4.1]

[forsyth2]

@tomvothecoder -- @chengzhuzhang mentioned this could be related to an issue in e3sm_diags that is ultimately coming from xcdat. I'm not seeing a related issue in e3sm_diags or in xcdat though. Let me know if this issue does look to be related to that and/or if there's a relevant issue/PR. Thanks!

[chengzhuzhang]

The error looks similar to what has been seen here. I think they might be relevant.

[forsyth2]

Yes, that does appear to be the same error. I'm still confused why it would show up despite no code changes to zppy or zppy-interfaces. I suppose if there was a recent xcdat release, then the conda solver might be picking up a newer version (the one with the bug), even using the same dev.yml.

[tomvothecoder]

Yes, that does appear to be the same error. I'm still confused why it would show up despite no code changes to zppy or zppy-interfaces. I suppose if there was a recent xcdat release, then the conda solver might be picking up a newer version (the one with the bug), even using the same dev.yml.

This issue propagated from an external change, either from numpy and/or xarray (related GH issue). xCDAT v0.11.2 includes a patch and will be available shortly on conda-forge.

EDIT: Opened this GitHub issue in Xarray. This issues stems from a change in Pandas 3.0 somewhere.

[forsyth2]

Thanks for the context & issue @tomvothecoder!

[forsyth2]

@tomvothecoder I see pydata/xarray#11133 was closed with

I don't think there is anything to do from xarray's perspective.

Does that mean there's a necessary change further up the call hierarchy (i.e., in xcdat, or indeed in zppy itself)?

[tomvothecoder]

@forsyth2 xCDAT v0.11.2 introduces a fix. Please constrain to that version. No other changes needed.

[forsyth2]

@tomvothecoder Great, thank you!

[forsyth2]

I've confirmed this error is no longer present on main, as of the 2026-03-05/06 test