PMP viewer generation on NERSC

[lee1043]

@kristinchang3 Using the latest main branch I was trying to generate PMP viewer under NERSC's www directory but getting the following issue. Can you please take a look and make edits as needed?

from pcmdi_metrics.viewer import generate_pmp_output_viewer_multimodel
output_dir = "/global/cfs/projectdirs/m4581/www/PMP/viewer"
generate_pmp_output_viewer_multimodel(output_dir=output_dir)

skipping Failed to fetch URL: variability_modes, cmip5, amip
skipping Failed to fetch URL: variability_modes, cmip6, amip
skipping Failed to fetch URL: enso_metric, cmip5, amip
skipping Failed to fetch URL: enso_metric, cmip6, amip

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[3], [line 1](vscode-notebook-cell:?execution_count=3&line=1)
----> [1](vscode-notebook-cell:?execution_count=3&line=1) generate_pmp_output_viewer_multimodel(output_dir=output_dir)

File ~/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/pcmdi_metrics/viewer/pmp_output_viewer.py:119, in generate_pmp_output_viewer_multimodel(mips, exps, metrics, output_dir, assets_path)
    [117]pcmdi_metrics/viewer/pmp_output_viewer.py:117) if "mean_climate" in metrics:
    [118]pcmdi_metrics/viewer/pmp_output_viewer.py:118)     mean_clim_dict = viewer_dict["mean_climate"]
--> [119]pcmdi_metrics/viewer/pmp_output_viewer.py:119)     mean_clim_divedown_layout = create_mean_clim_divedown_layout(
    [120]pcmdi_metrics/viewer/pmp_output_viewer.py:120)         mean_clim_dict, mips, exps, todays_date, assets_path
    [121]pcmdi_metrics/viewer/pmp_output_viewer.py:121)     )
    [122]pcmdi_metrics/viewer/pmp_output_viewer.py:122)     output_file(os.path.join(output_dir, "mean_climate_divedown.html"))
    [123]pcmdi_metrics/viewer/pmp_output_viewer.py:123)     save(mean_clim_divedown_layout)

File ~/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/pcmdi_metrics/viewer/pmp_output_viewer.py:232, in create_mean_clim_divedown_layout(mean_climate_dict, mips, exps, todays_date, assets_path)
    [208]pcmdi_metrics/viewer/pmp_output_viewer.py:208) def create_mean_clim_divedown_layout(
    [209]pcmdi_metrics/viewer/pmp_output_viewer.py:209)     mean_climate_dict, mips, exps, todays_date, assets_path
    [210]pcmdi_metrics/viewer/pmp_output_viewer.py:210) ):
    [211]pcmdi_metrics/viewer/pmp_output_viewer.py:211)     """
    [212]pcmdi_metrics/viewer/pmp_output_viewer.py:212)     Creates a bokeh layout object for mean climate dive down plots.
    [213]pcmdi_metrics/viewer/pmp_output_viewer.py:213) 
   (...)
    [230]pcmdi_metrics/viewer/pmp_output_viewer.py:230)         Arranged bokeh grid of the custom PMP Viewer banner, title text, multichoice dropdown filter widgets, and data table.
    [231]pcmdi_metrics/viewer/pmp_output_viewer.py:231)     """
--> [232]pcmdi_metrics/viewer/pmp_output_viewer.py:232)     df = create_mean_clim_divedown_df(mean_climate_dict, mips, assets_path)
    [233]pcmdi_metrics/viewer/pmp_output_viewer.py:233)     source = ColumnDataSource(data=dict(df))
    [234]pcmdi_metrics/viewer/pmp_output_viewer.py:234)     filtered_data = df.loc[
    [235]pcmdi_metrics/viewer/pmp_output_viewer.py:235)         (df["Region"] == "global") & (df["Experiment"] == "historical")
    [236]pcmdi_metrics/viewer/pmp_output_viewer.py:236)     ]

File ~/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/pcmdi_metrics/viewer/pmp_output_viewer.py:965, in create_mean_clim_divedown_df(mean_clim_dict, mips, assets_path)
    [941]pcmdi_metrics/viewer/pmp_output_viewer.py:941) def create_mean_clim_divedown_df(mean_clim_dict, mips, assets_path):
    [942]pcmdi_metrics/viewer/pmp_output_viewer.py:942)     """
    [943]pcmdi_metrics/viewer/pmp_output_viewer.py:943)     Creates a pandas dataframe with links to each season mean climate dive down image from the PMP Database Archive.
    [944]pcmdi_metrics/viewer/pmp_output_viewer.py:944) 
   (...)
    [955]pcmdi_metrics/viewer/pmp_output_viewer.py:955)         A dataframe of mean climate dive down info and plot links to be converted to a bokeh data table.
    [956]pcmdi_metrics/viewer/pmp_output_viewer.py:956)     """
    [957]pcmdi_metrics/viewer/pmp_output_viewer.py:957)     (
    [958]pcmdi_metrics/viewer/pmp_output_viewer.py:958)         exps,
    [959]pcmdi_metrics/viewer/pmp_output_viewer.py:959)         cmip6_models,
    [960]pcmdi_metrics/viewer/pmp_output_viewer.py:960)         cmip5_models,
    [961]pcmdi_metrics/viewer/pmp_output_viewer.py:961)         all_models,
    [962]pcmdi_metrics/viewer/pmp_output_viewer.py:962)         all_vars,
    [963]pcmdi_metrics/viewer/pmp_output_viewer.py:963)         regions,
    [964]pcmdi_metrics/viewer/pmp_output_viewer.py:964)         seasons,
--> [965]pcmdi_metrics/viewer/pmp_output_viewer.py:965)     ) = retrieve_lists(mean_clim_dict, "mean_climate", mips)
    [967]pcmdi_metrics/viewer/pmp_output_viewer.py:967)     multi_index = pd.MultiIndex.from_product(
    [968]pcmdi_metrics/viewer/pmp_output_viewer.py:968)         [mips, exps, all_models, all_vars, regions],
    [969]pcmdi_metrics/viewer/pmp_output_viewer.py:969)         names=["MIP", "Experiment", "Model", "Variable", "Region"],
    [970]pcmdi_metrics/viewer/pmp_output_viewer.py:970)     )
    [971]pcmdi_metrics/viewer/pmp_output_viewer.py:971)     df = pd.DataFrame(index=multi_index).reset_index()

File ~/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/pcmdi_metrics/viewer/pmp_output_viewer.py:1513, in retrieve_lists(metrics_dict, metric_name, mips)
   [1511]pcmdi_metrics/viewer/pmp_output_viewer.py:1511) if mip == "cmip5":
   [1512]pcmdi_metrics/viewer/pmp_output_viewer.py:1512)     for exp in exps:
-> [1513]pcmdi_metrics/viewer/pmp_output_viewer.py:1513)         json_data = load_json_from_url(metrics_dict["cmip5"][exp][0])
   [1514]pcmdi_metrics/viewer/pmp_output_viewer.py:1514)         cmip5_models_temp.append(list(json_data["RESULTS"].keys()))
   [1515]pcmdi_metrics/viewer/pmp_output_viewer.py:1515)     for m5 in cmip5_models_temp:

IndexError: list index out of range

[lee1043]

@kristinchang3 Below is for a simplified test, which still failing. Any chance it look familiar to you?

Code to reproduce error (test_viewer.py):

from pcmdi_metrics.viewer import generate_pmp_output_viewer_multimodel
output_dir = "."
generate_pmp_output_viewer_multimodel(output_dir=output_dir)

> python test_viewer.py 

/global/homes/l/lee1043/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/esmpy/interface/loadESMF.py:94: VersionWarning: ESMF installation version 8.8.0, ESMPy version 8.8.0b0
  warnings.warn("ESMF installation version {}, ESMPy version {}".format(
skipping Failed to fetch URL: variability_modes, cmip5, amip
skipping Failed to fetch URL: variability_modes, cmip6, amip
skipping Failed to fetch URL: enso_metric, cmip5, amip
skipping Failed to fetch URL: enso_metric, cmip6, amip
Traceback (most recent call last):
  File "/global/cfs/cdirs/m4581/lee1043/PMP/PMP_viewer/test_viewer.py", line 3, in <module>
    generate_pmp_output_viewer_multimodel(output_dir=output_dir)
  File "/global/homes/l/lee1043/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/pcmdi_metrics/viewer/pmp_output_viewer.py", line 119, in generate_pmp_output_viewer_multimodel
    mean_clim_divedown_layout = create_mean_clim_divedown_layout(
  File "/global/homes/l/lee1043/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/pcmdi_metrics/viewer/pmp_output_viewer.py", line 232, in create_mean_clim_divedown_layout
    df = create_mean_clim_divedown_df(mean_climate_dict, mips, assets_path)
  File "/global/homes/l/lee1043/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/pcmdi_metrics/viewer/pmp_output_viewer.py", line 965, in create_mean_clim_divedown_df
    ) = retrieve_lists(mean_clim_dict, "mean_climate", mips)
  File "/global/homes/l/lee1043/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/pcmdi_metrics/viewer/pmp_output_viewer.py", line 1513, in retrieve_lists
    json_data = load_json_from_url(metrics_dict["cmip5"][exp][0])
IndexError: list index out of range

[kristinchang3]

Hi @lee1043 this function: find_pmp_archive_json_urls() seems to be returning an empty array for mean climate. I tested it on its own: find_pmp_archive_json_urls(metric='mean_climate', mip='cmip6', exp='historical') and still get an empty array.

I did some investigation in the database API and find that "matches" in find_json_files_in_the_directory(url) is not returning any matches.

Were there any changes made to the pmp results archive that may be causing this?