Skip to content

More data to download for demo (sea-ice model example files added) #1366

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
lee1043 wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

More data to download for demo (sea-ice model example files added) #1366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6729466
update
lee1043 Apr 29, 2026
5a8f56f
clean up
lee1043 May 6, 2026
51da467
clean up
lee1043 May 6, 2026
6d0d71f
Merge branch 'main' into demo_data_update
lee1043 May 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 14 additions & 12 deletions doc/jupyter/Demo/Demo_0_download_data.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"\n",
"Download sample data and set up parameter files\n",
"\n",
"This Notebook sets up the data for the other demos. It is recommended that you run this and other demos in a conda environment with **xcdat** and **pcmdi_metrics** installed. The files to download are listed in https://pcmdi.llnl.gov/pss/pmpdata/pmp_tutorial_files.v20240201.txt.\n",
"This Notebook sets up the data for the other demos. It is recommended that you run this and other demos in a conda environment with **xcdat** and **pcmdi_metrics** installed. The files to download are listed in https://pcmdi.llnl.gov/pss/pmpdata/pmp_tutorial_files.v20260427.txt.\n",
"\n",
"**Note:** All observational datasets used in the PMP analysis are published through the [ESGF obs4MIPs project](https://esgf-node.llnl.gov/). In addition, several derived climatological products—while not currently available via ESGF—can be accessed directly via download.\n",
"\n",
Expand All @@ -26,7 +26,7 @@
"source": [
"# Lets get the file containing the data needed for this demo\n",
"import requests\n",
"r = requests.get(\"https://pcmdi.llnl.gov/pss/pmpdata/pmp_tutorial_files.v20240201.txt\")\n",
"r = requests.get(\"https://pcmdi.llnl.gov/pss/pmpdata/pmp_tutorial_files.v20260427.txt\")\n",
"with open(\"data_files.txt\",\"wb\") as f:\n",
" f.write(r.content)"
]
Expand Down Expand Up @@ -67,14 +67,6 @@
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/global/homes/l/lee1043/.conda/envs/pmp_devel_20250305/lib/python3.10/site-packages/esmpy/interface/loadESMF.py:94: VersionWarning: ESMF installation version 8.8.0, ESMPy version 8.8.0b0\n",
" warnings.warn(\"ESMF installation version {}, ESMPy version {}\".format(\n"
]
},
{
"name": "stdout",
"output_type": "stream",
Expand All @@ -86,14 +78,24 @@
"Downloading: 'CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc\n",
"Downloading: 'CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc\n",
"Downloading: 'CMIP5_demo_clims/cmip6.historical.MCM-UA-1-0.r1i1p1f1.mon.zg.198101-200512.AC.v20201119.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip6.historical.MCM-UA-1-0.r1i1p1f1.mon.zg.198101-200512.AC.v20201119.nc\n",
"Downloading: 'CMIP5_demo_data/psl_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/psl_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n",
"Downloading: 'CMIP5_demo_data/sftlf_fx_ACCESS1-0_amip_r0i0p0.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/sftlf_fx_ACCESS1-0_amip_r0i0p0.nc\n",
"Downloading: 'CMIP5_demo_data/cmip5.amip.ACCESS1-0.sftlf.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/cmip5.amip.ACCESS1-0.sftlf.nc\n",
"Downloading: 'CMIP5_demo_data/cmip5.historical.GISS-E2-H.sftlf.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/cmip5.historical.GISS-E2-H.sftlf.nc\n",
"Downloading: 'CMIP5_demo_data/ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n",
"Downloading: 'CMIP5_demo_timeseries/historical/atmos/day/pr/pr_day_GISS-E2-H_historical_r6i1p1_20000101-20051231.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_timeseries/historical/atmos/day/pr/pr_day_GISS-E2-H_historical_r6i1p1_20000101-20051231.nc\n",
"Downloading: 'CMIP5_demo_timeseries/historical/atmos/mon/siconc_SImon_E3SM-1-1_historical_r1i1p1f1_gr_201001-201412.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_timeseries/historical/atmos/mon/siconc_SImon_E3SM-1-1_historical_r1i1p1f1_gr_201001-201412.nc\n",
"Downloading: 'CMIP5_demo_timeseries/historical/atmos/mon/siconc_SImon_E3SM-1-1_historical_r1i1p1f1_gr_201001-201412.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_timeseries/historical/atmos/mon/siconc_SImon_E3SM-1-1_historical_r1i1p1f1_gr_201001-201412.nc\n",
"Downloading: 'CMIP5_demo_timeseries/historical/atmos/mon/siconc_SImon_E3SM-1-1_historical_r1i1p1f1_gr_201001-201412.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_timeseries/historical/atmos/mon/siconc_SImon_E3SM-1-1_historical_r1i1p1f1_gr_201001-201412.nc\n",
"Downloading: 'obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-0/v20210804/rlut_mon_CERES-EBAF-4-0_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-0/v20210804/rlut_mon_CERES-EBAF-4-0_PCMDI_gn.200301-201812.AC.v20210804.nc\n",
"Downloading: 'obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-1/v20210804/rlut_mon_CERES-EBAF-4-1_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-1/v20210804/rlut_mon_CERES-EBAF-4-1_PCMDI_gn.200301-201812.AC.v20210804.nc\n",
"Downloading: 'obs4MIPs_PCMDI_clims/pr/GPCP-2-3/v20210804/pr_mon_GPCP-2-3_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_clims/pr/GPCP-2-3/v20210804/pr_mon_GPCP-2-3_PCMDI_gn.200301-201812.AC.v20210804.nc\n",
"Downloading: 'obs4MIPs_PCMDI_clims/zg/ERA-INT/v20210804/zg_mon_ERA-INT_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_clims/zg/ERA-INT/v20210804/zg_mon_ERA-INT_PCMDI_gn.200301-201812.AC.v20210804.nc\n",
"Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n",
"Downloading: 'obs4MIPs_PCMDI_monthly/NASA-LaRC/CERES-EBAF-4-1/mon/rlut/gn/v20210727/rlut_mon_CERES-EBAF-4-1_PCMDI_gn_200301-201812.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NASA-LaRC/CERES-EBAF-4-1/mon/rlut/gn/v20210727/rlut_mon_CERES-EBAF-4-1_PCMDI_gn_200301-201812.nc\n",
"Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc\n",
"Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc\n",
"Downloading: 'obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc\n",
"Downloading: 'obs4MIPs_PCMDI_daily/NASA-JPL/GPCP-1-3/day/pr/gn/latest/pr_day_GPCP-1-3_PCMDI_gn_19961002-20170101.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_daily/NASA-JPL/GPCP-1-3/day/pr/gn/latest/pr_day_GPCP-1-3_PCMDI_gn_19961002-20170101.nc\n",
"Downloading: 'misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc\n",
"Downloading: 'misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc\n",
Expand Down Expand Up @@ -165,7 +167,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "pmp_devel_20250305",
"display_name": "pmp_devel_20250908",
"language": "python",
"name": "python3"
},
Expand All @@ -179,7 +181,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.10"
"version": "3.13.11"
},
"selected_variables": [],
"vcdat_file_path": "",
Expand Down
27 changes: 19 additions & 8 deletions doc/jupyter/Demo/sea_ice_line_plots.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@

# Load data -- model

# "/p/user_pub/pmp/demo/sea-ice/links_siconc/E3SM-1-0/historical/r1i2p2f1/siconc/siconc_SImon_E3SM-1-0_historical_r1i2p2f1_*_*.nc"
ds = xc.open_mfdataset(
"/p/user_pub/pmp/demo/sea-ice/links_siconc/E3SM-1-0/historical/r1i2p2f1/siconc/siconc_SImon_E3SM-1-0_historical_r1i2p2f1_*_*.nc"
"demo_data/CMIP5_demo_timeseries/historical/atmos/mon/siconc_SImon_E3SM-1-1_historical_r1i1p1f1_gr_201001-201412.nc"
)

# "/p/user_pub/pmp/demo/sea-ice/links_area/E3SM-1-0/areacello_Ofx_E3SM-1-0_historical_r1i1p1f1_gr.nc"
area = xc.open_dataset(
"/p/user_pub/pmp/demo/sea-ice/links_area/E3SM-1-0/areacello_Ofx_E3SM-1-0_historical_r1i1p1f1_gr.nc"
"demo_data/misc_demo_data/fx/areacello_Ofx_E3SM-1-0_historical_r1i1p1f1_gr.nc"
)

arctic = (
Expand All @@ -29,7 +32,10 @@

# Load data -- observation

obs_file = "/p/user_pub/pmp/demo/sea-ice/EUMETSAT/OSI-SAF-450-a-3-0/v20231201/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc"
# obs_file = "/p/user_pub/pmp/demo/sea-ice/EUMETSAT/OSI-SAF-450-a-3-0/v20231201/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc"
obs_file = (
"demo_data/misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc"
)
obs = xc.open_dataset(obs_file)
obs_area = 625
obs_arctic = (
Expand All @@ -44,15 +50,18 @@


# Time series plot
arctic.sel({"time": slice("1981-01-01", "2010-12-31")}).plot(label="E3SM-1-0")
# arctic.sel({"time": slice("1981-01-01", "2010-12-31")}).plot(label="E3SM-1-0")
arctic.sel({"time": slice("2010-01-01", "2014-12-31")}).plot(label="E3SM-1-0")
obs_arctic.plot(label="OSI-SAF")
plt.title("Arctic monthly sea ice extent")
plt.ylabel("Extent (km${^2}$)")
plt.xlabel("time")
plt.xlim(
[
cftime.DatetimeNoLeap(1981, 1, 16, 12, 0, 0, 0, has_year_zero=True),
cftime.DatetimeNoLeap(2010, 12, 16, 12, 0, 0, 0, has_year_zero=True),
# cftime.DatetimeNoLeap(1981, 1, 16, 12, 0, 0, 0, has_year_zero=True),
# cftime.DatetimeNoLeap(2010, 12, 16, 12, 0, 0, 0, has_year_zero=True),
cftime.DatetimeNoLeap(2010, 1, 16, 12, 0, 0, 0, has_year_zero=True),
cftime.DatetimeNoLeap(2014, 12, 16, 12, 0, 0, 0, has_year_zero=True),
]
)
plt.legend(loc="upper right", fontsize=9)
Expand All @@ -65,7 +74,8 @@
coords={"time": ds.time},
)
arctic_clim = arctic_ds.sel(
{"time": slice("1981-01-01", "2010-12-31")}
# {"time": slice("1981-01-01", "2010-12-31")}
{"time": slice("2010-01-01", "2014-12-31")}
).temporal.climatology("siconc", freq="month")
arctic_clim["time"] = [x for x in range(1, 13)]

Expand All @@ -78,7 +88,8 @@

arctic_clim.siconc.plot(label="E3SM-1-0")
obs_clim.ice_conc.plot(label="OSI-SAF")
plt.title("Arctic climatological sea ice extent\n1981-2010")
# plt.title("Arctic climatological sea ice extent\n1981-2010")
plt.title("Arctic climatological sea ice extent\2010-2014")
plt.xlabel("month")
plt.ylabel("Extent (km${^2}$)")
plt.xlim([1, 12])
Expand Down
26 changes: 16 additions & 10 deletions doc/jupyter/Demo/sea_ice_param.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@
# --------------------
# List of models to include in analysis
test_data_set = [
"E3SM-1-0",
"E3SM-1-1",
]

# realization can be a single realization, a list of realizations, or "*" for all realizations
realization = "r1i2p2f1"
realization = "r1i1p1f1"

# test_data_path is a template for the model data parent directory
test_data_path = "/p/user_pub/pmp/demo/sea-ice/links_siconc/%(model)/historical/%(realization)/siconc/"
test_data_path = "demo_data/CMIP5_demo_timeseries/historical/atmos/mon"

# filename_template is a template for the model data file name
# combine it with test_data_path to get complete data path
Expand All @@ -21,14 +21,16 @@
var = "siconc"

# Start and end years for model data
msyear = 1981
meyear = 2010
msyear = 2010
meyear = 2014

# Factor for adjusting model data to decimal rather than percent units
ModUnitsAdjust = (True, "multiply", 1e-2)

# Template for the grid area file
area_template = "/p/user_pub/pmp/demo/sea-ice/links_area/%(model)/*.nc"
area_template = (
"demo_data/misc_demo_data/fx/areacello_Ofx_E3SM-1-0_historical_r1i1p1f1_gr.nc"
)

# Area variable name; likely 'areacello' or 'areacella' for CMIP6
area_var = "areacello"
Expand All @@ -43,8 +45,12 @@
# Settings for the observational data
# ------------------------------------
# Paths to northern and southern hemisphere files
reference_data_path_nh = "/p/user_pub/pmp/demo/sea-ice/EUMETSAT/OSI-SAF-450-a-3-0/v20231201/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc"
reference_data_path_sh = "/p/user_pub/pmp/demo/sea-ice/EUMETSAT/OSI-SAF-450-a-3-0/v20231201/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc"
reference_data_path_nh = (
"demo_data/misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc"
)
reference_data_path_sh = (
"demo_data/misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc"
)

# Factor for adjusting obs data to decimal rather than percent units
ObsUnitsAdjust = (True, "multiply", 1e-2)
Expand All @@ -53,8 +59,8 @@
reference_data_set = "OSI-SAF"

# Start and end years for obs
osyear = 1988
oeyear = 2020
osyear = 2010
oeyear = 2014

# Name of sea ice concentration variable
obs_var = "ice_conc"
Expand Down
6 changes: 4 additions & 2 deletions doc/jupyter/Demo/sea_ice_sector_plots.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
# ----------
print("Creating Arctic map")
# Load and process data
obs_file = "/p/user_pub/pmp/demo/sea-ice/EUMETSAT/OSI-SAF-450-a-3-0/v20231201/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc"
obs_file = (
"demo_data/misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc"
)
obs = xc.open_dataset(obs_file)
obs = obs.mean("time")
mask = create_land_sea_mask(obs, lon_key="lon", lat_key="lat")
Expand Down Expand Up @@ -89,7 +91,7 @@
# ----------
print("Creating Antarctic map")
# Load and process data
f_os_s = "/p/user_pub/pmp/demo/sea-ice/EUMETSAT/OSI-SAF-450-a-3-0/v20231201/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc"
f_os_s = "demo_data/misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc"
obs = xc.open_dataset(f_os_s)
obs = obs.mean("time")
mask = create_land_sea_mask(obs, lon_key="lon", lat_key="lat")
Expand Down
12 changes: 9 additions & 3 deletions pcmdi_metrics/sea_ice/sea_ice_driver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,11 +345,13 @@
list_of_runs = realizations
else:
list_of_runs = realizations
print(list_of_runs)
print("runs: ", list_of_runs)

# Model grid area
print(lib.replace_multi(area_template, tags))
area = xc.open_dataset(glob.glob(lib.replace_multi(area_template, tags))[0])
area_path = glob.glob(lib.replace_multi(area_template, tags))[0]
print("area_path:", area_path)
area = xc.open_dataset(area_path)
area[area_var] = lib.adjust_units(area[area_var], AreaUnitsAdjust)

if len(list_of_runs) > 0:
Expand All @@ -363,6 +365,7 @@
"%(realization)": run,
}
test_data_tmp = lib.replace_multi(test_data_path, tags)
print("test_data_tmp:", test_data_tmp)
if "*" in test_data_tmp:
# Get the most recent version for last wildcard
ind = test_data_tmp.split("/")[::-1].index("*")
Expand All @@ -371,8 +374,11 @@
globbed.sort()
test_data_tmp = globbed[-1]
test_data_full_path = os.path.join(test_data_tmp, filename_template)
print("test_data_full_path 1:", test_data_full_path)
test_data_full_path = lib.replace_multi(test_data_full_path, tags)
print("test_data_full_path 2:", test_data_full_path)
test_data_full_path = glob.glob(test_data_full_path)
print("test_data_full_path 3:", test_data_full_path)
test_data_full_path.sort()
if len(test_data_full_path) == 0:
print("")
Expand Down Expand Up @@ -505,7 +511,7 @@
# Get regional metrics
# --------------------
print("\n-------------------------------------------")
print("Calculating model regional average metrics \nfor ", model)
print(f"Calculating model regional average metrics \nfor {model}")
print("--------------------------------------------")

for rgn in real_clim:
Expand Down
Loading