Skip to content

Commit 2142f16

Browse files
forsyth2forsyth2
committed
Updates on Perlmutter
1 parent c4529a8 commit 2142f16

File tree

2 files changed

+86
-40
lines changed

2 files changed

+86
-40
lines changed

‎utils/generate_tables.py‎

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
# Make edits on Perlmutter so HPSS is available.
99
# TODO: update this file!!!
1010
# TODO: update HPSS paths in v1 csv
11+
# TODO: create tables
1112

1213
# Functions to compute fields for simulations ###########################################
1314
def get_data_size_and_hpss(hpss_path: str) -> Tuple[str, str]:
@@ -41,22 +42,25 @@ def get_data_size_and_hpss(hpss_path: str) -> Tuple[str, str]:
4142
hpss = ""
4243
return (data_size, hpss)
4344

44-
def get_esgf(source_id: str, model_version: str, experiment: str, ensemble_num: str, cmip_only: str, node: str) -> str:
45+
def get_esgf(source_id: str, model_version: str, experiment: str, ensemble_num: str, link_type: str, node: str) -> str:
46+
esgf: str
4547
if node == "cels.anl":
4648
esgf = f"`CMIP <https://esgf-node.{node}.gov/search/?project=CMIP6&activeFacets=%7B%22source_id%22%3A%22{source_id}%22%2C%22experiment_id%22%3A%22{experiment}%22%2C%22variant_label%22%3A%22r{ensemble_num}i1p1f1%22%7D>`_"
4749
elif experiment and ensemble_num:
4850
# See https://github.com/E3SM-Project/CMIP6-Metadata/pull/9#issuecomment-1246086256 for the table of ensemble numbers
49-
# remove v from model_version
50-
esgf = f"`Native <https://esgf-node.{node}.gov/search/e3sm/?model_version={model_version[1:]}_0&experiment={experiment}&ensemble_member=ens{ensemble_num}>`_"
51+
# Note that `[1:]`` removes `v` from `model_version`
52+
esgf_native: str = f"`Native <https://esgf-node.{node}.gov/search/e3sm/?model_version={model_version[1:]}_0&experiment={experiment}&ensemble_member=ens{ensemble_num}>`_"
5153
if experiment == 'hist-all-xGHG-xaer':
5254
experiment_id = 'hist-nat'
5355
else:
5456
experiment_id = experiment
55-
esgf_cmip = f"`CMIP <https://esgf-node.{node}.gov/search/cmip6/?source_id={source_id}&experiment_id={experiment_id}&variant_label=r{ensemble_num}i1p1f1>`_"
56-
if cmip_only:
57+
esgf_cmip: str = f"`CMIP <https://esgf-node.{node}.gov/search/cmip6/?source_id={source_id}&experiment_id={experiment_id}&variant_label=r{ensemble_num}i1p1f1>`_"
58+
if link_type == "cmip":
5759
esgf = esgf_cmip
60+
elif link_type == "native":
61+
esgf = esgf_native
5862
else:
59-
esgf = esgf_cmip + ', ' + esgf
63+
esgf = esgf_cmip + ', ' + esgf_native
6064
else:
6165
esgf = ""
6266
return esgf
@@ -92,7 +96,21 @@ def __init__(self, simulation_dict):
9296
self.experiment = simulation_dict["experiment"]
9397

9498
self.ensemble_num = simulation_dict["ensemble_num"]
95-
self.cmip_only = simulation_dict["cmip_only"]
99+
self.link_type = simulation_dict["link_type"]
100+
101+
if "hpss_path" in simulation_dict:
102+
# If `hpss_path` is specified, then it's a non-standard path
103+
hpss_path = simulation_dict["hpss_path"]
104+
else:
105+
# TODO: update the paths in the v1 csv; do the new paths match this?
106+
# LR/repaired: Do these need to be in a subdir at all???
107+
# 20180129.DECKv1b_piControl.ne30_oEC.edison/
108+
# 20180215.DECKv1b_1pctCO2.ne30_oEC.edison/
109+
# 20180215.DECKv1b_abrupt4xCO2.ne30_oEC.edison/
110+
# 20180215.DECKv1b_H1.ne30_oEC.edison/
111+
# 20180220.DECKv1b_H2.ne30_oEC.edison/
112+
# 20180302.DECKv1b_H3.ne30_oEC.edison/
113+
hpss_path = f"/home/projects/e3sm/www/{self.group}/E3SM{self.model_version}/{self.resolution}/{self.simulation_name}"
96114
if "node" in simulation_dict.keys():
97115
self.node = simulation_dict["node"]
98116
else:
@@ -114,7 +132,7 @@ def __init__(self, simulation_dict):
114132
source_id = f"E3SM-{self.model_version[1]}-0"
115133
else:
116134
raise RuntimeError(f"Invalid model-version={self.model_version}")
117-
self.esgf = get_esgf(source_id, self.model_version, self.experiment, self.ensemble_num, self.cmip_only, self.node)
135+
self.esgf = get_esgf(source_id, self.model_version, self.experiment, self.ensemble_num, self.link_type, self.node)
118136

119137
self.run_script_original = get_run_script_original(self.model_version, self.simulation_name)
120138
self.run_script_reproduction = get_run_script_reproduction(self.model_version, self.simulation_name)
@@ -167,6 +185,7 @@ def append(self, group):
167185
self.groups.update([(group.name, group)])
168186

169187
# Construct simulations ###########################################
188+
170189
def read_simulations(csv_file):
171190
# model_version > group > resolution > category > simulation_name,
172191
versions: OrderedDict[str: ModelVersion] = OrderedDict()
@@ -185,6 +204,8 @@ def read_simulations(csv_file):
185204
if len(row) != len(header):
186205
raise RuntimeError(f"header has {len(header)} labels, but row has {len(row)} entries")
187206
simulation_dict[label] = row[i].strip()
207+
if "cmip_only" in simulation_dict:
208+
simulation_dict["link_type"] = "cmip"
188209
model_version_name = simulation_dict["model_version"]
189210
group_name = simulation_dict["group"]
190211
resolution_name = simulation_dict["resolution"]
@@ -260,7 +281,7 @@ def construct_pages(csv_file: str, model_version: str, group_name: str, include_
260281
resolutions,
261282
["Simulation", "Data Size (TB)", "ESGF Links", "HPSS Path"],
262283
f"../docs/source/{model_version}/{group_name}/simulation_data/simulation_table.rst",
263-
[65, 15, 400, 80]
284+
[65, 15, 400, 90]
264285
)
265286
if include_reproduction_scripts:
266287
generate_table(
@@ -277,4 +298,5 @@ def construct_pages(csv_file: str, model_version: str, group_name: str, include_
277298
if __name__ == "__main__":
278299
#construct_pages("simulations_v2.csv", "v2", "WaterCycle")
279300
#construct_pages("simulations_v2_1.csv", "v2.1", "WaterCycle")
280-
construct_pages("simulations_v2_1.csv", "v2.1", "BGC")
301+
#construct_pages("simulations_v2_1.csv", "v2.1", "BGC")
302+
construct_pages("simulations_v1_water_cycle.csv", "v1", "WaterCycle")

‎utils/simulations_v1_water_cycle.csv‎

Lines changed: 54 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,54 @@
1-
model_version, group, resolution, hpss_path, checksum, ensemble_num, link_type,
2-
v1, WaterCycle, LR, /home/g/golaz/2018/E3SM_simulations/repaired/20180129.DECKv1b_piControl.ne30_oEC.edison, , 1, ,
3-
v1, WaterCycle, LR, /home/g/golaz/2018/E3SM_simulations/repaired/20180215.DECKv1b_abrupt4xCO2.ne30_oEC.edison, , 1, ,
4-
v1, WaterCycle, LR, /home/g/golaz/2019/E3SM_simulations/20190722.DECKv1b_abrupt4xCO2.ne30_oEC3.compy, , 1, ,
5-
v1, WaterCycle, LR, /home/g/golaz/2018/E3SM_simulations/repaired/20180215.DECKv1b_1pctCO2.ne30_oEC.edison, , 1, ,
6-
v1, WaterCycle, LR, /home/g/golaz/2018/E3SM_simulations/repaired/20180215.DECKv1b_H1.ne30_oEC.edison, , 1, ,
7-
v1, WaterCycle, LR, /home/g/golaz/2018/E3SM_simulations/repaired/20180220.DECKv1b_H2.ne30_oEC.edison, , 2, ,
8-
v1, WaterCycle, LR, /home/z/zshaheen/2018/E3SM_simulations/repaired/20180302.DECKv1b_H3.ne30_oEC.edison, , 3, ,
9-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180305.DECKv1b_H4.ne30_oEC.edison, , 4, ,
10-
v1, WaterCycle, LR, /home/g/golaz/2018/E3SM_simulations/20180307.DECKv1b_H5.ne30_oEC.edison, , 5, ,
11-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180316.DECKv1b_A1.ne30_oEC.edison, , 1, ,
12-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180622.DECKv1b_A2.ne30_oEC.edison, , 2, ,
13-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180716.DECKv1b_A3.ne30_oEC.edison, , 3, ,
14-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180508.DECKv1b_A1_1850allF.ne30_oEC.edison, , , ,
15-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180622.DECKv1b_A2_1850allF.ne30_oEC.edison, , , ,
16-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180716.DECKv1b_A3_1850allF.ne30_oEC.edison, , , ,
17-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180508.DECKv1b_A1_1850aeroF.ne30_oEC.edison, , , ,
18-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180622.DECKv1b_A2_1850aeroF.ne30_oEC.edison, , , ,
19-
v1, WaterCycle, LR, /home/t/tang30/2018/E3SM_simulations/20180716.DECKv1b_A3_1850aeroF.ne30_oEC.edison, , , ,
20-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20190404.DECKv1b_H1_hist-GHG.ne30_oEC.edison, , , ,
21-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20190404.DECKv1b_H2_hist-GHG.ne30_oEC.edison, , , ,
22-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20190404.DECKv1b_H3_hist-GHG.ne30_oEC.edison, , , ,
23-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20190807.DECKv1b_P1_SSP5-8.5.ne30_oEC.cori-knl, , , ,
24-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20190821.DECKv1b_P2_SSP5-8.5.ne30_oEC.cori-knl, , , ,
25-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20190821.DECKv1b_P3_SSP5-8.5.ne30_oEC.cori-knl, , , ,
26-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20190821.DECKv1b_P4_SSP5-8.5.ne30_oEC.cori-knl, , , ,
27-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20190821.DECKv1b_P5_SSP5-8.5.ne30_oEC.cori-knl, , , ,
28-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20191019.DECKv1b_P1_SSP5-8.5-GHG.ne30_oEC.cori-knl, , , ,
29-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20191019.DECKv1b_P2_SSP5-8.5-GHG.ne30_oEC.cori-knl, , , ,
30-
v1, WaterCycle, LR, /home/t/tang30/2019/E3SM_simulations/20191019.DECKv1b_P3_SSP5-8.5-GHG.ne30_oEC.cori-knl, , , ,
1+
model_version, group, resolution, category, simulation_name, machine, checksum, experiment, ensemble_num, link_type, node,
2+
v1, WaterCycle, LR, DECK, 20180129.DECKv1b_piControl.ne30_oEC.edison, edison, piControl, 1, , ,
3+
v1, WaterCycle, LR, DECK, 20180215.DECKv1b_abrupt4xCO2.ne30_oEC.edison, edison, , abrupt-4xCO2, 1, , ,
4+
v1, WaterCycle, LR, DECK, 20190722.DECKv1b_abrupt4xCO2.ne30_oEC3.compy, compy, , abrupt-4xCO2, 2, , ,
5+
v1, WaterCycle, LR, DECK, 20180215.DECKv1b_1pctCO2.ne30_oEC.edison, edison, 1pctC02, 1, , ,
6+
v1, WaterCycle, LR, DECK, 20180215.DECKv1b_H1.ne30_oEC.edison, edison, , historical, 1, , ,
7+
v1, WaterCycle, LR, DECK, 20180220.DECKv1b_H2.ne30_oEC.edison, edison, , historical, 2, , ,
8+
v1, WaterCycle, LR, DECK, 20180302.DECKv1b_H3.ne30_oEC.edison, edison, , historical, 3, , ,
9+
v1, WaterCycle, LR, DECK, 20180305.DECKv1b_H4.ne30_oEC.edison, edison, , historical, 4, , ,
10+
v1, WaterCycle, LR, DECK, 20180307.DECKv1b_H5.ne30_oEC.edison, edison, , historical, 5, , ,
11+
v1, WaterCycle, LR, DECK, 20180316.DECKv1b_A1.ne30_oEC.edison, edison, , amip, 1, , ,
12+
v1, WaterCycle, LR, DECK, 20180622.DECKv1b_A2.ne30_oEC.edison, edison, , amip, 2, , ,
13+
v1, WaterCycle, LR, DECK, 20180716.DECKv1b_A3.ne30_oEC.edison, edison, , amip, 3, ,
14+
v1, WaterCycle, LR, DECK, 20180508.DECKv1b_A1_1850allF.ne30_oEC.edison, edison, , amip_1850allF, 1, , ,
15+
v1, WaterCycle, LR, DECK, 20180622.DECKv1b_A2_1850allF.ne30_oEC.edison, edison, , amip_1850allF, 2, , ,
16+
v1, WaterCycle, LR, DECK, 20180716.DECKv1b_A3_1850allF.ne30_oEC.edison, edison, , amip_1850allF, 3, , ,
17+
v1, WaterCycle, LR, DECK, 20180508.DECKv1b_A1_1850aeroF.ne30_oEC.edison, edison, , amip_1850aeroF, 1, , ,
18+
v1, WaterCycle, LR, DECK, 20180622.DECKv1b_A2_1850aeroF.ne30_oEC.edison, edison, , amip_1850aeroF, 2, , ,
19+
v1, WaterCycle, LR, DECK, 20180716.DECKv1b_A3_1850aeroF.ne30_oEC.edison, edison, , amip_1850aeroF, 3, , ,
20+
v1, WaterCycle, LR, DECK, 20190404.DECKv1b_H1_hist-GHG.ne30_oEC.edison, edison, , damip_hist-GHG, 1, , ,
21+
v1, WaterCycle, LR, DECK, 20190404.DECKv1b_H2_hist-GHG.ne30_oEC.edison, edison, , damip_hist-GHG, 2, , ,
22+
v1, WaterCycle, LR, DECK, 20190404.DECKv1b_H3_hist-GHG.ne30_oEC.edison, edison, , damip_hist-GHG, 3, , ,
23+
v1, WaterCycle, LR, DECK, 20190807.DECKv1b_P1_SSP5-8.5.ne30_oEC.cori-knl, cori-knl, , projection, 1, , ,
24+
v1, WaterCycle, LR, DECK, 20190821.DECKv1b_P2_SSP5-8.5.ne30_oEC.cori-knl, cori-knl, , projection, 2, , ,
25+
v1, WaterCycle, LR, DECK, 20190821.DECKv1b_P3_SSP5-8.5.ne30_oEC.cori-knl, cori-knl, , projection, 3, , ,
26+
v1, WaterCycle, LR, DECK, 20190821.DECKv1b_P4_SSP5-8.5.ne30_oEC.cori-knl, cori-knl, , projection, 4, , ,
27+
v1, WaterCycle, LR, DECK, 20190821.DECKv1b_P5_SSP5-8.5.ne30_oEC.cori-knl, cori-knl, , projection, 5, , ,
28+
v1, WaterCycle, LR, DECK, 20191019.DECKv1b_P1_SSP5-8.5-GHG.ne30_oEC.cori-knl, cori-knl, , damip_ssp5-8.5-GHG, 1, , ,
29+
v1, WaterCycle, LR, DECK, 20191019.DECKv1b_P2_SSP5-8.5-GHG.ne30_oEC.cori-knl, cori-knl, , damip_ssp5-8.5-GHG, 2, , ,
30+
v1, WaterCycle, LR, DECK, 20191019.DECKv1b_P3_SSP5-8.5-GHG.ne30_oEC.cori-knl, cori-knl, , damip_ssp5-8.5-GHG, 3, , ,
31+
32+
33+
v1, WaterCycle, HR, DECK, 20211021-maint-1.0-tro.A_WCYCLSSP585_CMIP6_HR.ne120_oRRS18v3_ICG.unc12-3rd-attempt, ???, , ssp5_8.5, 1, , ,
34+
v1, WaterCycle, HR, DECK, 20200517-maint-1.0-tro.A_WCYCL20TRS_CMIP6_HR.ne120_oRRS18v3_ICG.unc11, ???, , , , , ,
35+
v1, WaterCycle, HR, DECK, 202101027-maint-1.0-tro.A_WCYCL20TRS_CMIP6_HR.ne120_oRRS18v3_ICG.unc12, ???, , , , , ,
36+
v1, WaterCycle, HR, DECK, theta.20190910.branch_noCNT.n825def.unc06.A_WCYCL1950S_CMIP6_HR.ne120_oRRS18v3_ICG, , , , , ,
37+
v1, WaterCycle, HR, DECK, 20210112.A_WCYCL1950S_CMIP6_HR.ne120_oRRS18v3_ICG.unc06, ???, , , , , ,
38+
v1, WaterCycle, HR, DECK, 20210104.maint-1.0-A_WCYCL20TRS_CMIP6_LRtunedHR.ne30_oECv3_ICG.cori-knl.unc11, ???, , , , , ,
39+
v1, WaterCycle, HR, DECK, theta.20180906.branch_noCNT.A_WCYCL1950S_CMIP6_HR.ne120_oRRS18v3_ICG, theta, , , , , ,
40+
v1, WaterCycle, HR, DECK, 20190509.A_WCYCL1950S_CMIP6_LRtunedHR.ne30_oECv3_ICG.anvil, anvil, , , , , ,
41+
v1, WaterCycle, HR, DECK, theta.20190910.branch_noCNT.n438b.unc03.A_WCYCL1950S_CMIP6_HR.ne120_oRRS18v3_ICG, theta, , , , , ,
42+
v1, WaterCycle, HR, DECK, cori-knl.20190214_maint-1.0.F2010-CMIP6-HR.dailySST.noCNT.ne120_oRRS18v3, cori-knl, , , , , ,
43+
v1, WaterCycle, HR, DECK, cori-knl.20190214_maint-1.0.F2010C5-CMIP6-HR.ARE.nudgeUV.1850aero.ne120_oRRS18v3, cori-knl, , , , , ,
44+
v1, WaterCycle, HR, DECK, cori-knl.20190214_maint-1.0.F2010-CMIP6-HR.noCNT.ARE.nudgeUV.ne120_oRRS18v3, cori-knl, , , , , ,
45+
v1, WaterCycle, HR, DECK, cori-knl.20190214_maint-1.0.F2010-CMIP6-HR.dailySSTplus4K.noCNT.ne120_oRRS18v3, cori-knl, , , , , ,
46+
v1, WaterCycle, HR, DECK, cori-haswell.20190513.F2010C5-CMIP6-LR.ne30_oECv3, cori-haswell, , , , , ,
47+
v1, WaterCycle, HR, DECK, edison.20190415_maint-1.0.F2010C5-CMIP6-LR.ARE.nudgeUV.ne30_oECv3, edison, , , , , ,
48+
v1, WaterCycle, HR, DECK, edison.20190415_maint-1.0.F2010C5-CMIP6-LR.ARE.nudgeUV.1850aero.ne30_oECv3, edison, , , , , ,
49+
v1, WaterCycle, HR, DECK, cori-haswell.20190513.F2010C5-CMIP6-LR.plus4K.ne30_oECv3, cori-haswell, , , , , ,
50+
v1, WaterCycle, HR, DECK, cori-haswell.20190513.F2010C5-CMIP6-LR.plus4K.ne30_oECv3, cori-haswell, , , , , ,
51+
v1, WaterCycle, HR, DECK, cori-haswell.20190513.F2010LRtunedHR.noCNT.ne30_oECv3, cori-haswell, , , , , ,
52+
v1, WaterCycle, HR, DECK, cori-haswell.20190513.F2010LRtunedHR.noCNT.ARE.nudgeUV.ne30_oECv3, cori-haswell, , , , , ,
53+
v1, WaterCycle, HR, DECK, cori-haswell.20190513.F2010LRtunedHR.noCNT.ARE.nudgeUV.1850aero.ne30_oECv3, cori-haswell, , , , , ,
54+
v1, WaterCycle, HR, DECK, cori-haswell.20190513.F2010LRtunedHR.plus4K.noCNT.ne30_oECv3, cori-haswell, , , , , ,

0 commit comments

Comments
 (0)