diff --git a/docs/source/index.rst b/docs/source/index.rst index f090220..59855f7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,10 +16,12 @@ simulations. :caption: Contents: v2/index + v2.1/index SCREAMv0/index SCREAMv1/index + .. Indices and tables .. ================== .. diff --git a/docs/source/v2.1/WaterCycle/index.rst b/docs/source/v2.1/WaterCycle/index.rst new file mode 100644 index 0000000..223e3ef --- /dev/null +++ b/docs/source/v2.1/WaterCycle/index.rst @@ -0,0 +1,17 @@ +E3SMv2.1 (Water Cycle) +==================== + +The simulation campaign for E3SMv2.1 was performed with Low Resolution (LR) configuration + +* **v2_1.LR** (lower resolution) + +If you use data from this simulation campaign, please cite the following overview +manuscript: + +* `Smith et al. 2024 (GMD) `_ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + simulation_data/index diff --git a/docs/source/v2.1/WaterCycle/simulation_data/index.rst b/docs/source/v2.1/WaterCycle/simulation_data/index.rst new file mode 100644 index 0000000..515d974 --- /dev/null +++ b/docs/source/v2.1/WaterCycle/simulation_data/index.rst @@ -0,0 +1,56 @@ +*************** +Simulation Data +*************** + +The E3SMv2.1 simulation data is available on **ESGF** and **NERSC HPSS**. + +The preferred retrieval method is **ESGF**. Native output is available at `ESGF `_, and a subset of the data is reformatted to conform to CMIP conventions and submmited to the CMIP6 ESGF archive (ESGF links are provided in the table below for published data). + +Additionally, all native model output data has also been archived on **NERSC HPSS** using `zstash `_. + +**If you have an account on NERSC**, you can retrieve the data locally or remotely using Globus. + +To download simulation data locally on a NERSC machine: :: + + zstash extract --hpss= + +To download simulation data remotely using the zstash Globus interface: :: + + zstash extract --hpss=globus://nersc/ + +or :: + + zstash extract --hpss=globus://9cd89cfd-6d04-11e5-ba46-22000b92c6ec/ + +Note that the data management tool `zstash `_ is available from the `E3SM-Unified `_ conda environment. An example of retrieving all **eam.h0** (monthly atmosphere output files) between **years 0030 and 0049** for the v2_1.LR.piControl simulation at NERSC locally is demonstrated as below in two steps: + +1. To activate E3SM-Unified environment by: + :: + + source /global/common/software/e3sm/anaconda_envs/load_latest_e3sm_unified_pm-cpu.sh + +2. To retrieve files with zstash command: + :: + + zstash extract --hpss=/home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.piControl "*.eam.h0.00[3-4]?-??.nc" + + +For more information, refer to `zstash usage `_. + + +**If you do not have access to NERSC**, you can download simulation data directly through the NERSC HPSS +`web interface `_. +Note that this will be slow and inefficient since you'll have to download the tar files. + +**v2_1.LR** simulations data has been archived on NERSC HPSS under: :: + + /home/projects/e3sm/www/WaterCycle/E3SMv2_1 + + +Original run scripts can be found `here `_. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + simulation_table \ No newline at end of file diff --git a/docs/source/v2.1/WaterCycle/simulation_data/simulation_table.rst b/docs/source/v2.1/WaterCycle/simulation_data/simulation_table.rst new file mode 100644 index 0000000..36d0f54 --- /dev/null +++ b/docs/source/v2.1/WaterCycle/simulation_data/simulation_table.rst @@ -0,0 +1,29 @@ +********************************** +v2.1 WaterCycle simulation table +********************************** + ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| Simulation | Data Size (TB) | ESGF Links | HPSS Path | ++===================================================================+=================+==================================================================================================================================================================================================================================================================================================================================================================================================================+==================================================================================+ +| **LR > DECK** | | | | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.piControl | 45 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.piControl | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.1pctCO2_0101 | 14 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.1pctCO2_0101 | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.abrupt-4xCO2_0101 | 14 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.abrupt-4xCO2_0101 | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.amip_0101 | 1 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.amip_0101 | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| **LR > Historical** | | | | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.historical_0101 | 14 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.historical_0101 | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.historical_0151 | 15 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.historical_0151 | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.historical_0201 | 15 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.historical_0201 | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.historical_0251 | 15 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.historical_0251 | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ +| v2_1.LR.historical_0301 | 15 | `CMIP `_ | /home/projects/e3sm/www/WaterCycle/E3SMv2_1/v2_1.LR.historical_0301 | ++-------------------------------------------------------------------+-----------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------+ diff --git a/docs/source/v2.1/index.rst b/docs/source/v2.1/index.rst new file mode 100644 index 0000000..5eb2681 --- /dev/null +++ b/docs/source/v2.1/index.rst @@ -0,0 +1,8 @@ +v2.1 +========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + WaterCycle/index diff --git a/run_scripts/v2.1/original/run.v2_1.LR.1pctCO2_0101.sh b/run_scripts/v2.1/original/run.v2_1.LR.1pctCO2_0101.sh new file mode 100755 index 0000000..8253bd4 --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.1pctCO2_0101.sh @@ -0,0 +1,465 @@ +#!/bin/bash -fe + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="WCYCL1850-1pctCO2" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221103" +readonly BRANCH="8d668b7c015a17608d59d9513e9b425ed8995e26" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME="v2_1.LR.1pctCO2_0101" +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR" + +# Run options +readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="0001-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.1pctCO2_0101/init" +readonly RUN_REFCASE="v2_1.LR.piControl" +readonly RUN_REFDATE="0101-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build +readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'production' for full simulation +readonly run='production' +if [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly PELAYOUT=${layout} + readonly WALLTIME="00:20:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly PELAYOUT="ML" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="50" + readonly REST_OPTION="nyears" + readonly REST_N="1" + readonly RESUBMIT="2" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=false +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +patch_mpas_streams() { + +echo + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/run_scripts/v2.1/original/run.v2_1.LR.abrupt-4xCO2_0101.sh b/run_scripts/v2.1/original/run.v2_1.LR.abrupt-4xCO2_0101.sh new file mode 100755 index 0000000..66cdf0a --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.abrupt-4xCO2_0101.sh @@ -0,0 +1,465 @@ +#!/bin/bash -fe + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="WCYCL1850-4xCO2" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221103" +readonly BRANCH="8d668b7c015a17608d59d9513e9b425ed8995e26" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME="v2_1.LR.abrupt-4xCO2_0101" +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR" + +# Run options +readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="0001-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.abrupt-4xCO2_0101/init" +readonly RUN_REFCASE="v2_1.LR.piControl" +readonly RUN_REFDATE="0101-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build +readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'production' for full simulation +readonly run='production' +if [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly PELAYOUT=${layout} + readonly WALLTIME="00:20:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly PELAYOUT="M" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="30" + readonly REST_OPTION="nyears" + readonly REST_N="1" + readonly RESUBMIT="4" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=false +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +patch_mpas_streams() { + +echo + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/run_scripts/v2.1/original/run.v2_1.LR.amip_0101.sh b/run_scripts/v2.1/original/run.v2_1.LR.amip_0101.sh new file mode 100755 index 0000000..0922b18 --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.amip_0101.sh @@ -0,0 +1,520 @@ +#!/bin/bash -fe + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="F20TR" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221012" +readonly BRANCH="b02bfec1784525d8f05c9fe7d60cc044ca435c40" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME="v2_1.LR.amip_0101" +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR" + +# Run options +readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="1971-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.amip_0101/init" +readonly RUN_REFCASE="v2_1.LR.historical_0101" +readonly RUN_REFDATE="1971-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build +readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'production' for full simulation +readonly run='production' +#readonly run='custom-10_2x5_ndays' +if [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly PELAYOUT=${layout} + readonly WALLTIME="2:00:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly PELAYOUT="custom-30" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="44" # How often to stop the model, should be a multiple of REST_N + readonly REST_OPTION="nyears" + readonly REST_N="1" # How often to write a restart file + readonly RESUBMIT="0" # Submissions after initial one + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=false +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Custom PE layout +custom_pelayout + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' + +! Override +check_finidat_fsurdat_consistency = .false. + +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +# ===================================== +# Customize MPAS stream files if needed +# ===================================== + +patch_mpas_streams() { + +echo + +} + +# ===================================================== +# Custom PE layout: custom-N where N is number of nodes +# ===================================================== + +custom_pelayout() { + +if [[ ${PELAYOUT} == custom-* ]]; +then + echo $'\n CUSTOMIZE PROCESSOR CONFIGURATION:' + + # Number of cores per node (machine specific) + if [ "${MACHINE}" == "chrysalis" ]; then + ncore=64 + elif [ "${MACHINE}" == "compy" ]; then + ncore=40 + else + echo 'ERROR: MACHINE = '${MACHINE}' is not supported for custom PE layout.' + exit 400 + fi + + # Extract number of nodes + tmp=($(echo ${PELAYOUT} | tr "-" " ")) + nnodes=${tmp[1]} + + # Customize + pushd ${CASE_SCRIPTS_DIR} + ./xmlchange NTASKS=$(( $nnodes * $ncore )) + ./xmlchange NTHRDS=1 + ./xmlchange MAX_MPITASKS_PER_NODE=$ncore + ./xmlchange MAX_TASKS_PER_NODE=$ncore + popd + +fi + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + if [[ ${PELAYOUT} == custom-* ]]; + then + layout="M" # temporary placeholder for create_newcase + else + layout=${PELAYOUT} + + fi + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/run_scripts/v2.1/original/run.v2_1.LR.historical_0101.sh b/run_scripts/v2.1/original/run.v2_1.LR.historical_0101.sh new file mode 100755 index 0000000..beebcd7 --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.historical_0101.sh @@ -0,0 +1,465 @@ +#!/bin/bash -fe + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="WCYCL20TR" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221012" +readonly BRANCH="b02bfec1784525d8f05c9fe7d60cc044ca435c40" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME="v2_1.LR.historical_0101" +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR.histens" + +# Run options +readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="1850-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.historical_0101/init" +readonly RUN_REFCASE="v2_1.LR.piControl" +readonly RUN_REFDATE="0101-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build +readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'production' for full simulation +readonly run='production' +if [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly PELAYOUT=${layout} + readonly WALLTIME="00:20:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly PELAYOUT="ML" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="55" + readonly REST_OPTION="nyears" + readonly REST_N="1" + readonly RESUBMIT="2" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=false +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +patch_mpas_streams() { + +echo + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/run_scripts/v2.1/original/run.v2_1.LR.historical_0151.sh b/run_scripts/v2.1/original/run.v2_1.LR.historical_0151.sh new file mode 100755 index 0000000..e7cc130 --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.historical_0151.sh @@ -0,0 +1,498 @@ +#!/bin/bash -fe + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="WCYCL20TR" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221012" +readonly BRANCH="b02bfec1784525d8f05c9fe7d60cc044ca435c40" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME="v2_1.LR.historical_0151" +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR.histens" + +# Run options +readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="1850-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.historical_0151/init" +readonly RUN_REFCASE="v2_1.LR.piControl" +readonly RUN_REFDATE="0151-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'rerun' for rerunning a subset, +# or 'production' for full simulation +readonly run='production' +if [ "${run}" == "rerun" ]; then + + readonly restart_year="0241" + readonly restart_files=${restart_year}-01-01-00000 + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/reruns/${restart_year}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/reruns/${restart_year}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/reruns/${restart_year}/archive + readonly PELAYOUT="L" + readonly WALLTIME="10:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="10" + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=1 # or ${STOP_N} + readonly RESUBMIT=0 + readonly BFBFLAG="FALSE" + readonly DO_SHORT_TERM_ARCHIVING=false + + # Prepare initial conditions files + echo $'\n----- Copying initial conditions for rerun -----\n' + mkdir -p ${CASE_RUN_DIR} + pushd ${CASE_RUN_DIR} + cp -a ${CASE_ROOT}/archive/rest/${restart_files}/* . + popd + +elif [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/tests/${run}/archive + readonly PELAYOUT=${layout} + readonly WALLTIME="00:20:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + readonly PELAYOUT="M" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="33" + readonly REST_OPTION="nyears" + readonly REST_N="1" + readonly RESUBMIT="4" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=false +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV', 'so4_c1AQH2SO4', 'so4_c2AQH2SO4', 'so4_c3AQH2SO4', 'so4_c1AQSO4', 'so4_c2AQSO4', 'so4_c3AQSO4', 'so4_a2_sfnnuc1' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' + !! for aerosol forcing decomposition + rad_diag_1 = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2', 'A:O3:O3', 'N:N2O:N2O', 'N:CH4:CH4', 'N:CFC11:CFC11', 'N:CFC12:CFC12' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +patch_mpas_streams() { + +echo + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # BFBFLAG for coupler (set explicitly only in special conditions) + if [ ! -z "${BFBFLAG}" ]; then + ./xmlchange BFBFLAG=${BFBFLAG} + fi + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/run_scripts/v2.1/original/run.v2_1.LR.historical_0201.sh b/run_scripts/v2.1/original/run.v2_1.LR.historical_0201.sh new file mode 100755 index 0000000..42efdb3 --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.historical_0201.sh @@ -0,0 +1,498 @@ +#!/bin/bash -fe + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="WCYCL20TR" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221012" +readonly BRANCH="b02bfec1784525d8f05c9fe7d60cc044ca435c40" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME="v2_1.LR.historical_0201" +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR.histens" + +# Run options +readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="1850-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.historical_0201/init" +readonly RUN_REFCASE="v2_1.LR.piControl" +readonly RUN_REFDATE="0201-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'rerun' for rerunning a subset, +# or 'production' for full simulation +readonly run='production' +if [ "${run}" == "rerun" ]; then + + readonly restart_year="0241" + readonly restart_files=${restart_year}-01-01-00000 + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/reruns/${restart_year}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/reruns/${restart_year}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/reruns/${restart_year}/archive + readonly PELAYOUT="L" + readonly WALLTIME="10:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="10" + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=1 # or ${STOP_N} + readonly RESUBMIT=0 + readonly BFBFLAG="FALSE" + readonly DO_SHORT_TERM_ARCHIVING=false + + # Prepare initial conditions files + echo $'\n----- Copying initial conditions for rerun -----\n' + mkdir -p ${CASE_RUN_DIR} + pushd ${CASE_RUN_DIR} + cp -a ${CASE_ROOT}/archive/rest/${restart_files}/* . + popd + +elif [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/tests/${run}/archive + readonly PELAYOUT=${layout} + readonly WALLTIME="00:20:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + readonly PELAYOUT="ML" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="55" + readonly REST_OPTION="nyears" + readonly REST_N="1" + readonly RESUBMIT="2" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=false +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV', 'so4_c1AQH2SO4', 'so4_c2AQH2SO4', 'so4_c3AQH2SO4', 'so4_c1AQSO4', 'so4_c2AQSO4', 'so4_c3AQSO4', 'so4_a2_sfnnuc1' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' + !! for aerosol forcing decomposition + rad_diag_1 = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2', 'A:O3:O3', 'N:N2O:N2O', 'N:CH4:CH4', 'N:CFC11:CFC11', 'N:CFC12:CFC12' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +patch_mpas_streams() { + +echo + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # BFBFLAG for coupler (set explicitly only in special conditions) + if [ ! -z "${BFBFLAG}" ]; then + ./xmlchange BFBFLAG=${BFBFLAG} + fi + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/run_scripts/v2.1/original/run.v2_1.LR.historical_0251.sh b/run_scripts/v2.1/original/run.v2_1.LR.historical_0251.sh new file mode 100755 index 0000000..bf3453e --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.historical_0251.sh @@ -0,0 +1,498 @@ +#!/bin/bash -fe + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="WCYCL20TR" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221012" +readonly BRANCH="b02bfec1784525d8f05c9fe7d60cc044ca435c40" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME="v2_1.LR.historical_0251" +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR.histens" + +# Run options +readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="1850-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.historical_0251/init" +readonly RUN_REFCASE="v2_1.LR.piControl" +readonly RUN_REFDATE="0251-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'rerun' for rerunning a subset, +# or 'production' for full simulation +readonly run='production' +if [ "${run}" == "rerun" ]; then + + readonly restart_year="0241" + readonly restart_files=${restart_year}-01-01-00000 + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/reruns/${restart_year}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/reruns/${restart_year}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/reruns/${restart_year}/archive + readonly PELAYOUT="L" + readonly WALLTIME="10:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="10" + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=1 # or ${STOP_N} + readonly RESUBMIT=0 + readonly BFBFLAG="FALSE" + readonly DO_SHORT_TERM_ARCHIVING=false + + # Prepare initial conditions files + echo $'\n----- Copying initial conditions for rerun -----\n' + mkdir -p ${CASE_RUN_DIR} + pushd ${CASE_RUN_DIR} + cp -a ${CASE_ROOT}/archive/rest/${restart_files}/* . + popd + +elif [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/tests/${run}/archive + readonly PELAYOUT=${layout} + readonly WALLTIME="00:20:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + readonly PELAYOUT="ML" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="55" + readonly REST_OPTION="nyears" + readonly REST_N="1" + readonly RESUBMIT="2" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=false +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV', 'so4_c1AQH2SO4', 'so4_c2AQH2SO4', 'so4_c3AQH2SO4', 'so4_c1AQSO4', 'so4_c2AQSO4', 'so4_c3AQSO4', 'so4_a2_sfnnuc1' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' + !! for aerosol forcing decomposition + rad_diag_1 = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2', 'A:O3:O3', 'N:N2O:N2O', 'N:CH4:CH4', 'N:CFC11:CFC11', 'N:CFC12:CFC12' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +patch_mpas_streams() { + +echo + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # BFBFLAG for coupler (set explicitly only in special conditions) + if [ ! -z "${BFBFLAG}" ]; then + ./xmlchange BFBFLAG=${BFBFLAG} + fi + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/run_scripts/v2.1/original/run.v2_1.LR.historical_0301.sh b/run_scripts/v2.1/original/run.v2_1.LR.historical_0301.sh new file mode 100755 index 0000000..a71529b --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.historical_0301.sh @@ -0,0 +1,498 @@ +#!/bin/bash -fe + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="WCYCL20TR" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221012" +readonly BRANCH="b02bfec1784525d8f05c9fe7d60cc044ca435c40" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME="v2_1.LR.historical_0301" +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR.histens" + +# Run options +readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly START_DATE="1850-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.historical_0301/init" +readonly RUN_REFCASE="v2_1.LR.piControl" +readonly RUN_REFDATE="0301-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'rerun' for rerunning a subset, +# or 'production' for full simulation +readonly run='production' +if [ "${run}" == "rerun" ]; then + + readonly restart_year="0241" + readonly restart_files=${restart_year}-01-01-00000 + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/reruns/${restart_year}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/reruns/${restart_year}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/reruns/${restart_year}/archive + readonly PELAYOUT="L" + readonly WALLTIME="10:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="10" + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=1 # or ${STOP_N} + readonly RESUBMIT=0 + readonly BFBFLAG="FALSE" + readonly DO_SHORT_TERM_ARCHIVING=false + + # Prepare initial conditions files + echo $'\n----- Copying initial conditions for rerun -----\n' + mkdir -p ${CASE_RUN_DIR} + pushd ${CASE_RUN_DIR} + cp -a ${CASE_ROOT}/archive/rest/${restart_files}/* . + popd + +elif [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/tests/${run}/archive + readonly PELAYOUT=${layout} + readonly WALLTIME="00:20:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + readonly PELAYOUT="ML" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="55" + readonly REST_OPTION="nyears" + readonly REST_N="1" + readonly RESUBMIT="2" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=true +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV', 'so4_c1AQH2SO4', 'so4_c2AQH2SO4', 'so4_c3AQH2SO4', 'so4_c1AQSO4', 'so4_c2AQSO4', 'so4_c3AQSO4', 'so4_a2_sfnnuc1' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' + !! for aerosol forcing decomposition + rad_diag_1 = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2', 'A:O3:O3', 'N:N2O:N2O', 'N:CH4:CH4', 'N:CFC11:CFC11', 'N:CFC12:CFC12' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +patch_mpas_streams() { + +echo + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # BFBFLAG for coupler (set explicitly only in special conditions) + if [ ! -z "${BFBFLAG}" ]; then + ./xmlchange BFBFLAG=${BFBFLAG} + fi + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/run_scripts/v2.1/original/run.v2_1.LR.piControl.sh b/run_scripts/v2.1/original/run.v2_1.LR.piControl.sh new file mode 100755 index 0000000..088c99a --- /dev/null +++ b/run_scripts/v2.1/original/run.v2_1.LR.piControl.sh @@ -0,0 +1,498 @@ +#!/bin/bash -e + +# E3SM Water Cycle v2.1 run_e3sm script template. +# +# Bash coding style inspired by: +# http://kfirlavi.herokuapp.com/blog/2012/11/14/defensive-bash-programming + +main() { + +# For debugging, uncomment line below +#set -x + +# --- Configuration flags ---- + +# Machine and project +readonly MACHINE=chrysalis +readonly PROJECT="e3sm" + +# Simulation +readonly COMPSET="WCYCL1850" +readonly RESOLUTION="ne30pg2_EC30to60E2r2" +# Code and compilation +readonly CHECKOUT="20221012" +readonly BRANCH="b02bfec1784525d8f05c9fe7d60cc044ca435c40" +readonly CHERRY=( ) +readonly DEBUG_COMPILE=false + +# BEFORE RUNNING : CHANGE the following CASE_NAME to desired value +readonly CASE_NAME=v2_1.LR.piControl +# If this is part of a simulation campaign, ask your group lead about using a case_group label +readonly CASE_GROUP="v2_1.LR" + +# Run options +#readonly MODEL_START_TYPE="hybrid" # 'initial', 'continue', 'branch', 'hybrid' +readonly MODEL_START_TYPE="continue" # for reruns +readonly START_DATE="0001-01-01" + +# Additional options for 'branch' and 'hybrid' +readonly GET_REFCASE=TRUE +readonly RUN_REFDIR="/lcrc/group/e3sm/ac.golaz/E3SMv2_1/v2_1.LR.piControl/init" +readonly RUN_REFCASE="20221012.submeso.piControl.ne30pg2_EC30to60E2r2.chrysalis" +readonly RUN_REFDATE="0301-01-01" # same as MODEL_START_DATE for 'branch', can be different for 'hybrid' + +# Set paths +readonly CODE_ROOT="${HOME}/E3SMv2_1/code/${CHECKOUT}" +readonly CASE_ROOT="/lcrc/group/e3sm/${USER}/E3SMv2_1/${CASE_NAME}" + +# Sub-directories +readonly CASE_BUILD_DIR=${CASE_ROOT}/build + +# Define type of run +# short tests: 'XS_2x5_ndays', 'XS_1x10_ndays', 'S_1x10_ndays', +# 'M_1x10_ndays', 'ML_1x10_ndays', 'L_1x10_ndays' +# or 'rerun' for rerunning a subset, +# or 'production' for full simulation +#readonly run='production' +readonly run='rerun' +if [ "${run}" == "rerun" ]; then + + readonly restart_year="0241" + readonly restart_files=${restart_year}-01-01-00000 + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/reruns/${restart_year}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/reruns/${restart_year}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/reruns/${restart_year}/archive + readonly PELAYOUT="L" + readonly WALLTIME="10:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="10" + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=1 # or ${STOP_N} + readonly RESUBMIT=0 + readonly BFBFLAG="FALSE" + readonly DO_SHORT_TERM_ARCHIVING=false + + # Prepare initial conditions files + echo $'\n----- Copying initial conditions for rerun -----\n' + mkdir -p ${CASE_RUN_DIR} + pushd ${CASE_RUN_DIR} + cp -a ${CASE_ROOT}/archive/rest/${restart_files}/* . + popd + +elif [ "${run}" != "production" ]; then + + # Short test simulations + tmp=($(echo $run | tr "_" " ")) + layout=${tmp[0]} + units=${tmp[2]} + resubmit=$(( ${tmp[1]%%x*} -1 )) + length=${tmp[1]##*x} + + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/tests/${run}/archive + readonly PELAYOUT=${layout} + readonly WALLTIME="00:20:00" + readonly STOP_OPTION=${units} + readonly STOP_N=${length} + readonly REST_OPTION=${STOP_OPTION} + readonly REST_N=${STOP_N} + readonly RESUBMIT=${resubmit} + readonly DO_SHORT_TERM_ARCHIVING=false + +else + + # Production simulation + readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts + readonly CASE_RUN_DIR=${CASE_ROOT}/run + readonly CASE_ARCHIVE_DIR=${CASE_ROOT}/archive + readonly PELAYOUT="L" + readonly WALLTIME="48:00:00" + readonly STOP_OPTION="nyears" + readonly STOP_N="75" + readonly REST_OPTION="nyears" + readonly REST_N="1" + readonly RESUBMIT="6" + readonly DO_SHORT_TERM_ARCHIVING=false +fi + +# Coupler history +readonly HIST_OPTION="nyears" +readonly HIST_N="1" + +# Leave empty (unless you understand what it does) +readonly OLD_EXECUTABLE="" + +# --- Toggle flags for what to do ---- +do_fetch_code=false +do_create_newcase=true +do_case_setup=true +do_case_build=false +do_case_submit=true + +# --- Now, do the work --- + +# Make directories created by this script world-readable +umask 022 + +# Fetch code from Github +fetch_code + +# Create case +create_newcase + +# Setup +case_setup + +# Build +case_build + +# Configure runtime options +runtime_options + +# Copy script into case_script directory for provenance +copy_script + +# Submit +case_submit + +# All done +echo $'\n----- All done -----\n' + +} + +# ======================= +# Custom user_nl settings +# ======================= + +user_nl() { + +cat << EOF >> user_nl_eam + nhtfrq = 0,-24,-6,-6,-3,-24,0 + mfilt = 1,30,120,120,240,30,1 + avgflag_pertape = 'A','A','I','A','A','A','I' + fexcl1 = 'CFAD_SR532_CAL', 'LINOZ_DO3', 'LINOZ_DO3_PSC', 'LINOZ_O3CLIM', 'LINOZ_O3COL', 'LINOZ_SSO3', 'hstobie_linoz' + fincl1 = 'extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL', 'TREFMNAV', 'TREFMXAV' + fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ','TOZ', 'FLDS', 'FLNS', 'FSDS', 'FSNS', 'SHFLX', 'LHFLX', 'TGCLDCWP', 'TGCLDIWP', 'TGCLDLWP', 'CLDTOT', 'T250', 'T200', 'T150', 'T100', 'T050', 'T025', 'T010', 'T005', 'T002', 'T001', 'TTOP', 'U250', 'U150', 'U100', 'U050', 'U025', 'U010', 'U005', 'U002', 'U001', 'UTOP', 'FSNT', 'FLNT' + fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT', 'Z700', 'TBOT:M' + fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500' + fincl5 = 'PRECT','PRECC','TUQ','TVQ','QFLX','SHFLX','U90M','V90M' + fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN' + fincl7 = 'O3', 'PS', 'TROP_P' +EOF + +cat << EOF >> user_nl_elm + hist_dov2xy = .true.,.true. + hist_fincl2 = 'H2OSNO', 'FSNO', 'QRUNOFF', 'QSNOMELT', 'FSNO_EFF', 'SNORDSL', 'SNOW', 'FSDS', 'FSR', 'FLDS', 'FIRE', 'FIRA' + hist_mfilt = 1,365 + hist_nhtfrq = 0,-24 + hist_avgflag_pertape = 'A','A' +EOF + +cat << EOF >> user_nl_mosart + rtmhist_fincl2 = 'RIVER_DISCHARGE_OVER_LAND_LIQ' + rtmhist_mfilt = 1,365 + rtmhist_ndens = 2 + rtmhist_nhtfrq = 0,-24 +EOF + +} + +patch_mpas_streams() { + +echo + +} + +###################################################### +### Most users won't need to change anything below ### +###################################################### + +#----------------------------------------------------- +fetch_code() { + + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + + echo $'\n----- Starting fetch_code -----\n' + local path=${CODE_ROOT} + local repo=e3sm + + echo "Cloning $repo repository branch $BRANCH under $path" + if [ -d "${path}" ]; then + echo "ERROR: Directory already exists. Not overwriting" + exit 20 + fi + mkdir -p ${path} + pushd ${path} + + # This will put repository, with all code + git clone git@github.com:E3SM-Project/${repo}.git . + + # Setup git hooks + rm -rf .git/hooks + git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks + git config commit.template .git/hooks/commit.template + + # Check out desired branch + git checkout ${BRANCH} + + # Custom addition + if [ "${CHERRY}" != "" ]; then + echo ----- WARNING: adding git cherry-pick ----- + for commit in "${CHERRY[@]}" + do + echo ${commit} + git cherry-pick ${commit} + done + echo ------------------------------------------- + fi + + # Bring in all submodule components + git submodule update --init --recursive + + popd +} + +#----------------------------------------------------- +create_newcase() { + + if [ "${do_create_newcase,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + echo $'\n----- Starting create_newcase -----\n' + + # Base arguments + args=" --case ${CASE_NAME} \ + --output-root ${CASE_ROOT} \ + --script-root ${CASE_SCRIPTS_DIR} \ + --handle-preexisting-dirs u \ + --compset ${COMPSET} \ + --res ${RESOLUTION} \ + --machine ${MACHINE} \ + --walltime ${WALLTIME} \ + --pecount ${PELAYOUT}" + + # Oprional arguments + if [ ! -z "${PROJECT}" ]; then + args="${args} --project ${PROJECT}" + fi + if [ ! -z "${CASE_GROUP}" ]; then + args="${args} --case-group ${CASE_GROUP}" + fi + if [ ! -z "${QUEUE}" ]; then + args="${args} --queue ${QUEUE}" + fi + + ${CODE_ROOT}/cime/scripts/create_newcase ${args} + + if [ $? != 0 ]; then + echo $'\nNote: if create_newcase failed because sub-directory already exists:' + echo $' * delete old case_script sub-directory' + echo $' * or set do_newcase=false\n' + exit 35 + fi + +} + +#----------------------------------------------------- +case_setup() { + + if [ "${do_case_setup,,}" != "true" ]; then + echo $'\n----- Skipping case_setup -----\n' + return + fi + + echo $'\n----- Starting case_setup -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Setup some CIME directories + ./xmlchange EXEROOT=${CASE_BUILD_DIR} + ./xmlchange RUNDIR=${CASE_RUN_DIR} + + # Short term archiving + ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} + ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} + + # Build with COSP, except for a data atmosphere (datm) + if [ `./xmlquery --value COMP_ATM` == "datm" ]; then + echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' + else + echo $'\nConfiguring E3SM to use the COSP simulator\n' + ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' + fi + + # Extracts input_data_dir in case it is needed for user edits to the namelist later + local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` + + # Custom user_nl + user_nl + + # Finally, run CIME case.setup + ./case.setup --reset + + popd +} + +#----------------------------------------------------- +case_build() { + + pushd ${CASE_SCRIPTS_DIR} + + # do_case_build = false + if [ "${do_case_build,,}" != "true" ]; then + + echo $'\n----- case_build -----\n' + + if [ "${OLD_EXECUTABLE}" == "" ]; then + # Ues previously built executable, make sure it exists + if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then + echo 'Skipping build because $do_case_build = '${do_case_build} + else + echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' + exit 297 + fi + else + # If absolute pathname exists and is executable, reuse pre-exiting executable + if [ -x ${OLD_EXECUTABLE} ]; then + echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} + cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ + else + echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' + exit 297 + fi + fi + echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' + ./xmlchange BUILD_COMPLETE=TRUE + + # do_case_build = true + else + + echo $'\n----- Starting case_build -----\n' + + # Turn on debug compilation option if requested + if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then + ./xmlchange DEBUG=${DEBUG_COMPILE^^} + fi + + # Run CIME case.build + ./case.build + + # Some user_nl settings won't be updated to *_in files under the run directory + # Call preview_namelists to make sure *_in and user_nl files are consistent. + ./preview_namelists + + fi + + popd +} + +#----------------------------------------------------- +runtime_options() { + + echo $'\n----- Starting runtime_options -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Set simulation start date + ./xmlchange RUN_STARTDATE=${START_DATE} + + # Segment length + ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} + + # Restart frequency + ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} + + # Coupler history + ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} + + # Coupler budgets (always on) + ./xmlchange BUDGETS=TRUE + + # BFBFLAG for coupler (set explicitly only in special conditions) + if [ ! -z "${BFBFLAG}" ]; then + ./xmlchange BFBFLAG=${BFBFLAG} + fi + + # Set resubmissions + if (( RESUBMIT > 0 )); then + ./xmlchange RESUBMIT=${RESUBMIT} + fi + + # Run type + # Start from default of user-specified initial conditions + if [ "${MODEL_START_TYPE,,}" == "initial" ]; then + ./xmlchange RUN_TYPE="startup" + ./xmlchange CONTINUE_RUN="FALSE" + + # Continue existing run + elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then + ./xmlchange CONTINUE_RUN="TRUE" + + elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then + ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} + ./xmlchange GET_REFCASE=${GET_REFCASE} + ./xmlchange RUN_REFDIR=${RUN_REFDIR} + ./xmlchange RUN_REFCASE=${RUN_REFCASE} + ./xmlchange RUN_REFDATE=${RUN_REFDATE} + echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} + echo '$RUN_REFDIR = '${RUN_REFDIR} + echo '$RUN_REFCASE = '${RUN_REFCASE} + echo '$RUN_REFDATE = '${START_DATE} + + else + echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' + exit 380 + fi + + # Patch mpas streams files + patch_mpas_streams + + popd +} + +#----------------------------------------------------- +case_submit() { + + if [ "${do_case_submit,,}" != "true" ]; then + echo $'\n----- Skipping case_submit -----\n' + return + fi + + echo $'\n----- Starting case_submit -----\n' + pushd ${CASE_SCRIPTS_DIR} + + # Run CIME case.submit + ./case.submit + + popd +} + +#----------------------------------------------------- +copy_script() { + + echo $'\n----- Saving run script for provenance -----\n' + + local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance + mkdir -p ${script_provenance_dir} + local this_script_name=`basename $0` + local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` + cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} + +} + +#----------------------------------------------------- +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Now, actually run the script +#----------------------------------------------------- +main diff --git a/utils/generate_html.bash b/utils/generate_html.bash index 1435a8e..de2768b 100755 --- a/utils/generate_html.bash +++ b/utils/generate_html.bash @@ -1,13 +1,13 @@ -pr_num=49 +pr_num=50 # Chrysalis -destination_dir=/lcrc/group/e3sm/public_html/diagnostic_output/$USER/data_docs_${pr_num} -web_page="https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/$USER/data_docs_${pr_num}/html/" +#destination_dir=/lcrc/group/e3sm/public_html/diagnostic_output/$USER/data_docs_${pr_num} +#web_page="https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/$USER/data_docs_${pr_num}/html/" # Perlmutter -#destination_dir=/global/cfs/cdirs/e3sm/www/$USER/data_docs_${pr_num} -#web_page="https://portal.nersc.gov/cfs/e3sm/$USER/data_docs_${pr_num}/html/" +destination_dir=/global/cfs/cdirs/e3sm/www/$USER/data_docs_${pr_num} +web_page="https://portal.nersc.gov/cfs/e3sm/$USER/data_docs_${pr_num}/html/" -#python generate_tables.py +python generate_tables.py cd ../docs/ && make html rm -rf ls ${destination_dir} mv _build ${destination_dir} diff --git a/utils/generate_tables.py b/utils/generate_tables.py index 12d740e..66150a9 100644 --- a/utils/generate_tables.py +++ b/utils/generate_tables.py @@ -11,7 +11,11 @@ def get_data_size_and_hpss(hpss_path: str) -> Tuple[str, str]: output = "out.txt" if os.path.exists(output): os.remove(output) - os.system(f'(hsi "du {hpss_path}") 2>&1 | tee {output}') + try: + os.system(f'(hsi "du {hpss_path}") 2>&1 | tee {output}') + except Exception as e: + print(f"hsi failed: {e}") + return ("", "") num_bytes = "0" with open(output, "r") as f: for line in f: @@ -33,16 +37,18 @@ def get_data_size_and_hpss(hpss_path: str) -> Tuple[str, str]: hpss = "" return (data_size, hpss) -def get_esgf(source_id: str, model_version: str, experiment: str, ensemble_num: str, cmip_only: str) -> str: - if experiment and ensemble_num: +def get_esgf(source_id: str, model_version: str, experiment: str, ensemble_num: str, cmip_only: str, node: str) -> str: + if node == "cels.anl": + esgf = f"`CMIP `_" + elif experiment and ensemble_num: # See https://github.com/E3SM-Project/CMIP6-Metadata/pull/9#issuecomment-1246086256 for the table of ensemble numbers # remove v from model_version - esgf = f"`Native `_" + esgf = f"`Native `_" if experiment == 'hist-all-xGHG-xaer': experiment_id = 'hist-nat' else: experiment_id = experiment - esgf_cmip = f"`CMIP `_" + esgf_cmip = f"`CMIP `_" if cmip_only: esgf = esgf_cmip else: @@ -83,15 +89,28 @@ def __init__(self, simulation_dict): self.ensemble_num = simulation_dict["ensemble_num"] self.cmip_only = simulation_dict["cmip_only"] + if "node" in simulation_dict.keys(): + self.node = simulation_dict["node"] + else: + self.node = "llnl" - hpss_path = f"/home/projects/e3sm/www/{self.group}/E3SM{self.model_version}/{self.resolution}/{self.simulation_name}" + if "." in self.model_version: + displayed_version: str = self.model_version.replace(".", "_") + hpss_path = f"/home/projects/e3sm/www/{self.group}/E3SM{displayed_version}/{self.simulation_name}" + else: + hpss_path = f"/home/projects/e3sm/www/{self.group}/E3SM{self.model_version}/{self.resolution}/{self.simulation_name}" self.data_size, self.hpss = get_data_size_and_hpss(hpss_path) - if self.resolution == "NARRM": - source_id = f"E3SM-{self.model_version[1:]}-0-{self.resolution}" + if (len(self.model_version) == 4) and (self.model_version[2] == "."): + source_id = f"E3SM-{self.model_version[1]}-{self.model_version[3]}" + elif (len (self.model_version) == 2): + if self.resolution == "NARRM": + source_id = f"E3SM-{self.model_version[1]}-0-{self.resolution}" + else: + source_id = f"E3SM-{self.model_version[1]}-0" else: - source_id = f"E3SM-{self.model_version[1:]}-0" - self.esgf = get_esgf(source_id, self.model_version, self.experiment, self.ensemble_num, self.cmip_only) + raise RuntimeError(f"Invalid model-version={self.model_version}") + self.esgf = get_esgf(source_id, self.model_version, self.experiment, self.ensemble_num, self.cmip_only, self.node) self.run_script_original = get_run_script_original(self.model_version, self.simulation_name) self.run_script_reproduction = get_run_script_reproduction(self.model_version, self.simulation_name) @@ -189,14 +208,14 @@ def read_simulations(csv_file): return versions # Construct table display of simulations ########################################### -def pad_cells(cells, col_divider, cell_paddings): +def pad_cells(cells: List[str], col_divider: str, cell_paddings: List[int]) -> str: string = col_divider for i in range(len(cells)): string += " " + cells[i].ljust(cell_paddings[i] + 1) + col_divider string += "\n" return string -def pad_cells_row_dividers(marker, cell_paddings): +def pad_cells_row_dividers(marker: str, cell_paddings: List[int]) -> str: string = "+" for i in range(len(cell_paddings)): string += marker*(cell_paddings[i]+2) + "+" @@ -226,9 +245,10 @@ def generate_table(page_type: str, resolutions: OrderedDict[str, Category], head file_write.write(pad_cells(simulation.get_row(output_file), "|", cell_paddings)) file_write.write(pad_cells_row_dividers("-", cell_paddings)) -def construct_pages(csv_file, model_version, group_name): +def construct_pages(csv_file: str, model_version: str, group_name: str, include_reproduction_scripts: bool = False): versions: OrderedDict[str, ModelVersion] = read_simulations(csv_file) resolutions: OrderedDict[str, Category] = versions[model_version].groups[group_name].resolutions + # TODO: add proper subdirs and index.rst files in docs/ generate_table( f"{model_version} {group_name} simulation table", resolutions, @@ -236,16 +256,18 @@ def construct_pages(csv_file, model_version, group_name): f"../docs/source/{model_version}/{group_name}/simulation_data/simulation_table.rst", [65, 15, 400, 80] ) - generate_table( - f"{model_version} {group_name} reproduction table", - resolutions, - ["Simulation", "Machine", "10 day checksum", "Reproduction Script", "Original Script (requires significant changes to run!!)",], - f"../docs/source/{model_version}/{group_name}/reproducing_simulations/reproduction_table.rst", - # TODO: The script boxes have to be 200 characters wide to fit in the links... - # This is unfortunate because the actual displayed text is quite short. - # https://github.com/E3SM-Project/e3sm_data_docs/issues/30 may fix this. - [65, 11, 34, 200, 200] - ) + if include_reproduction_scripts: + generate_table( + f"{model_version} {group_name} reproduction table", + resolutions, + ["Simulation", "Machine", "10 day checksum", "Reproduction Script", "Original Script (requires significant changes to run!!)",], + f"../docs/source/{model_version}/{group_name}/reproducing_simulations/reproduction_table.rst", + # TODO: The script boxes have to be 200 characters wide to fit in the links... + # This is unfortunate because the actual displayed text is quite short. + # https://github.com/E3SM-Project/e3sm_data_docs/issues/30 may fix this. + [65, 11, 34, 200, 200] + ) if __name__ == "__main__": - construct_pages("simulations.csv", "v2", "WaterCycle") + #construct_pages("simulations_v2.csv", "v2", "WaterCycle") + construct_pages("simulations_v2_1.csv", "v2.1", "WaterCycle") diff --git a/utils/simulations.csv b/utils/simulations_v2.csv similarity index 99% rename from utils/simulations.csv rename to utils/simulations_v2.csv index e580163..acfcf99 100644 --- a/utils/simulations.csv +++ b/utils/simulations_v2.csv @@ -88,5 +88,5 @@ v2, WaterCycle, NARRM, AMIP, v2.NARRM.amip_0101, chrysalis, 930b7fc7e946910c3c8e v2, WaterCycle, NARRM, AMIP, v2.NARRM.amip_0201, chrysalis, a8326dd3922cbf32dccedb494fcedffb, amip, 2, , v2, WaterCycle, NARRM, AMIP, v2.NARRM.amip_0301, chrysalis, f8bcd50a7e9c5ef8253908b73ee7471c, amip, 3, , v2, WaterCycle, NARRM, AMIP, v2.NARRM.amip_0101_bonus, chrysalis, , , , , -v2, WaterCycle, NARRM, Other, v2.NA.F20TR.6h.f1.1850aer, , , , , , , -v2, WaterCycle, NARRM, Other, v2.NA.F20TR.6h.f1.2010aer , , , , , , +v2, WaterCycle, NARRM, Other, v2.NA.F20TR.6h.f1.1850aer, , , , , , +v2, WaterCycle, NARRM, Other, v2.NA.F20TR.6h.f1.2010aer, , , , , , diff --git a/utils/simulations_v2_1.csv b/utils/simulations_v2_1.csv new file mode 100644 index 0000000..5f79324 --- /dev/null +++ b/utils/simulations_v2_1.csv @@ -0,0 +1,10 @@ +model_version, group, resolution, category, simulation_name, machine, checksum, experiment, ensemble_num, cmip_only, node, +v2.1, WaterCycle, LR, DECK, v2_1.LR.piControl, chrysalis, ebc5565e182e8b1cc887c185ca3eb144, piControl, 1, cmip_only, cels.anl, +v2.1, WaterCycle, LR, DECK, v2_1.LR.1pctCO2_0101, chrysalis, b7676ccc791941d1bdadde05c59d70b0, 1pctCO2, 1, cmip_only, cels.anl, +v2.1, WaterCycle, LR, DECK, v2_1.LR.abrupt-4xCO2_0101, chrysalis, fe50274534774678d55cb320412d1667, abrupt-4xCO2, 1, cmip_only, cels.anl, +v2.1, WaterCycle, LR, DECK, v2_1.LR.amip_0101, chrysalis, 832ca9c9db8b4988a109f8c5872b0fe7, amip, 1, cmip_only, cels.anl, +v2.1, WaterCycle, LR, Historical, v2_1.LR.historical_0101, chrysalis, 43efe5ba9216b535be21bd8474b15c22, historical, 1, cmip_only, cels.anl, +v2.1, WaterCycle, LR, Historical, v2_1.LR.historical_0151, chrysalis, , historical, 2, cmip_only, cels.anl, +v2.1, WaterCycle, LR, Historical, v2_1.LR.historical_0201, chrysalis, , historical, 3, cmip_only, cels.anl, +v2.1, WaterCycle, LR, Historical, v2_1.LR.historical_0251, chrysalis, , historical, 4, cmip_only, cels.anl, +v2.1, WaterCycle, LR, Historical, v2_1.LR.historical_0301, chrysalis, , historical, 5, cmip_only, cels.anl,