Skip to content

enable DA cycling on gaea C5/C6 #3255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aerorahul merged 6 commits into from
Jan 24, 2025
Merged

enable DA cycling on gaea C5/C6 #3255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d3a4405
re-baseline
Jan 24, 2025
d4ad639
update hash to include gaeac5/c6 build support
Jan 24, 2025
fe59599
fix typos
Jan 24, 2025
05f65c4
add comment back in
Jan 24, 2025
0a0e56f
fix typo
Jan 24, 2025
5d2227c
revert to develop
Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 206 additions & 19 deletions env/GAEAC5.env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,118 @@ else
exit 2
fi

if [[ "${step}" = "prep" ]]; then
case ${step} in
"prep" | "prepbufr")

export POE="NO"
export BACK="NO"
export BACK=${BACK:-"YES"}
export sys_tp="GAEAC5"
export launcher_PREP="srun"
;;
"prep_emissions")

elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
export APRUN="${APRUN_default}"
;;
"waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")

export CFP_MP="YES"
[[ "${step}" = "waveprep" ]] && export MP_PULSE=0
export wavempexec=${launcher}
export wave_mpmd=${mpmd_opt}

;;
"atmanlvar")

export NTHREADS_ATMANLVAR=${NTHREADSmax}
export APRUN_ATMANLVAR="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLVAR}"
;;
"atmanlfv3inc")

export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
;;
"atmensanlobs")

export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
export APRUN_ATMENSANLOBS="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
;;
"atmensanlsol")

export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
export APRUN_ATMENSANLSOL="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
;;
"atmensanlletkf")

export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
export APRUN_ATMENSANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}"
;;
"atmensanlfv3inc")

export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
export APRUN_ATMENSANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
;;
"aeroanlvar")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

export NTHREADS_AEROANL=${NTHREADSmax}
export APRUN_AEROANL="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANL}"
;;
"aeroanlgenb")

export NTHREADS_AEROANLGENB=${NTHREADSmax}
export APRUN_AEROANLGENB="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANLGENB}"
;;
"prepobsaero")

export NTHREADS_PREPOBSAERO=${NTHREADS1}
export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
;;
"snowanl")

export APRUN_CALCFIMS="${launcher} -n 1"

export NTHREADS_SNOWANL=${NTHREADSmax}
export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}"

export APRUN_APPLY_INCR="${launcher} -n 6"
;;
"esnowrecen")

export NTHREADS_ESNOWRECEN=${NTHREADSmax}
export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"

export APRUN_APPLY_INCR="${launcher} -n 6"
;;

"marinebmat")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
export APRUN_MARINEBMAT="${APRUN_default}"
;;
"marineanlvar")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
export APRUN_MARINEANLVAR="${APRUN_default}"
;;
"ocnanalecen")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

max_threads_per_task=$((max_tasks_per_node / tasks_per_node_ocnanalecen))

export NTHREADS_OCNANALECEN=${threads_per_task_ocnanalecen:-${max_threads_per_task}}
[[ ${NTHREADS_OCNANALECEN} -gt ${max_threads_per_task} ]] && export NTHREADS_OCNANALECEN=${max_threads_per_task}
export APRUN_OCNANALECEN="${launcher} -n ${ntasks_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
;;
"marineanlchkpt")

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

export NTHREADS_OCNANAL=${NTHREADSmax}
export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}"
;;
"anal" | "analcalc")

export MKL_NUM_THREADS=4
export MKL_CBWR=AUTO
Expand All @@ -50,7 +154,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

export NTHREADS_GSI=${NTHREADSmax}
export NTHREADS_GSI=${threads_per_task_anal:-${max_threads_per_task}}
export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"

export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
Expand All @@ -65,44 +169,127 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
export NTHREADS_GAUSFCANL=1
ntasks_gausfcanl=${ntasks_gausfcanl:-1}
export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"

elif [[ "${step}" = "sfcanl" ]]; then
;;
"sfcanl")

export NTHREADS_CYCLE=${threads_per_task:-14}
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
;;
"eobs")

export MKL_NUM_THREADS=4
export MKL_CBWR=AUTO

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"


export NTHREADS_GSI=${NTHREADSmax}
[[ ${NTHREADS_GSI} -gt ${max_threads_per_task} ]] && export NTHREADS_GSI=${max_threads_per_task}
export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"
;;
"eupd")

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"


export NTHREADS_ENKF=${NTHREADSmax}
export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}} --cpus-per-task=${NTHREADS_ENKF}"
;;
"fcst" | "efcs")

elif [[ "${step}" = "fcst" ]]; then
export OMP_STACKSIZE=512M

(( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
(( ufs_ntasks = nnodes*tasks_per_node ))
# With ESMF threading, the model wants to use the full node
export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
unset nnodes ufs_ntasks
;;

elif [[ "${step}" = "upp" ]]; then
"upp")

export NTHREADS_UPP=${NTHREADS1}
export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}"
;;

elif [[ "${step}" = "atmos_products" ]]; then
"atmos_products")

export USE_CFP="YES" # Use MPMD for downstream product generation on GaeaC5
export USE_CFP="YES" # Use MPMD for downstream product generation
;;

elif [[ "${step}" = "oceanice_products" ]]; then
"oceanice_products")

export NTHREADS_OCNICEPOST=${NTHREADS1}
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
;;

elif [[ "${step}" = "fit2obs" ]]; then
"ecen")

export NTHREADS_FIT2OBS=${NTHREADS1}
export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
export NTHREADS_ECEN=${NTHREADSmax}
export APRUN_ECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ECEN}"

export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
[[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
export APRUN_CHGRES="time"

export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
[[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
export APRUN_CALCINC="${APRUN_default} --cpus-per-task=${NTHREADS_CALCINC}"

;;
"esfc")

elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
export NTHREADS_ESFC=${NTHREADSmax}
export APRUN_ESFC="${APRUN_default} --cpus-per-task=${NTHREADS_ESFC}"

export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"

;;
"epos")

export NTHREADS_EPOS=${NTHREADSmax}
export APRUN_EPOS="${APRUN_default} --cpus-per-task=${NTHREADS_EPOS}"

;;
"postsnd")

export CFP_MP="YES"
if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
export wavempexec=${launcher}
export wave_mpmd=${mpmd_opt}

fi
export NTHREADS_POSTSND=${NTHREADS1}
export APRUN_POSTSND="${APRUN_default} --cpus-per-task=${NTHREADS_POSTSND}"

export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
[[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}
export APRUN_POSTSNDCFP="${launcher} -n ${ntasks_postsndcfp} ${mpmd_opt}"

;;
"awips")

export NTHREADS_AWIPS=${NTHREADS1}
export APRUN_AWIPSCFP="${APRUN_default} ${mpmd_opt}"

;;
"gempak")

echo "WARNING: ${step} is not enabled on ${machine}!"

;;
"fit2obs")

export NTHREADS_FIT2OBS=${NTHREADS1}
export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"

;;
*)
# Some other job not yet defined here
echo "WARNING: The job step ${step} does not specify GAEAC5-specific resources"
;;
esac

Loading