fv3_qsub.IN_wcoss2

#!/bin/bash

#PBS -o out
#PBS -e err
#PBS -N @[JBNME]
#PBS -A @[ACCNR]
#PBS -q @[QUEUE]
#PBS -l select=@[NODES]:ncpus=@[NCPUS]:mpiprocs=@[TPN]:ompthreads=@[THRD]
#PBS -l place=vscatter:exclhost
#PBS -l walltime=00:@[WLCLK]:00

set -eux

cd $PBS_O_WORKDIR

echo -n " $( date +%s )," >  job_timestamp.txt

set +x
module use $PWD/modulefiles
module load modules.fv3
module load cray-pals
module load craype-network-ucx
module load cray-mpich-ucx
module list
set -x

echo "Model started:  " `date`

export OMP_NUM_THREADS=@[THRD]

export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
export ESMF_RUNTIME_PROFILE=ON
export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"

export FI_OFI_RXM_RX_SIZE=40000
export FI_OFI_RXM_TX_SIZE=40000
export FI_OFI_RXM_SAR_LIMIT=3145728
export OMP_PLACES=cores
export OMP_STACKSIZE=2048M
export MPICH_MPIIO_HINTS="*:romio_cb_write=enable"

# This "if" block is part of the rt.sh self-tests in error-test.conf. It emulates the model failing to run.
if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then
    echo "The job should abort now, with exit status 1." 1>&2
    echo "If error checking is working, the metascheduler should mark the job as failed." 1>&2
    false
fi

mpiexec -n @[TASKS] -ppn @[TPN] --cpu-bind depth --depth @[THRD] ./fv3.exe

echo "Model ended:    " `date`
echo -n " $( date +%s )," >> job_timestamp.txt