Skip to content

Commit 2b410bb

Browse files
jaelynlitzjaelynlitz
committed
add dashboard scripts and unified compy cron job
1 parent 22354e9 commit 2b410bb

File tree

3 files changed

+318
-0
lines changed

3 files changed

+318
-0
lines changed

‎jenkins/compy_cron.sh‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/bin/bash
2+
3+
main() {
4+
5+
#---------------------------------------------------------------
6+
# User-defined configuration
7+
#---------------------------------------------------------------
8+
#path to E3SM
9+
code_root="/compyfs/litz372/e3sm_scratch/performance_testing/E3SM"
10+
11+
#machine to run the test on
12+
mach="compy"
13+
14+
#compiler
15+
compiler=intel
16+
17+
#set resolution
18+
resolution=ne30pg2_ne30pg2
19+
20+
#set the length of the simulation (ie Ld5 for 5 days or Ln5 for 5 timesteps)
21+
simulation_length=Ld5
22+
23+
#directory for all the past and current data
24+
data_dest="/qfs/projects/eagles/litz372/performance_data/${simulation_length}"
25+
26+
#path where the latest time series plot is saved, accessible to the whole project
27+
share_dest="/compyfs/www/litz372/performance_data"
28+
29+
#url where the plot will be available - based on the share_dest
30+
share_url_ne30="https://compy-dtn.pnl.gov/litz372/performance_data/performance_comp_${resolution}_${simulation_length}.png"
31+
32+
#load modules
33+
source /etc/profile.d/modules.sh
34+
module load python/3.11.5
35+
36+
ulimit -d unlimited
37+
ulimit -s unlimited
38+
ulimit -c unlimited
39+
40+
#---------------------------------------------------------------
41+
# User-defined configuration ENDs
42+
#---------------------------------------------------------------
43+
44+
#---------------------------------------------------------------
45+
# RUN NE30 SIMULATION
46+
#---------------------------------------------------------------
47+
48+
/compyfs/litz372/e3sm_scratch/performance_testing/E3SM_test_scripts/jenkins/mam4xx_compare_performance.sh -r $resolution -c $compiler -t $simulation_length -m $mach -p $code_root -d $data_dest -s $share_dest -u $share_url_ne30
49+
50+
#---------------------------------------------------------------
51+
# RUN NE4 SIMULATION
52+
#---------------------------------------------------------------
53+
54+
resolution=ne4pg2_ne4pg2
55+
share_url_ne4="https://compy-dtn.pnl.gov/litz372/performance_data/performance_comp_${resolution}_${simulation_length}.png"
56+
/compyfs/litz372/e3sm_scratch/performance_testing/E3SM_test_scripts/jenkins/mam4xx_compare_performance.sh -r $resolution -c $compiler -t $simulation_length -m $mach -p $code_root -d $data_dest -s $share_dest -u $share_url_ne4
57+
58+
#---------------------------------------------------------------
59+
# RUN PERFORMANCE BREAKDOWN
60+
#---------------------------------------------------------------
61+
62+
module unload python
63+
module load python/miniconda4.12.0
64+
/compyfs/litz372/e3sm_scratch/compare_model_performance/E3SM_test_scripts/jenkins/compy_model_performance.sh
65+
66+
#---------------------------------------------------------------
67+
# APPEND LINE GRAPHS TO BREAKDOWN
68+
#---------------------------------------------------------------
69+
70+
#plot line in dash comp
71+
source ${code_root}/../.venv/bin/activate
72+
output_graph=${share_dest}/../compare_performance/plot.html
73+
echo $share_url_ne4
74+
echo $share_url_ne30
75+
echo $output_graph
76+
python3 /compyfs/litz372/e3sm_scratch/compare_model_performance/E3SM_test_scripts/jenkins/plot_html.py -i ${share_url_ne4} -o ${output_graph}
77+
python3 /compyfs/litz372/e3sm_scratch/compare_model_performance/E3SM_test_scripts/jenkins/plot_html.py -i ${share_url_ne30} -o ${output_graph}
78+
79+
}
80+
81+
#--------------------------
82+
# Start the script
83+
#--------------------------
84+
main

‎jenkins/compy_model_performance.sh‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
#!/bin/bash
2+
3+
#-------------------------------------------------------------------------------
4+
#-------------------------------------------------------------------------------
5+
# Description:
6+
# This script automates the process of comparing model performance between master and development branches of E3SM.
7+
#
8+
# Usage:
9+
# Customize the user input section below before running the script.
10+
#---------------------------------------------------------------------------
11+
#---------------------------------------------------------------------------
12+
13+
main() {
14+
15+
#---------------------------------------------------------------
16+
# User-defined configuration
17+
#---------------------------------------------------------------
18+
19+
# Perlmutter
20+
# machine=pm-cpu
21+
# compiler=gnu
22+
# project=e3sm
23+
# workdir=/pscratch/sd/m/meng/compare_model_performance
24+
# plotdir=/global/cfs/cdirs/e3sm/www/meng/compare_performance # make sure this a www in your Community directory to check the plot as a html page
25+
# html_address=https://portal.nersc.gov/cfs/e3sm/meng/compare_performance
26+
# module load python
27+
28+
# Compy
29+
machine=compy
30+
compiler=intel
31+
project=e3sm
32+
workdir=/compyfs/litz372/e3sm_scratch/compare_model_performance
33+
plotdir=/compyfs/www/litz372/compare_performance
34+
html_address=https://compy-dtn.pnl.gov/litz372/compare_performance
35+
source /etc/profile.d/modules.sh
36+
module load python/miniconda4.12.0
37+
source /share/apps/python/miniconda4.12.0/etc/profile.d/conda.sh
38+
source ${workdir}/venv/bin/activate
39+
40+
if [ ! -d $plotdir ]; then
41+
mkdir -p $plotdir
42+
fi
43+
44+
compset=F2010-EAMxx-MAM4xx #F2010-SCREAMv1
45+
# resolution=ne4pg2_oQU480
46+
resolution=ne30pg2_ne30pg2
47+
pe=P32
48+
runtime=Ln5
49+
queue=debug
50+
wallclock_time=00:05:00
51+
52+
# SMS test run
53+
case=SMS_$pe_$runtime.$resolution.$compset.${machine}_$compiler
54+
55+
branch1=master
56+
57+
datestr=`date +'%Y%m%d_%H%M%S'`
58+
casename1=master
59+
60+
code_root1=$workdir/E3SM-$casename1
61+
62+
do_fetch_code=true
63+
do_run_case=true
64+
65+
# If you only need to tweak the plot, set the plot_str to the date string of the run.
66+
do_plot=true
67+
#plot_str=20250703_030001
68+
69+
#---------------------------------------------------------------
70+
# User-defined configuration ENDs
71+
#---------------------------------------------------------------
72+
73+
# Fetch code from GitHub
74+
fetch_code $branch1 $code_root1 &
75+
wait
76+
77+
if [ $? != 0 ]; then
78+
echo "Error fetching code from GitHub"
79+
exit 1
80+
fi
81+
82+
# Create SMS test case and run it
83+
run_case $code_root1 $casename1 &
84+
wait
85+
86+
if [ $? != 0 ]; then
87+
echo "Error compiling or running case"
88+
exit 1
89+
fi
90+
91+
# Grab timing data and do the plotting
92+
if [ "${do_plot,,}" != "true" ]; then
93+
echo $'\n----- Skipping plot -----\n'
94+
return
95+
fi
96+
if [ -z "${plot_str+x}" ]; then
97+
plot_str=$datestr
98+
fi
99+
100+
case_root1=$workdir/$case.${casename1}_${plot_str}
101+
102+
python ${workdir}/E3SM_test_scripts/jenkins/mam4xx_compare_model_performance_plot.py \
103+
--case1 $case_root1 --casename1 $casename1 \
104+
--outdir $plotdir --html $html_address
105+
106+
if [ $? != 0 ]; then
107+
echo "Error plotting model performance"
108+
exit 1
109+
fi
110+
111+
echo "Model performance comparison complete!"
112+
113+
}
114+
115+
#---------------------
116+
# Function Definitions
117+
#---------------------
118+
119+
fetch_code() {
120+
if [ "${do_fetch_code,,}" != "true" ]; then
121+
echo $'\n----- Skipping fetch_code -----\n'
122+
return
123+
fi
124+
local branch=$1
125+
local path=$2
126+
127+
echo "Fetching code from $branch branch to $path..."
128+
if [ -d $path ]; then
129+
echo "Code directory $path already exists. Removing it..."
130+
rm -rf $path
131+
fi
132+
133+
mkdir -p $path
134+
pushd $path
135+
git clone [email protected]:E3SM-Project/E3SM.git .
136+
if [ '$branch' != 'master' ]; then
137+
git checkout $branch
138+
fi
139+
git submodule update --init --recursive
140+
popd
141+
}
142+
143+
run_case() {
144+
if [ "${do_run_case,,}" != "true" ]; then
145+
echo $'\n----- Skipping create_newcase -----\n'
146+
return
147+
fi
148+
149+
local code_root=$1
150+
local casename=$2
151+
152+
local interval=60 # seconds to wait between each check
153+
154+
path=$workdir/$case.${casename}_${datestr}
155+
156+
# create new case and build it
157+
echo "Creating case $case.${casename}_${datestr}..."
158+
$code_root/cime/scripts/create_test $case \
159+
-t ${casename}_${datestr} -p $project -q $queue --output-root $workdir --no-run #--no-build
160+
161+
if [ $? != 0 ]; then
162+
echo "Error creating case $case"
163+
exit 1
164+
fi
165+
166+
# submit job and monitor status
167+
pushd $path
168+
./xmlchange JOB_WALLCLOCK_TIME=$wallclock_time
169+
./case.submit >& log.submit
170+
171+
if [ $? != 0 ]; then
172+
echo "Error submitting job for $case"
173+
exit 1
174+
fi
175+
176+
jobID=`awk -F " " '($1=="Submitted") {print $NF}' log.submit | tail -n 1 `
177+
178+
while true; do
179+
sacct --job $jobID > log.job_stat
180+
jobST=` awk "{if(NR==3) print}" log.job_stat | awk '{printf"%s\n",$6}'`
181+
182+
if [ "${jobST}" == "COMPLETED" ]; then
183+
printf "%s Run complete: %-30s jobID: %s\n" "$(date +'%Y-%m-%d %H:%M:%S')" "$casename" "$jobID"
184+
break
185+
elif [[ ("${jobST}" == "FAILED") || ("${jobST}" == *"CANCELLED"*) || ("${jobST}" == "TIMEOUT") ]]; then
186+
printf "%s Run failed: %-30s jobID: %s\n" "$(date +'%Y-%m-%d %H:%M:%S')" "$casename" "$jobID"
187+
exit 1
188+
else
189+
printf "%s Waiting for case: %-30s to complete... jobID: %s\n" "$(date +'%Y-%m-%d %H:%M:%S')" "$casename" "$jobID"
190+
sleep $interval
191+
fi
192+
done
193+
194+
popd
195+
}
196+
197+
# Silent versions of popd and pushd
198+
pushd() {
199+
command pushd "$@" > /dev/null
200+
}
201+
popd() {
202+
command popd "$@" > /dev/null
203+
}
204+
205+
# Run the script
206+
main

‎jenkins/plot_html.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import argparse
2+
import matplotlib.pyplot as plt
3+
import sys
4+
5+
def parse_args(argv):
6+
parser = argparse.ArgumentParser(description='Plot EAMxx-MAM4xx performance data')
7+
parser.add_argument('-i', '--input', metavar='r', type=ascii, nargs='+',
8+
help='input performance comp graph')
9+
10+
parser.add_argument('-o', '--output', metavar='m', type=ascii, nargs='+',
11+
help='html output dash graph')
12+
13+
args = vars(parser.parse_args())
14+
15+
input_graph = args["input"][0].strip("'""'")
16+
output_graph = args["output"][0].strip("'""'")
17+
18+
return input_graph, output_graph
19+
20+
input_graph, output_graph = parse_args(sys.argv[1:])
21+
22+
# Append the image to the HTML file
23+
html_content = f'<img src="{input_graph}" alt="comparison between EAMxx and EAMxx+MAM4xx">\n'
24+
with open(f'{output_graph}', "a") as html_file:
25+
html_file.write(html_content)
26+
27+
# Display the HTML file path
28+
print(f"HTML file generated: {output_graph}")

0 commit comments

Comments
 (0)