diff --git a/jenkins/compy_cron.sh b/jenkins/compy_cron.sh new file mode 100755 index 0000000..46e4762 --- /dev/null +++ b/jenkins/compy_cron.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +main() { + + #--------------------------------------------------------------- + # User-defined configuration + #--------------------------------------------------------------- + #path to E3SM + code_root="/compyfs/litz372/e3sm_scratch/performance_testing/E3SM" + + #machine to run the test on + mach="compy" + + #compiler + compiler=intel + + #set resolution + resolution=ne30pg2_ne30pg2 + + #set the length of the simulation (ie Ld5 for 5 days or Ln5 for 5 timesteps) + simulation_length=Ld5 + + #directory for all the past and current data + data_dest="/qfs/projects/eagles/litz372/performance_data/${simulation_length}" + + #path where the latest time series plot is saved, accessible to the whole project + share_dest="/compyfs/www/litz372/performance_data" + + #url where the plot will be available - based on the share_dest + share_url_ne30="https://compy-dtn.pnl.gov/litz372/performance_data/performance_comp_${resolution}_${simulation_length}.png" + + #boolean if E3SM git should fetch and reset + hard_reset_E3SM=true + + #load modules + source /etc/profile.d/modules.sh + module load python/3.11.5 + + ulimit -d unlimited + ulimit -s unlimited + ulimit -c unlimited + + #--------------------------------------------------------------- + # User-defined configuration ENDs + #--------------------------------------------------------------- + + #--------------------------------------------------------------- + # RUN NE30 SIMULATION + #--------------------------------------------------------------- + + /compyfs/litz372/e3sm_scratch/performance_testing/E3SM_test_scripts/jenkins/mam4xx_compare_performance.sh -r $resolution -c $compiler -t $simulation_length -m $mach -p $code_root -f $hard_reset_E3SM -d $data_dest -s $share_dest -u $share_url_ne30 + + #--------------------------------------------------------------- + # RUN NE4 SIMULATION + #--------------------------------------------------------------- + + resolution=ne4pg2_ne4pg2 + hard_reset_E3SM=false + share_url_ne4="https://compy-dtn.pnl.gov/litz372/performance_data/performance_comp_${resolution}_${simulation_length}.png" + /compyfs/litz372/e3sm_scratch/performance_testing/E3SM_test_scripts/jenkins/mam4xx_compare_performance.sh -r $resolution -c $compiler -t $simulation_length -m $mach -p $code_root -f $hard_reset_E3SM -d $data_dest -s $share_dest -u $share_url_ne4 + + #--------------------------------------------------------------- + # RUN PERFORMANCE BREAKDOWN + #--------------------------------------------------------------- + + module unload python + module load python/miniconda4.12.0 + /compyfs/litz372/e3sm_scratch/compare_model_performance/E3SM_test_scripts/jenkins/compy_model_performance.sh + + #--------------------------------------------------------------- + # APPEND LINE GRAPHS TO BREAKDOWN + #--------------------------------------------------------------- + + #plot line in dash comp + source ${code_root}/../.venv/bin/activate + output_graph=${share_dest}/../compare_performance/plot.html + echo $share_url_ne4 + echo $share_url_ne30 + echo $output_graph + python3 /compyfs/litz372/e3sm_scratch/compare_model_performance/E3SM_test_scripts/jenkins/mam4xx_append_plot.py -i ${share_url_ne4} -o ${output_graph} + python3 /compyfs/litz372/e3sm_scratch/compare_model_performance/E3SM_test_scripts/jenkins/mam4xx_append_plot.py -i ${share_url_ne30} -o ${output_graph} + +} + +#-------------------------- +# Start the script +#-------------------------- +main diff --git a/jenkins/compy_mam4xx_compare_performance.sh b/jenkins/compy_mam4xx_compare_performance.sh index 0aee5a6..7feb092 100755 --- a/jenkins/compy_mam4xx_compare_performance.sh +++ b/jenkins/compy_mam4xx_compare_performance.sh @@ -7,7 +7,8 @@ main() { #--------------------------------------------------------------- #path to E3SM code_root="/compyfs/litz372/e3sm_scratch/performance_testing/E3SM" - fetch_root=true + #boolean if E3SM git should fetch and reset + hard_reset_E3SM=true #machine to run the test on mach="compy" @@ -43,7 +44,7 @@ main() { #--------------------------------------------------------------- #TODO: - /compyfs/litz372/e3sm_scratch/performance_testing/E3SM_test_scripts/jenkins/mam4xx_compare_performance.sh -r $resolution -c $compiler -t $simulation_length -m $mach -p $code_root -f $fetch_root -d $data_dest -s $share_dest -u $share_url + /compyfs/litz372/e3sm_scratch/performance_testing/E3SM_test_scripts/jenkins/mam4xx_compare_performance.sh -r $resolution -c $compiler -t $simulation_length -m $mach -p $code_root -f $hard_reset_E3SM -d $data_dest -s $share_dest -u $share_url } diff --git a/jenkins/compy_model_performance.sh b/jenkins/compy_model_performance.sh new file mode 100755 index 0000000..8c6652a --- /dev/null +++ b/jenkins/compy_model_performance.sh @@ -0,0 +1,206 @@ +#!/bin/bash + +#------------------------------------------------------------------------------- +#------------------------------------------------------------------------------- +# Description: +# This script automates the process of comparing model performance between master and development branches of E3SM. +# +# Usage: +# Customize the user input section below before running the script. +#--------------------------------------------------------------------------- +#--------------------------------------------------------------------------- + +main() { + + #--------------------------------------------------------------- + # User-defined configuration + #--------------------------------------------------------------- + + # Perlmutter + # machine=pm-cpu + # compiler=gnu + # project=e3sm + # workdir=/pscratch/sd/m/meng/compare_model_performance + # plotdir=/global/cfs/cdirs/e3sm/www/meng/compare_performance # make sure this a www in your Community directory to check the plot as a html page + # html_address=https://portal.nersc.gov/cfs/e3sm/meng/compare_performance + # module load python + + # Compy + machine=compy + compiler=intel + project=e3sm + workdir=/compyfs/litz372/e3sm_scratch/compare_model_performance + plotdir=/compyfs/www/litz372/compare_performance + html_address=https://compy-dtn.pnl.gov/litz372/compare_performance + source /etc/profile.d/modules.sh + module load python/miniconda4.12.0 + source /share/apps/python/miniconda4.12.0/etc/profile.d/conda.sh + source ${workdir}/venv/bin/activate + + if [ ! -d $plotdir ]; then + mkdir -p $plotdir + fi + + compset=F2010-EAMxx-MAM4xx #F2010-SCREAMv1 +# resolution=ne4pg2_oQU480 + resolution=ne30pg2_ne30pg2 + pe=P32 + runtime=Ln5 + queue=debug + wallclock_time=00:05:00 + + # SMS test run + case=SMS_$pe_$runtime.$resolution.$compset.${machine}_$compiler + + branch1=master + + datestr=`date +'%Y%m%d_%H%M%S'` + casename1=master + + code_root1=$workdir/E3SM-$casename1 + + do_fetch_code=true + do_run_case=true + + # If you only need to tweak the plot, set the plot_str to the date string of the run. + do_plot=true + #plot_str=20250703_030001 + + #--------------------------------------------------------------- + # User-defined configuration ENDs + #--------------------------------------------------------------- + + # Fetch code from GitHub + fetch_code $branch1 $code_root1 & + wait + + if [ $? != 0 ]; then + echo "Error fetching code from GitHub" + exit 1 + fi + + # Create SMS test case and run it + run_case $code_root1 $casename1 & + wait + + if [ $? != 0 ]; then + echo "Error compiling or running case" + exit 1 + fi + + # Grab timing data and do the plotting + if [ "${do_plot,,}" != "true" ]; then + echo $'\n----- Skipping plot -----\n' + return + fi + if [ -z "${plot_str+x}" ]; then + plot_str=$datestr + fi + + case_root1=$workdir/$case.${casename1}_${plot_str} + + python ${workdir}/E3SM_test_scripts/jenkins/mam4xx_compare_model_performance_plot.py \ + --case1 $case_root1 --casename1 $casename1 \ + --outdir $plotdir --html $html_address + + if [ $? != 0 ]; then + echo "Error plotting model performance" + exit 1 + fi + + echo "Model performance comparison complete!" + +} + +#--------------------- +# Function Definitions +#--------------------- + +fetch_code() { + if [ "${do_fetch_code,,}" != "true" ]; then + echo $'\n----- Skipping fetch_code -----\n' + return + fi + local branch=$1 + local path=$2 + + echo "Fetching code from $branch branch to $path..." + if [ -d $path ]; then + echo "Code directory $path already exists. Removing it..." + rm -rf $path + fi + + mkdir -p $path + pushd $path + git clone git@github.com:E3SM-Project/E3SM.git . + if [ '$branch' != 'master' ]; then + git checkout $branch + fi + git submodule update --init --recursive + popd +} + +run_case() { + if [ "${do_run_case,,}" != "true" ]; then + echo $'\n----- Skipping create_newcase -----\n' + return + fi + + local code_root=$1 + local casename=$2 + + local interval=60 # seconds to wait between each check + + path=$workdir/$case.${casename}_${datestr} + + # create new case and build it + echo "Creating case $case.${casename}_${datestr}..." + $code_root/cime/scripts/create_test $case \ + -t ${casename}_${datestr} -p $project -q $queue --output-root $workdir --no-run #--no-build + + if [ $? != 0 ]; then + echo "Error creating case $case" + exit 1 + fi + + # submit job and monitor status + pushd $path + ./xmlchange JOB_WALLCLOCK_TIME=$wallclock_time + ./case.submit >& log.submit + + if [ $? != 0 ]; then + echo "Error submitting job for $case" + exit 1 + fi + + jobID=`awk -F " " '($1=="Submitted") {print $NF}' log.submit | tail -n 1 ` + + while true; do + sacct --job $jobID > log.job_stat + jobST=` awk "{if(NR==3) print}" log.job_stat | awk '{printf"%s\n",$6}'` + + if [ "${jobST}" == "COMPLETED" ]; then + printf "%s Run complete: %-30s jobID: %s\n" "$(date +'%Y-%m-%d %H:%M:%S')" "$casename" "$jobID" + break + elif [[ ("${jobST}" == "FAILED") || ("${jobST}" == *"CANCELLED"*) || ("${jobST}" == "TIMEOUT") ]]; then + printf "%s Run failed: %-30s jobID: %s\n" "$(date +'%Y-%m-%d %H:%M:%S')" "$casename" "$jobID" + exit 1 + else + printf "%s Waiting for case: %-30s to complete... jobID: %s\n" "$(date +'%Y-%m-%d %H:%M:%S')" "$casename" "$jobID" + sleep $interval + fi + done + + popd +} + +# Silent versions of popd and pushd +pushd() { + command pushd "$@" > /dev/null +} +popd() { + command popd "$@" > /dev/null +} + +# Run the script +main diff --git a/jenkins/mam4xx_append_plot.py b/jenkins/mam4xx_append_plot.py new file mode 100644 index 0000000..9ba0271 --- /dev/null +++ b/jenkins/mam4xx_append_plot.py @@ -0,0 +1,28 @@ +import argparse +import matplotlib.pyplot as plt +import sys + +def parse_args(argv): + parser = argparse.ArgumentParser(description='Plot EAMxx-MAM4xx performance data') + parser.add_argument('-i', '--input', metavar='r', type=ascii, nargs='+', + help='input performance comp graph') + + parser.add_argument('-o', '--output', metavar='m', type=ascii, nargs='+', + help='html output dash graph') + + args = vars(parser.parse_args()) + + input_graph = args["input"][0].strip("'""'") + output_graph = args["output"][0].strip("'""'") + + return input_graph, output_graph + +input_graph, output_graph = parse_args(sys.argv[1:]) + +# Append the image to the HTML file +html_content = f'comparison between EAMxx and EAMxx+MAM4xx\n' +with open(f'{output_graph}', "a") as html_file: + html_file.write(html_content) + +# Display the HTML file path +print(f"HTML file generated: {output_graph}") diff --git a/jenkins/mam4xx_compare_model_performance_plot.py b/jenkins/mam4xx_compare_model_performance_plot.py new file mode 100644 index 0000000..5b32c45 --- /dev/null +++ b/jenkins/mam4xx_compare_model_performance_plot.py @@ -0,0 +1,260 @@ +import argparse +from glob import glob +import pandas as pd +import numpy as np +import os +import matplotlib.pyplot as plt +from datetime import datetime +import subprocess + + +parser = argparse.ArgumentParser() +parser.add_argument("--case1", type=str, help="Path to case1 run") +parser.add_argument("--casename1", type=str, help="Short name for case1 run") +#parser.add_argument("--case2", type=str, help="Path to case2 run") +#parser.add_argument("--casename2", type=str, help="Short name for case2 run") +parser.add_argument("--outdir", type=str, help="Output directory for images") +parser.add_argument("--html", type=str, help="Generate HTML file with images") + +args = parser.parse_args() + +case1 = args.case1 +#case2 = args.case2 +casename1 = args.casename1 +#casename2 = args.casename2 +outdir = args.outdir +html = args.html + +def grab_timing(case): + + with open(f'{case}/log.job_stat', 'r') as f: + lines = [line.strip() for line in f] + jobid = lines[2].split()[0] + status = lines[2].split()[5] + + if status == 'COMPLETED': + timinglog = glob(f'{case}/timing/e3sm_timing_stats.{jobid}.*')[0] + df = pd.read_csv(timinglog, sep='\s+', header=4) + s1 = df['name'].str.startswith('a:EAMxx::') + s2 = df['name'].str.endswith('::run') + df2 = df[s1 & s2][1:][['name', 'walltotal']] + processes = [p for p in df2['name'].str.split('::').str[1]] + proc_new = [p not in ['EAMxx', 'physics', 'mac_aero_mic'] for p in processes] + + df3 = df2[proc_new].sort_values('name') + processes = [p for p in df3['name'].str.split('::').str[1]] + + compset = case.split('/')[-1].split('.')[2] + if compset == 'F2010-EAMxx-MAM4xx': aer_proc = [p.startswith('mam') for p in processes] + if compset == 'F2010-SCREAMv1': aer_proc = [p.startswith('spa') for p in processes] + + dfaer = df3[aer_proc].reset_index(drop=True) + dfeam = df3[~np.array(aer_proc)] + dftmp = pd.DataFrame({'name': 'a:EAMxx::aerosols::run', 'walltotal': dfaer['walltotal'].sum()}, index=[99]) + dfeam = pd.concat([dfeam, dftmp]).reset_index(drop=True) + return dfeam, dfaer + else: + print(f'Run {case} not completed') + return None + + +def plot_mam4_process(dfeam, dfaer, axl, axr): + + eam_ratios = dfeam['walltotal'] / dfeam['walltotal'].sum() + labels = [p.replace('SurfaceCoupling', 'SC') for p in dfeam['name'].str.split('::').str[1]] + labels[labels.index('aerosols')] = 'mam4' + + explode = np.zeros(len(labels)) + explode[-1] = 0.08 + # rotate so that first wedge is split by the x-axis + angle = eam_ratios.values[-1] / 2 * 360 + # angle = 0 + autopct = lambda v: f'{v:.1f}%' if v >= 1 else None + colors = plt.get_cmap('Set3').colors[:] + wedges, *_ = axl.pie(eam_ratios, startangle=angle, colors=colors, autopct=autopct, + explode=explode) + + kw = dict(xycoords='data', textcoords='data', arrowprops=dict(arrowstyle="-"), zorder=0, va="center") + l = [] + for i, p in enumerate(wedges): + if eam_ratios[i] < 0.01: + l.append(f'{labels[i]} ({eam_ratios[i]:.1%})') + continue + else: + ang = (p.theta2 - p.theta1)/2. + p.theta1 + y = np.sin(np.deg2rad(ang)) + x = np.cos(np.deg2rad(ang)) + horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] + connectionstyle = "angle,angleA=0,angleB={}".format(ang) + kw["arrowprops"].update({"connectionstyle": connectionstyle,"color":colors[i]}) + axl.annotate(labels[i], xy=(x, y), xytext=(1.2*np.sign(x), y), + horizontalalignment=horizontalalignment, **kw) + axl.axis('equal') + # create handles and labels for legend, take only those where value is < 1 + handles = [h for h,r in zip(wedges,eam_ratios) if r < 0.01] + axl.legend(handles, l, fontsize='small', ncols=2, bbox_to_anchor=(0.5,0.01), loc='lower center', + title='EAMxx process < 1%', title_fontsize='small', handlelength=0.5, handleheight=0.5, handletextpad=0.5, columnspacing=0.5) + axl.set_title('EAMxx/MAM4xx', fontsize='large', y=0.8) + + + mam_ratios = dfaer['walltotal'] / dfaer['walltotal'].sum() + mam_labels = ['_'.join(p.split('_')[1:]) for p in dfaer['name'].str.split('::').str[1]] + wedges2, *_ = axr.pie(mam_ratios, autopct=autopct, colors=colors) + l2 = [] + for i, p in enumerate(wedges2): + if mam_ratios[i] < 0.01: + l2.append(f'{mam_labels[i]} ({mam_ratios[i]:.1%})') + continue + else: + ang = (p.theta2 - p.theta1)/2. + p.theta1 + y = np.sin(np.deg2rad(ang)) + x = np.cos(np.deg2rad(ang)) + horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] + connectionstyle = "angle,angleA=0,angleB={}".format(ang) + kw["arrowprops"].update({"connectionstyle": connectionstyle,"color":colors[i]}) + axr.annotate(mam_labels[i], xy=(x, y), xytext=(1.15*np.sign(x), y), + horizontalalignment=horizontalalignment, **kw) + handles = [h for h,r in zip(wedges2,mam_ratios) if r < 0.01] + axr.legend(handles, l2, fontsize='small', ncols=2, bbox_to_anchor=(0.5,0.01), loc='lower center', + title='MAM4xx process < 1%', title_fontsize='small', handlelength=0.5, handleheight=0.5, handletextpad=0.5, columnspacing=0.5) + axr.set_title('MAM4xx', fontsize='large', y=0.8) + axr.axis('equal') + + +def plot_eam_process(dfeam, ax): + + eam_ratios = dfeam['walltotal'] / dfeam['walltotal'].sum() + labels = [p.replace('SurfaceCoupling', 'SC') for p in dfeam['name'].str.split('::').str[1]] + labels[labels.index('aerosols')] = 'spa' + + explode = np.zeros(len(labels)) + explode[-1] = 0.08 + # rotate so that first wedge is split by the x-axis + angle = eam_ratios.values[-1] / 2 * 360 + # angle = 0 + autopct = lambda v: f'{v:.1f}%' if v >= 1 else None + colors = plt.get_cmap('Set3').colors[:] + wedges, *_ = ax.pie(eam_ratios, startangle=angle, colors=colors, autopct=autopct, + explode=explode) + + kw = dict(xycoords='data', textcoords='data', arrowprops=dict(arrowstyle="-"), zorder=0, va="center") + l = [] + for i, p in enumerate(wedges): + if eam_ratios[i] < 0.01: + l.append(f'{labels[i]} ({eam_ratios[i]:.1%})') + continue + else: + ang = (p.theta2 - p.theta1)/2. + p.theta1 + y = np.sin(np.deg2rad(ang)) + x = np.cos(np.deg2rad(ang)) + horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] + connectionstyle = "angle,angleA=0,angleB={}".format(ang) + kw["arrowprops"].update({"connectionstyle": connectionstyle,"color":colors[i]}) + ax.annotate(labels[i], xy=(x, y), xytext=(1.2*np.sign(x), y), + horizontalalignment=horizontalalignment, **kw) + ax.axis('equal') + # create handles and labels for legend, take only those where value is < 1 + handles = [h for h,r in zip(wedges,eam_ratios) if r < 0.01] + ax.legend(handles, l, fontsize='small', title='EAMxx process < 1%', title_fontsize='small', + handlelength=0.5, handleheight=0.5, handletextpad=0.5, columnspacing=0.5, + # ncols=2, bbox_to_anchor=(0.5,0.01), loc='lower center', + ) + ax.set_title('EAMxx', fontsize='large') + + +t1, taer1 = grab_timing(case1) +#t2, taer2 = grab_timing(case2) + +expname = '.'.join(os.path.basename(case1).split('.')[1:4]) +now_str = datetime.now().strftime("%Y%m%d-%H%M%S") + +#if False: # t1 is not None and t2 is not None: +if t1 is not None: + + plt.rcParams.update({'font.size': 12}) + + # plot eamxx process + fig, ax = plt.subplots(constrained_layout=True, figsize=(9, 4)) + + run_time = {} + eamprocess = [p for p in t1['name'].str.split('::').str[1]] + run_time[casename1] = t1['walltotal'] +# run_time[casename2] = t2['walltotal'] + + x = np.arange(len(eamprocess)) + width = 0.4 + multiplier = 0 + for cn, rt in run_time.items(): + offset = width * multiplier + rects = ax.bar(x + offset , rt, width, label=cn) + + multiplier += 1 + ax.legend(fontsize='large', loc='upper left', ) + eamproc = [p.replace('SurfaceCoupling', 'SC') for p in eamprocess] + ax.set_xticks(x + width / 2, eamproc) + ax.set_xticklabels(eamproc, rotation=45, fontsize='large') + ax.set_ylabel('wallclock', fontsize='large') + + barplot = 'eamxx_process' + image0 = f"{outdir}/{barplot}_{now_str}.png" + fig.savefig(image0, dpi=300, bbox_inches='tight') + plt.close(fig) + + # plot mam4 process + compset = case1.split('/')[-1].split('.')[2] + if compset == 'F2010-EAMxx-MAM4xx': + fig, axs = plt.subplots(figsize=(10, 4.5), ncols=2, constrained_layout=True) + plot_mam4_process(t1, taer1, axs[0], axs[1]) + piechart1 = 'process_case1' + image1 = f"{outdir}/{piechart1}_{now_str}.png" + fig.savefig(image1, dpi=300, bbox_inches='tight') + plt.close(fig) + +# fig, axs = plt.subplots(figsize=(10, 4.5), ncols=2, constrained_layout=True) +# plot_mam4_process(t2, taer2, axs[0], axs[1]) +# piechart2 = 'process_case2' +# image2 = f"{outdir}/{piechart2}_{now_str}.png" +# fig.savefig(image2, dpi=300, bbox_inches='tight') +# plt.close(fig) + else: + fig, ax = plt.subplots(figsize=(10, 4.5), constrained_layout=True) + plot_eam_process(t1, ax) + piechart1 = 'process_case1' + image1 = f"{outdir}/{piechart1}_{now_str}.png" + fig.savefig(image1, dpi=300, bbox_inches='tight') + plt.close(fig) + +# fig, ax = plt.subplots(figsize=(10, 4.5), constrained_layout=True) +# plot_eam_process(t2, ax) +# piechart2 = 'process_case2' +# image2 = f"{outdir}/{piechart2}_{now_str}.png" +# fig.savefig(image2, dpi=300, bbox_inches='tight') +# plt.close(fig) + + +# Save the figure as an image file +html_content = "Compare Model Performance" + +# Add the image to the HTML content +png_files = [image0, image1] #, image2] +png_names = [expname, 'RUN1: '+casename1]#, 'RUN2: '+casename2] +for v, img in zip(png_names, png_files): + img = f'{html}/{os.path.basename(img)}' + html_content += f'

{v}

\n' + html_content += f'

\n' + html_content += "" + +# Write the HTML content to a file +html_file_path = outdir +#with open(f'{html_file_path}/compare_{casename1}_vs_{casename2}_{now_str}.html', "w") as html_file: +with open(f'{html_file_path}/plot_{casename1}_{now_str}.html', "w") as html_file: + html_file.write(html_content) + +# Display the HTML file path +#subprocess.run(['ln', '-sf', f'{html_file_path}/compare_{casename1}_vs_{casename2}_{now_str}.html', f'{html_file_path}/plot.html'], check=True) +subprocess.run(['ln', '-sf', f'{html_file_path}/plot_{casename1}_{now_str}.html', f'{html_file_path}/plot.html'], check=True) +print(f"HTML file generated: {html}/plot.html") + + + + diff --git a/jenkins/mam4xx_compare_performance.sh b/jenkins/mam4xx_compare_performance.sh index 88ac367..f7ecdd3 100755 --- a/jenkins/mam4xx_compare_performance.sh +++ b/jenkins/mam4xx_compare_performance.sh @@ -32,9 +32,12 @@ main() { #compset for EAMxx compset_eamxx="F2010-SCREAMv1" + + #set test modifier to 72 levels + test_mod="eamxx-L72" echo "Configuration: " - echo "Test ${testname} on ${mach} with ${compiler} compiler" + echo "Test ${testname} on ${mach} with ${compiler} compiler and ${test_mod} modifier" echo "Resolution ${resolution} and compsets ${compset_mam4xx} and ${compset_eamxx}" #--------------------------------------------------------------- @@ -47,7 +50,7 @@ main() { cd $code_root #reset code to the latest master - if $fetch_root; then + if $hard_reset_E3SM; then update_e3sm fi @@ -67,17 +70,22 @@ main() { #Run E3SM tests echo "starting EAMxx+MAM4xx test..." - run_e3sm_test $resolution $compset_mam4xx $mach $compiler $testname $temp_dir + run_e3sm_test $resolution $compset_mam4xx $mach $compiler $testname $temp_dir $test_mod echo "starting EAMxx default test..." - run_e3sm_test $resolution $compset_eamxx $mach $compiler $testname $temp_dir + run_e3sm_test $resolution $compset_eamxx $mach $compiler $testname $temp_dir $test_mod #wait for the tests directories to be created sleep 120 #check if the EAMxx+MAM4xx test completed - mam4xx_dir=${testname}.${resolution}.${compset_mam4xx}.${mach}_${compiler}.master_${date_str} + mam4xx_dir=${testname}.${resolution}.${compset_mam4xx}.${mach}_${compiler}.${test_mod}.master_${date_str} wait_for_run_completion $temp_dir/$mam4xx_dir + mam4xx_pass=$? + if [[ $mam4xx_pass -eq 1 ]]; then + echo "MAM4xx run FAILED. Stopping..." + exit + fi #Grab MAM4xx timing data cd $temp_dir/$mam4xx_dir/timing @@ -87,8 +95,14 @@ main() { echo "EAMxx+MAM4xx Model Cost - ${mam4xx_cost}" #check if the EAMxx test completed - eamxx_dir=${testname}.${resolution}.${compset_eamxx}.${mach}_${compiler}.master_${date_str} + eamxx_dir=${testname}.${resolution}.${compset_eamxx}.${mach}_${compiler}.${test_mod}.master_${date_str} wait_for_run_completion $temp_dir/$eamxx_dir + eamxx_pass=$? + if [[ $eamxx_pass -eq 1 ]]; then + echo "EAMxx run FAILED. Stopping..." + exit + fi + cd $temp_dir/$eamxx_dir/timing #Grab EAMxx timing data @@ -159,11 +173,12 @@ run_e3sm_test () { local compiler=$4 local testname=$5 local temp_dir=$6 + local test_mod=$7 #launch tests echo "Launching Test:" - echo "Creating test: ${testname}.${resolution}.${compset}" - ./create_test ${testname}.${resolution}.${compset} --compiler $compiler \ + echo "Creating test: ${testname}.${resolution}.${compset}.${mach}_${compiler}.${test_mod}" + ./create_test ${testname}.${resolution}.${compset}.${mach}_${compiler}.${test_mod} --compiler $compiler \ -p e3sm -t "master_${date_str}" --output-root "${temp_dir}" -m ${mach} & } @@ -182,12 +197,17 @@ wait_for_run_completion () { if grep -q "Timing" CaseStatus; then echo "Run complete! Timing info found in CaseStatus." break + elif grep -q "FAIL" TestStatus; then + echo "Run failed. See log for more." + cat CaseStatus + return 1 fi fi echo "Still waiting... elapsed: $((elapsed/60)) min" sleep "$interval" (( elapsed += interval )) done + return 0 } wait_til_dir_created() { @@ -230,7 +250,7 @@ while getopts ":r:c:t:m:p:f:d:s:u:" opt; do \?) echo "Invalid option -$OPTARG; please select a valid path using -p command line option" >&2 exit 1 ;; - f) fetch_root="$OPTARG" + f) hard_reset_E3SM="$OPTARG" ;; \?) echo "Invalid option -$OPTARG; please select a valid boolean using -f command line option" >&2 exit 1 @@ -289,8 +309,8 @@ if [ -z "${code_root}" ]; then exit 1 fi -if [ -z "${fetch_root}" ]; then - echo "Fetch code boolean is not set, please set it using -f command line option" +if [ -z "${hard_reset_E3SM}" ]; then + echo "Boolean for fetching latest E3SM is not set, please set it using -f command line option" exit 1 fi diff --git a/jenkins/pm-gpu_mam4xx_compare_performance.sh b/jenkins/pm-gpu_mam4xx_compare_performance.sh index 2508ea0..019f7d6 100755 --- a/jenkins/pm-gpu_mam4xx_compare_performance.sh +++ b/jenkins/pm-gpu_mam4xx_compare_performance.sh @@ -8,7 +8,8 @@ main() { #path to E3SM code_root="/pscratch/sd/e/e3smtest/jenkins/workspace/ACME_Perlmutter_mam4xx_compare_performance/E3SM" #code_root="/global/cfs/projectdirs/e3sm/litzingj/E3SM" - fetch_root=false + #boolean if E3SM git should fetch and reset + hard_reset_E3SM=false #machine to run the test on mach="pm-gpu" @@ -40,7 +41,7 @@ main() { #--------------------------------------------------------------- #TODO: - /global/cfs/projectdirs/e3sm/litzingj/E3SM_test_scripts/jenkins/mam4xx_compare_performance.sh -r $resolution -c $compiler -t $simulation_length -m $mach -p $code_root -f $fetch_root -d $data_dest -s $share_dest -u $share_url + /global/cfs/projectdirs/e3sm/litzingj/E3SM_test_scripts/jenkins/mam4xx_compare_performance.sh -r $resolution -c $compiler -t $simulation_length -m $mach -p $code_root -f $hard_reset_E3SM -d $data_dest -s $share_dest -u $share_url }