test_cloud_runner #485
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test_cloud_runner | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| fre-workflows-repo: | |
| description: "If using fork: Input fork name (example: singhd789/fre-workflows). Leave empty if not using a fork." | |
| required: false | |
| type: string | |
| fre-workflows-branch: | |
| description: 'If on fork: Input branch you are using on your fork.' | |
| required: false | |
| default: 'main' | |
| type: string | |
| fre-cli-repo: | |
| description: 'NOAA-GFDL/fre-cli repo' | |
| required: true | |
| type: choice | |
| default: 'NOAA-GFDL/fre-cli' | |
| options: | |
| - 'NOAA-GFDL/fre-cli' | |
| fre-cli-branch: | |
| description: 'Input which fre-cli branch to check out' | |
| required: true | |
| default: 'main' | |
| type: string | |
| push: | |
| branches: | |
| - main # triggers pipeline on push to main | |
| # cancel running jobs if theres a newer push | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test-cloud: | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout fre-workflows fork if specified | |
| if: ${{ github.event.inputs.fre-workflows-repo != ''}} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.fre-workflows-repo }} | |
| ref: ${{ github.event.inputs.fre-workflows-branch }} | |
| - name: Checkout fre-workflows repository if no fork specified | |
| if: ${{ github.event.inputs.fre-workflows-repo == ''}} | |
| uses: actions/checkout@v4 | |
| - name: Checkout fre-cli | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.fre-cli-repo }} | |
| ref: ${{ github.event.inputs.fre-cli-branch }} | |
| path: './fre-cli' | |
| submodules: true | |
| - name: Display fre-workflow and fre-cli repos and branches | |
| run: | | |
| # fre-workflows repo and branch | |
| echo "fre-workflows repo: $(git remote -v)" | |
| echo "fre-workflows branch: $(git branch)" | |
| # fre-cli repo and branch | |
| cd fre-cli | |
| echo "fre-cli repo: $(git remote -v)" | |
| echo "fre-cli branch: $(git branch)" | |
| cd - | |
| - name: Run ppp-container | |
| run: | | |
| # Print pwd then run script | |
| echo "Currently in:" | |
| cwd=$(pwd) | |
| echo $cwd | |
| gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar | |
| bind_paths="--bind /contrib-efs/container-test/ppp-setup/:/mnt:rw | |
| --bind /contrib2-bucket/c96L65_am5f11d12r0_amip/history:/mnt/history:ro | |
| --bind /contrib-efs/container-test/c96_grid/${gridspec_file}:/mnt/${gridspec_file}:ro" | |
| ## TO-DO: figure out pulling container instead | |
| container_path=/contrib-efs/container-test/fre-2025.04.sif | |
| runscript_path=${cwd}/for_gh_runner/runscript.sh | |
| singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path} | |
| # Can potentially do this to get list of tasks | |
| #task_list=$(cylc list test_pp__ptest__ttest) | |
| #task1=$(echo $task_list | cut -d ' ' -f1) .... | |
| - name: Upload workflow-run log files | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fre-workflow-logs | |
| path: /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log | |
| if-no-files-found: error | |
| - name: Print workflow summary | |
| continue-on-error: true | |
| run: | | |
| num_tasks=$(ls -l /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/????*/*/NN/job | wc -l) | |
| echo "number of tasks launched: ${num_tasks}" | |
| echo "" | |
| echo "here's all the job directories with job scripts:" | |
| ls -l /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/????*/*/NN/job | |
| echo "" | |
| echo "here's all the job scripts for 1980 only:" | |
| ls -l /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/1980*/*/NN/job | |
| echo "" | |
| echo "rose-suite conf for workflow:" | |
| cat /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/config/*rose-suite.conf | |
| echo "" | |
| - name: Print pp-starter successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print stage-history successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print regrid-xy successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print data-catalog-final (FAILURE-GUARDED) | |
| continue-on-error: true | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print mask-atmos-plevel successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/mask-atmos-plevel*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/mask-atmos-plevel*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print remap-pp-components successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print split-netcdf successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print climotology time average successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/climo-*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/climo-*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print combine-climo successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/combine-climo-*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/combine-climo*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print remap-climo successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-climo-*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-climo-*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print clean task successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| ## Keep in here for now as a just in case reference ## | |
| # - name: Print any succeeded tasks | |
| # run: | | |
| # # List tasks that have succeeded (and completed) in the workflow | |
| # grep -E ":succeeded.*completed" /contrib/container-test/ppp-setup/log.out | |
| # | |
| ## maybe, all the printinig above should be continue-on-error for informational purposes | |
| ## then, this task does the final determination if things ran right or wrong. | |
| ## githubs auto-annotations for warnings etc are good are helping one figure out where to sniff around | |
| # - name: Print any failed tasks | |
| # run: | | |
| # # Search for if there is an ERR status for any task | |
| # if grep -q "failed/ERR" /contrib/container-test/ppp-setup/log.out; then | |
| # echo "Task failures found: CHECK THE LOG FILES UPLOADED AS ARTIFACTS" | |
| # grep "failed/ERR" /contrib/container-test/ppp-setup/log.out | |
| # exit 1 | |
| # fi | |
| # | |
| # echo "Workflow ran successfully" | |
| ##To-do: | |
| # - figure out if container will be hosted on cloud or somehow pulled from registry | |
| # - examine behavior of runner in draft vs not draft mode |