test_cloud_runner.yml

name: test_cloud_runner
on:
  workflow_dispatch:
    inputs:
      fre-workflows-repo:
        description: "If using fork: Input fork name (example: singhd789/fre-workflows). Leave empty if not using a fork."
        required: false
        type: string
      fre-workflows-branch:
        description: 'If on fork: Input branch you are using on your fork.'
        required: false
        default: 'main'
        type: string
      fre-cli-repo:
        description: 'NOAA-GFDL/fre-cli repo'
        required: true
        type: choice
        default: 'NOAA-GFDL/fre-cli'
        options:
          - 'NOAA-GFDL/fre-cli'
      fre-cli-branch:
        description: 'Input which fre-cli branch to check out'
        required: true
        default: 'main'
        type: string
  push:
    branches:
      - main  # triggers pipeline on push to main

# cancel running jobs if theres a newer push
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test-cloud:
    runs-on: self-hosted
    steps:
      - name: Checkout fre-workflows fork if specified
        if: ${{ github.event.inputs.fre-workflows-repo != ''}}
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.inputs.fre-workflows-repo }}
          ref: ${{ github.event.inputs.fre-workflows-branch }}
      - name: Checkout fre-workflows repository if no fork specified
        if: ${{ github.event.inputs.fre-workflows-repo == ''}}
        uses: actions/checkout@v4
      - name: Checkout fre-cli
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.inputs.fre-cli-repo }}
          ref: ${{ github.event.inputs.fre-cli-branch }}
          path: './fre-cli'
          submodules: true
      - name: Display fre-workflow and fre-cli repos and branches
        run: |
          # fre-workflows repo and branch
          echo "fre-workflows repo: $(git remote -v)"
          echo "fre-workflows branch: $(git branch)"

          # fre-cli repo and branch
          cd fre-cli
          echo "fre-cli repo: $(git remote -v)"
          echo "fre-cli branch: $(git branch)"
          cd -
      - name: Run ppp-container
        run: |
          # Print pwd then run script
          echo "Currently in:"
          cwd=$(pwd)
          echo $cwd
          gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar
          bind_paths="--bind /contrib-efs/container-test/ppp-setup/:/mnt:rw
                      --bind /contrib2-bucket/c96L65_am5f11d12r0_amip/history:/mnt/history:ro
                      --bind /contrib-efs/container-test/c96_grid/${gridspec_file}:/mnt/${gridspec_file}:ro"

          ## TO-DO: figure out pulling container instead
          container_path=/contrib-efs/container-test/fre-2025.04.sif
          runscript_path=${cwd}/for_gh_runner/runscript.sh

          singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path}

          # Can potentially do this to get list of tasks
          #task_list=$(cylc list test_pp__ptest__ttest)
          #task1=$(echo $task_list | cut -d ' ' -f1) ....

      - name: Upload workflow-run log files
        uses: actions/upload-artifact@v4
        with:
          name: fre-workflow-logs
          path: /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log
          if-no-files-found: error

      - name: Print workflow summary
        continue-on-error: true
        run: |
          num_tasks=$(ls -l /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/????*/*/NN/job | wc -l)
          echo "number of tasks launched: ${num_tasks}"
          echo ""

          echo "here's all the job directories with job scripts:"
          ls -l /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/????*/*/NN/job
          echo ""

          echo "here's all the job scripts for 1980 only:"
          ls -l /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/1980*/*/NN/job
          echo ""

          echo "rose-suite conf for workflow:"
          cat /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/config/*rose-suite.conf
          echo ""

      - name: Print pp-starter successes or failures
        run: |
          set +e
          # Successes
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print stage-history successes or failures
        run: |
          set +e
          # Successes 
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print regrid-xy successes or failures
        run: |
          set +e
          # Successes 
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print data-catalog-final (FAILURE-GUARDED)
        continue-on-error: true
        run: |
          set +e
          # Successes 
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print  mask-atmos-plevel successes or failures
        run: |
          set +e
          # Successes
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/mask-atmos-plevel*/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/mask-atmos-plevel*/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print remap-pp-components successes or failures
        run: |
          set +e
          # Successes 
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print split-netcdf successes or failures
        run: |
          set +e
          # Successes 
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status)
          exit_status_2=$? 

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print climotology time average successes or failures
        run: |
          set +e
          # Successes 
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/climo-*/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/climo-*/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print combine-climo successes or failures
        run: |
          set +e
          # Successes 
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/combine-climo-*/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/combine-climo*/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print remap-climo successes or failures
        run: |
          set +e
          # Successes 
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-climo-*/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/remap-climo-*/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

      - name: Print clean task successes or failures
        run: |
          set +e
          # Successes
          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status)
          exit_status_1=$?

          # Failures
          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status)
          exit_status_2=$?

          if [ $exit_status_1 -eq 0 ]; then
              echo $success | sed 's/ /\n/g' #split string by spaces
          elif [ $exit_status_1 -eq 1 ]; then
              echo "No succeeded tasks found"
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

          if [ $exit_status_2 -eq 0 ]; then
              echo "Failures found"
              echo $failure | sed 's/ /\n/g' #split string by spaces
              exit 1   #exit with error
          elif [ $exit_status_2 -eq 1 ]; then
              echo "No task failures found."
          else
              echo "WARNING: error with grep execution"
              exit 2
          fi

## Keep in here for now as a just in case reference ##
#      - name: Print any succeeded tasks
#        run: |
#          # List tasks that have succeeded (and completed) in the workflow
#          grep -E ":succeeded.*completed" /contrib/container-test/ppp-setup/log.out
#

## maybe, all the printinig above should be continue-on-error for informational purposes
## then, this task does the final determination if things ran right or wrong.
## githubs auto-annotations for warnings etc are good are helping one figure out where to sniff around
#      - name: Print any failed tasks
#        run: |
#          # Search for if there is an ERR status for any task
#          if grep -q "failed/ERR" /contrib/container-test/ppp-setup/log.out; then
#              echo "Task failures found: CHECK THE LOG FILES UPLOADED AS ARTIFACTS"
#              grep "failed/ERR" /contrib/container-test/ppp-setup/log.out
#              exit 1
#          fi
#
#          echo "Workflow ran successfully"

##To-do:
# - figure out if container will be hosted on cloud or somehow pulled from registry
# - examine behavior of runner in draft vs not draft mode