Add refinediag preanalysis remote tests

.github/workflows/test_cloud_runner.yml

@@ -80,10 +80,6 @@ jobs:
 
           singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path}
 
-          # Can potentially do this to get list of tasks
-          #task_list=$(cylc list test_pp__ptest__ttest)
-          #task1=$(echo $task_list | cut -d ' ' -f1) ....
-
       - name: Upload workflow-run log files
         uses: actions/upload-artifact@v4
         with:
@@ -172,6 +168,106 @@ jobs:
               exit 2
           fi
 
+      - name: Print stage-history-refined successes or failures
+        run: |
+          set +e
+          # Successes
+          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history-refined/0[1-3]/job.status)
+          exit_status_1=$?
+          # Failures
+          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history-refined/0[1-3]/job.status)
+          exit_status_2=$?
+          if [ $exit_status_1 -eq 0 ]; then
+              echo $success | sed 's/ /\n/g' #split string by spaces
+          elif [ $exit_status_1 -eq 1 ]; then
+              echo "No succeeded tasks found"
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+          if [ $exit_status_2 -eq 0 ]; then
+              echo "Failures found"
+              echo $failure | sed 's/ /\n/g' #split string by spaces
+              exit 1   #exit with error
+          elif [ $exit_status_2 -eq 1 ]; then
+              echo "No task failures found."
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+
+      - name: Print refine-diag successes or failures
+        run: |
+          set +e
+          # Successes 
+          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/refineDiag-*/0[1-3]/job.status)
+          exit_status_1=$?
+
+          # Failures
+          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/refineDiag-*/0[1-3]/job.status)
+          exit_status_2=$?
+
+          if [ $exit_status_1 -eq 0 ]; then
+              echo $success | sed 's/ /\n/g' #split string by spaces
+
+              # Check if refine-diag task executed correctly
+              # history files copied into another history file 
+              # ........
+
+          elif [ $exit_status_1 -eq 1 ]; then
+              echo "No succeeded tasks found"
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+
+          if [ $exit_status_2 -eq 0 ]; then
+              echo "Failures found"
+              echo $failure | sed 's/ /\n/g' #split string by spaces
+              exit 1   #exit with error
+          elif [ $exit_status_2 -eq 1 ]; then
+              echo "No task failures found."
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+
+      - name: Print pre-analysis successes or failures
+        run: |
+          set +e
+          # Successes 
+          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/preAnalysis-*/0[1-3]/job.status)
+          exit_status_1=$?
+          # Failures
+          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/preAnalysis-*/0[1-3]/job.status)
+          exit_status_2=$?
+          if [ $exit_status_1 -eq 0 ]; then
+              echo $success | sed 's/ /\n/g' #split string by spaces
+              # Check if preanalysis task executed correctly: Check database exists
+              if [ -s "/contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/db" ]; then
+                  echo "Pre-analysis task produced non-empty db file"
+                  exit 0
+              else
+                  echo "Pre-analysis task did not produce expected, non-empty db file"
+                  exit 1
+              fi
+          elif [ $exit_status_1 -eq 1 ]; then
+              echo "No succeeded tasks found"
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+          if [ $exit_status_2 -eq 0 ]; then
+              echo "Failures found"
+              echo $failure | sed 's/ /\n/g' #split string by spaces
+              exit 1   #exit with error
+          elif [ $exit_status_2 -eq 1 ]; then
+              echo "No task failures found."
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+
       - name: Print regrid-xy successes or failures
         run: |
           set +e

flow.cylc

@@ -461,6 +461,7 @@
             env
             set -x
             mkdir -p $work $tempCache $refineDiagDir
+            hsmget -v -t -a $histDir -p {{ PTMP_DIR }}/$histDir -w $work $hsmdate/\*
             cd $work/$hsmdate
             ls
         """
@@ -496,7 +497,11 @@
             fi
             if [[ $refinedCount > 0 ]]; then
                 for file in $(ls -1 *nc); do
-                    list_ncvars.csh -st01234 $file |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log
+                    # list the variables names in the netcdf file
+                    ncdump -h $file |
+                    grep -E "float|double" |
+                    awk '{print $2}' |
+                    cut -d '(' -f1 |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log
                 done
             else
                 echo ERROR: RefineDiag script did not create any NetCDF files as it was expected to do

for_gh_runner/dummy_preanalysis.csh

@@ -0,0 +1,3 @@
+#!/bin/csh
+
+echo "Pre-analysis script is running!"

for_gh_runner/dummy_refinediag.csh

@@ -0,0 +1,26 @@
+#!/bin/csh
+
+## RefineDiag scripts reads raw history files and generates new, refined history files
+# Script copies test history file to another history file
+set input_dir = `pwd`
+
+if ( $?refineDiagDir ) then
+    set output_dir = $refineDiagDir
+else
+    echo "ERROR: refineDiagDir environment variable is not set."
+    exit 1
+endif
+
+if ( -d "$input_dir" ) then
+    #ls -aF
+    echo "Input directory found: $input_dir"
+    echo "good to go"
+    foreach INFILE (*atmos_month*.nc)
+        set OUTFILE = ${INFILE:r}_refined.${INFILE:e}
+        echo "OUTFILE $OUTFILE"
+        cp "$INFILE" "$output_dir/$OUTFILE"
+    end
+else
+    echo "Error: Input directory $input_dir does not exist."
+    exit 1
+endif

for_gh_runner/runscript.sh

@@ -94,13 +94,13 @@ fre_pp_steps () {
     #Not sure if needed because if no global.cylc found, cylc uses default, which utilizes background jobs anyway ...
     #export CYLC_CONF_PATH=/mnt/cylc-src/${name}/generic-global-config/
 
-    ## Configure the rose-suite and rose-app files for the workflow
-    echo -e "\nRunning fre pp configure-yaml, combining separate yaml configs into one, then writing rose-suite/app config files ..."
+    ## Configure the rose-suite file for the workflow
+    echo -e "\nRunning fre pp configure-yaml, combining separate yaml configs into one, then writing rose-suite config file ..."
     fre -vv pp configure-yaml -e ${expname} -p ${plat} -t ${targ} -y ${yamlfile}
     check_exit_status "CONFIGURE-YAML"
 
     ## Validate the configuration files
-    echo -e "\nRunning fre pp validate, validating rose-suite/app config files ..."
+    echo -e "\nRunning fre pp validate, validating rose-suite config file ..."
     fre -vv pp validate -e ${expname} -p ${plat} -t ${targ}
     check_exit_status "VALIDATE"
 

for_gh_runner/yaml_workflow/local_settings.yaml

@@ -1,8 +1,10 @@
 directories:
 ## to test staging data from archive, use the below
 #  history_dir:        "/archive/inl/test_cloud_runner_history_files"
+#
 # to have a faster turn-around (does not test archive data staging), use the below
-  history_dir:        "/work/inl/test_cloud_runner_history_files"
+  history_dir:         "/work/inl/test_cloud_runner_history_files"
+  refined_history_dir: "/work/d4s/test_cloud_runner_history_refined/proto4.0_c48l33o1degSPEAR"
   pp_dir:             !join [/work/$USER/test_cloud_runner_scratch_dir/, *FRE_STEM, /, pp]
 #  analysis_dir:       !join [/nbhome/$USER/, *FRE_STEM, /, *name]
   ptmp_dir:           "/xtmp/$USER/ptmp"
@@ -18,8 +20,17 @@ postprocess:
     pp_grid_spec:            *GRID_SPEC96_LOCAL
   switches:
     clean_work:              True
-    do_refinediag:           False
     do_atmos_plevel_masking: True
-    do_preanalysis:          False
     do_analysis:             False
     do_analysis_only:        False
+  preanalysis:
+    vitals:
+      script: "/work/d4s/test_cloud_runner_history_refined/preanalysis_script/gfdlvitals_refineDiag_om5.latest.csh"
+      inputs: ['atmos_month', 'aerosol_month']
+      do_preanalysis: True
+  refinediag:
+    ocean:
+      script: "/work/d4s/test_cloud_runner_history_refined/refinediag_script/refineDiag_atmos_FRE.csh"
+      inputs: ['ocean_monthly', 'ocean_bgc']
+      outputs: ['ocean_refined']
+      do_refinediag: True

for_gh_runner/yaml_workflow/settings.yaml

@@ -1,9 +1,10 @@
 # directories shared across tools
 directories:
-  history_dir:        "/mnt/history"
-  pp_dir:             "/mnt/pp"
+  history_dir:         "/mnt/history"
+  refined_history_dir: "/contrib-efs/container_test/refined_history" # "/mnt2/refined_history"
+  pp_dir:              "/mnt/pp"
 #  analysis_dir:       !join [/nbhome/$USER/, *FRE_STEM, /, *name]
-  ptmp_dir:           "/mnt/ptmp"
+  ptmp_dir:            "/mnt/ptmp"
 
 postprocess:
   settings:
@@ -15,8 +16,23 @@ postprocess:
     pp_grid_spec:            *GRID_SPEC96
   switches:
     clean_work:              True
-    do_refinediag:           False
     do_atmos_plevel_masking: True
-    do_preanalysis:          False
     do_analysis:             False
     do_analysis_only:        False
+
+  # to be first-step-user-scripts
+  preanalysis:
+    vitals:
+      script: "/contrib-efs/fre-workflows-runners/actions-runner-1/_work/fre-workflows/fre-workflows/for_gh_runner/dummy_preanalysis.csh"
+      inputs: ['atmos_month', 'aerosol_month'] #not used currently
+      # to be user-script_on
+      do_preanalysis: True
+
+  # to be refinediag-user-scripts
+  refinediag:
+    ocean:
+      script: "/contrib-efs/fre-workflows-runners/actions-runner-1/_work/fre-workflows/fre-workflows/for_gh_runner/dummy_refinediag.csh"
+      inputs: ['ocean_monthly', 'ocean_bgc'] # not used currently
+      outputs: ['ocean_refined']             # not used currently
+      # to be user_script_on
+      do_refinediag: True