Merge branch 'NOAA-EMC:develop' into feature/GW-RT_GlobalNML

Author: dpsarmie

.github/CODEOWNERS

@@ -208,3 +208,6 @@ ush/python/pygfs/ufswm/gfs.py @aerorahul
 ush/python/pygfs/ufswm/ufs.py @aerorahul
 ush/python/pygfs/utils/__init__.py @aerorahul
 ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA
+
+# Specific workflow scripts
+workflow/generate_workflows.sh @DavidHuber-NOAA

.github/pull_request_template.md

@@ -72,4 +72,5 @@ Example:
 - [ ] My changes generate no new warnings
 - [ ] New and existing tests pass with my changes
 - [ ] This change is covered by an existing CI test or a new one has been added
+- [ ] Any new scripts have been added to the .github/CODEOWNERS file with owners
 - [ ] I have made corresponding changes to the system documentation if necessary

ci/Jenkinsfile

@@ -5,7 +5,7 @@ def cases = ''
 def GH = 'none'
 // Location of the custom workspaces for each machine in the CI system.  They are persitent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
 def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
 def STATUS = 'Passed'
 
@@ -101,9 +101,10 @@ pipeline {
                 stages {
                     stage('Building') {
                         steps {
-                           catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                          catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                             script {
                                 def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system
+                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                 sh(script: "mkdir -p ${HOMEgfs}")
                                 ws(HOMEgfs) {
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
@@ -172,9 +173,10 @@ pipeline {
                                     }
                                     if (system == 'gfs') {
                                         cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                                        echo "Cases to run: ${cases}"
                                     }
                                }
-                           }
+                            }
                           }
                         }
                     }
@@ -192,32 +194,34 @@ pipeline {
                     def parallelStages = cases.collectEntries { caseName ->
                         ["${caseName}": {
                             stage("Create ${caseName}") {
-                                    catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-                                        script {
-                                                sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
-                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
-                                                system = yaml_case.experiment.system
-                                                def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
-                                                env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
-                                                try {
-                                                    error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
-                                                } catch (Exception error_create) {
-                                                    sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
-                                                    error("Case ${caseName} failed to create experment directory")
-                                                }
+                                script {
+                                        sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
+                                        def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                        def build_system = yaml_case.experiment.system
+                                        def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}"   // local HOMEgfs is used to populate the XML on per system basis
+                                        env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
+                                        env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
+                                        try {
+                                              error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
+                                        } catch (Exception error_create) {
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
+                                            error("Case ${caseName} failed to create experment directory")
                                         }
-                                }
+                                     }
                             }
 
                             stage("Running ${caseName}") {
                                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                                         script {
                                                 HOMEgfs = "${CUSTOM_WORKSPACE}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
+                                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                                 def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
+                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                                def build_system = yaml_case.experiment.system
                                                 try {
-                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {
@@ -268,11 +272,11 @@ pipeline {
             }
         }
 
-
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
             steps {
                 script {
+                    env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                     sh(script: """
                         labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name')
                         for label in \$labels; do

ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml

@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 384
   resdetocean: 0.25
   nens: 0
-  gfs_cyc: 4
+  interval: 6
   start: cold
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -9,7 +9,7 @@ arguments:
   resdetocean: 5.0
   resensatmos: 48
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: cold
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -13,7 +13,7 @@ arguments:
   idate: 2021032412
   edate: 2021032418
   nens: 0
-  gfs_cyc: 0
+  interval: 0
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -14,6 +14,6 @@ arguments:
   idate: 2021122018
   edate: 2021122106
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -13,11 +13,12 @@ arguments:
   idate: 2021122012
   edate: 2021122100
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml
 
 skip_ci_on_hosts:
+  - wcoss2
   - orion
   - gaea
   - hercules

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -13,7 +13,7 @@ arguments:
   idate: 2024022318
   edate: 2024022406
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
 

ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml

@@ -9,11 +9,14 @@ arguments:
   resdetocean: 1.0
   resensatmos: 96
   nens: 2
-  gfs_cyc: 1
+  interval: 6
   start: warm
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2020110100
   edate: 2020110100
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_replay_ci.yaml
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610
+
+skip_ci_on_hosts:
+  - wcoss2