Bugfix in compare_to_kgo (#112)

* Update continuous_integration.yml

* Update cosp_config.F90

* Update continuous_integration.yml

* Update continuous_integration.yml

* Update containerized-ci.yml

* Update to KGOs v005 for containerized-ci.

* v005 KGO for containerized-ci.

* md5 sums for v005 ifort and ifx KGOs.

* gfortran KGOs v004.

* Undo version upgrade of KGOs in CI tests.

* Undo changes in CI workflows.

* Add extra bin to force error.

* Trap failure in summary_stats.

* Trap error in difference and y tick labels.

* Always test against KGOs.

* Always test against KGOs.

* Undo extra ice cloud bin.

* Retain original formatting.

Author: alejandrobodas

.github/workflows/containerized-ci.yml

@@ -117,6 +117,7 @@ jobs:
         python compare_to_kgo.py ${KGO} ${TST} --atol=${ATOL} --rtol=${RTOL} --stats_file=${STATS}
     # 2. UM global snapshot.
     - name: UM global against known good output (KGO)
+      if: always()
       run: |
         cd driver
         KGO=data/outputs/UKMO/cosp2_output.um_global.${{ matrix.compiler }}.kgo.$KGO_VERSION.nc

.github/workflows/continuous_integration.yml

@@ -151,18 +151,20 @@ jobs:
         python compare_to_kgo.py ${KGO} ${TST} --atol=${ATOL} --rtol=${RTOL}
     # 2. UM global snapshot. 
     - name: UM global against known good output (KGO)
+      if: always()
       run: |
         cd driver
         KGO=data/outputs/UKMO/cosp2_output.um_global.${F90_SHORT_NAME}.kgo.$KGO_VERSION.nc
         TST=data/outputs/UKMO/cosp2_output.um_global.nc
         python compare_to_kgo.py ${KGO} ${TST} --atol=${ATOL} --rtol=${RTOL}
     # 3. UM global snapshot. Diagnostics on model levels.
     - name: UM global on model levels against known good output (KGO)
+      if: always()
       run: |
         cd driver
         KGO=data/outputs/UKMO/cosp2_output.um_global_model_levels.${F90_SHORT_NAME}.kgo.$KGO_VERSION.nc
         TST=data/outputs/UKMO/cosp2_output.um_global_model_levels.nc
-          python compare_to_kgo.py ${KGO} ${TST} --atol=${ATOL} --rtol=${RTOL}
+        python compare_to_kgo.py ${KGO} ${TST} --atol=${ATOL} --rtol=${RTOL}
     ###############################################################################
     # Produce plots when it fails during global snapshot tests,
     # and create a tarball with outputs.

driver/compare_to_kgo.py

@@ -70,7 +70,11 @@ def calculate_stats(tst, kgo, atol=0.0, rtol=None):
     """
     summary_stats = {'N':0, 'AvgDiff':0.0, 'MinDiff':0.0, 'MaxDiff':0.0, 'StDev':0.0}
     # All differences
-    d = tst - kgo
+    try:
+        d = tst - kgo
+    except:
+        print("Error: arrays have different shapes.")
+        d = kgo
     # Mask for differences larger than absolute tolerance
     maskAllDiff = (np.absolute(d) > atol)
     NallDiff = maskAllDiff.sum()

driver/data/outputs/UKMO/cosp2_output.um_global.gfortran.kgo.v004.nc.md5

@@ -0,0 +1 @@
+71cb2284eb577bc2facec5f28520fdef  cosp2_output.um_global.gfortran.kgo.v004.nc

driver/data/outputs/UKMO/cosp2_output.um_global.ifort.kgo.v005.nc.md5

@@ -0,0 +1 @@
+7ffad70c1e7107371c9a65b7d9c3d37e  cosp2_output.um_global.ifort.kgo.v005.nc

driver/data/outputs/UKMO/cosp2_output.um_global.ifx.kgo.v005.nc.md5

@@ -0,0 +1 @@
+922e12f2615cdd2fbe3c70cf051e7764  cosp2_output.um_global.ifx.kgo.v005.nc

driver/data/outputs/UKMO/cosp2_output.um_global_model_levels.gfortran.kgo.v004.nc.md5

@@ -0,0 +1 @@
+82d35194587672384d5a5d1ad0a84fb2  cosp2_output.um_global_model_levels.gfortran.kgo.v004.nc

driver/data/outputs/UKMO/cosp2_output_um.gfortran.kgo.v004.nc.md5

@@ -0,0 +1 @@
+886365174837f02220e17ec64b601cce  cosp2_output_um.gfortran.kgo.v004.nc

driver/data/outputs/UKMO/cosp2_output_um.ifort.kgo.v005.nc.md5

@@ -0,0 +1 @@
+57bef996ced3ee9553fd5cceb20c8c21  cosp2_output_um.ifort.kgo.v005.nc

driver/data/outputs/UKMO/cosp2_output_um.ifx.kgo.v005.nc.md5

@@ -0,0 +1 @@
+acca99a51056885026fb2fd9f23ca295  cosp2_output_um.ifx.kgo.v005.nc