Merge pull request #477 from eagles-project/mjs/ff_mxv

ff mam x validation for new compare script

Author: mjschmidt271

.github/workflows/gh_gcc-cpu.yml

@@ -75,7 +75,7 @@ jobs:
         # to get the array instead of a string, need the fromJSON()
         build-type: ${{ fromJSON(needs.define_matrix.outputs.build_type) }}
         fp-precision: ${{ fromJSON(needs.define_matrix.outputs.precision) }}
-
+    name: gcc-cpu / ${{ matrix.build-type }} - ${{ matrix.fp-precision }}
     runs-on: ${{ matrix.os }}
 
     # Environment variables

src/validation/aero_emissions/CMakeLists.txt

@@ -71,12 +71,13 @@ foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
 
   # add a test to run the skywalker driver
   add_test(run_${input} aero_emissions_driver ${AERO_EMISSIONS_VALIDATION_DIR}/${input}.yaml)
-  set_tests_properties(run_${input} PROPERTIES LABELS "${TestLabel}")
+  set_tests_properties(run_${input} PROPERTIES LABELS ${TestLabel})
 
   # add a test to validate mam4xx's results against the baseline.
   # Select a threshold error slightly bigger than the largest relative error for the threshold error.
   # compare_mam4xx_mam4.py <module1.py> <module2.py> <check_norms> <threshold error>
   add_test(validate_${input} python3 compare_mam4xx_mam4.py mam4xx_${input}.py mam_${input}.py True ${tol})
+  set_property(TEST validate_${input} PROPERTY SKIP_REGULAR_EXPRESSION "regex_fail_rel_tol")
   set_tests_properties(validate_${input} PROPERTIES DEPENDS run_${input})
   set_tests_properties(validate_${input} PROPERTIES LABELS ${TestLabel})
 endforeach()

src/validation/aero_model/CMakeLists.txt

@@ -33,24 +33,25 @@ set(TEST_LIST
     modal_aero_bcscavcoef_init_ts_0
     aero_model_wetdep_ts_379
     stand_aero_model_calcsize_water_uptake_dr_ts_379
-    baseline_aero_model_wetdep_ts_379 
+    baseline_aero_model_wetdep_ts_379
     )
 
-set(DEFAULT_TOL 1e-11)
 set(DEFAULT_TOL_BASELINE 1e-13)
 
 set(ERROR_THRESHOLDS  
-   5e-8
-   ${DEFAULT_TOL}
-   2e-4
-   2e-5
-   2e-5
-   ${DEFAULT_TOL_BASELINE}
+   5e-8                    # calc_1_impact_rate_ts_0
+   1e-11                   # modal_aero_bcscavcoef_get_ts_355
+   2e-4                    # modal_aero_bcscavcoef_init_ts_0
+   2e-5                    # aero_model_wetdep_ts_379
+   1e-9                    # stand_aero_model_calcsize_water_uptake_dr_ts_379
+   ${DEFAULT_TOL_BASELINE} # baseline_aero_model_wetdep_ts_379
    )
 
+set(TestLabel "aero_model")
+
 foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
   # copy the baseline file into place.
-  
+
   configure_file(
     ${AERO_MODEL_VALIDATION_DIR}/mam_${input}.py
     ${CMAKE_CURRENT_BINARY_DIR}/mam_${input}.py
@@ -59,11 +60,14 @@ foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
 
   # add a test to run the skywalker driver
   add_test(run_${input} aero_model_driver ${AERO_MODEL_VALIDATION_DIR}/${input}.yaml)
+  set_tests_properties(run_${input} PROPERTIES LABELS ${TestLabel})
 
   # add a test to validate mam4xx's results against the baseline.
   # Select a threshold error slightly bigger than the largest relative error for the threshold error.
   # compare_mam4xx_mam4.py <module1.py> <module2.py> <check_norms> <threshold error>
   add_test(validate_${input} python3 compare_mam4xx_mam4.py mam4xx_${input}.py mam_${input}.py True ${tol})
+  set_property(TEST validate_${input} PROPERTY SKIP_REGULAR_EXPRESSION "regex_fail_rel_tol")
   set_tests_properties(validate_${input} PROPERTIES DEPENDS run_${input})
+  set_tests_properties(validate_${input} PROPERTIES LABELS ${TestLabel})
 
 endforeach()

src/validation/aerosol_optics/CMakeLists.txt

@@ -57,24 +57,26 @@ set(TEST_LIST
     )
 # # matching the tests and errors, just for convenience
 
-set(DEFAULT_TOL 1e-13)
 set(ERROR_THRESHOLDS
-   2e-10
-   ${DEFAULT_TOL}
-   3e-10
-   3e-10
-   3e-10
-   2e-8
-   2e-8
-   6e-9
-   7e-11
-   2e-11
-   1e-12
-   7e-11
-   8e-11
-   8e-11
-   1e-14
+   2e-10 # binterp_ts_355
+   1e-13 # calc_diag_spec_ts_355
+   3e-10 # calc_refin_complex_ts_355_lw
+   3e-10 # calc_refin_complex_ts_355_sw
+   3e-10 # calc_volc_ext_ts_355
+   2e-8  # modal_size_parameters_ts_355_ismethod2_false
+   2e-8  # modal_size_parameters_ts_355_ismethod2_true
+   6e-9  # modal_aero_sw_ts_355
+   7e-11 # modal_aero_lw_ts_355
+   2e-11 # calc_parameterized_ts_355
+   1e-12 # update_aod_spec_ts_355
+   7e-11 # aer_rad_props_lw_ts_355
+   8e-11 # aer_rad_props_sw_ts_355
+   8e-11 # volcanic_cmip_sw_ts_355
+   1e-14 # data_transfer_state_q_qqwc_to_prog
    )
+
+set(TestLabel "aerosol_optics")
+
 foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
   # copy the baseline file into place.
 
@@ -86,10 +88,13 @@ foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
 
   # add a test to run the skywalker driver
   add_test(run_${input} aerosol_optics_driver ${AEROSOL_OPTICS_VALIDATION_DIR}/${input}.yaml)
+  set_tests_properties(run_${input} PROPERTIES LABELS ${TestLabel})
 
   # add a test to validate mam4xx's results against the baseline.
   # Select a threshold error slightly bigger than the largest relative error for the threshold error.
   # compare_mam4xx_mam4.py <module1.py> <module2.py> <check_norms> <threshold error>
   add_test(validate_${input} python3 compare_mam4xx_mam4.py mam4xx_${input}.py mam_${input}.py True ${tol})
+  set_property(TEST validate_${input} PROPERTY SKIP_REGULAR_EXPRESSION "regex_fail_rel_tol")
   set_tests_properties(validate_${input} PROPERTIES DEPENDS run_${input})
+  set_tests_properties(validate_${input} PROPERTIES LABELS ${TestLabel})
 endforeach()

src/validation/aging/CMakeLists.txt

@@ -29,13 +29,13 @@ set(TEST_LIST
     pcarbon_aging_1subarea
     )
 
-#set(DEFAULT_TOL 1e-9)
-
 set(ERROR_THRESHOLDS 8e-5 4e-4)
 
+set(TestLabel "aging")
+
 foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
   # copy the baseline file into place; is the skywalker file produced by fortran code?
-  
+
   configure_file(
     ${AGING_VALIDATION_DIR}/mam_${input}.py
     ${CMAKE_CURRENT_BINARY_DIR}/mam_${input}.py
@@ -44,12 +44,14 @@ foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
 
   # add a test to run the skywalker driver
   add_test(run_${input} aging_driver ${AGING_VALIDATION_DIR}/${input}.yaml)
+  set_tests_properties(run_${input} PROPERTIES LABELS ${TestLabel})
 
    # add a test to validate mam4xx's results against the baseline.
   # Select a threshold error slightly bigger than the largest relative error for the threshold error.
   # compare_mam4xx_mam4.py <module1.py> <module2.py> <check_norms> <threshold error>
   add_test(validate_${input} python3 compare_mam4xx_mam4.py mam4xx_${input}.py mam_${input}.py True ${tol})
+  set_property(TEST validate_${input} PROPERTY SKIP_REGULAR_EXPRESSION "regex_fail_rel_tol")
   set_tests_properties(validate_${input} PROPERTIES DEPENDS run_${input})
+  set_tests_properties(validate_${input} PROPERTIES LABELS ${TestLabel})
 
 endforeach()
-

src/validation/calcsize/CMakeLists.txt

@@ -43,19 +43,20 @@ set(TEST_LIST
     )
 # # matching the tests and errors, just for convenience
 
-set(DEFAULT_TOL 2e-6)
+set(DEFAULT_TOL 1e-12)
 set(ERROR_THRESHOLDS
-    1e-12
-    1e-12
-    2e-6
-    2e-6
-    1e-12
-    1e-12
-    5e-11
-    3e-5
-    1.5e-3
+    ${DEFAULT_TOL} # adjust_num_sizes_case1
+    ${DEFAULT_TOL} # adjust_num_sizes_case2
+    2e-6           # calcsize_e3sm
+    2e-6           # calcsize_sub
+    ${DEFAULT_TOL} # aitken_accum_exchange_case1
+    ${DEFAULT_TOL} # calcsize_compute_dry_volume
+    5e-11          # stand_modal_aero_calcsize_sub
+    3e-5           # stand_modal_aero_calcsize_sub_update_ptend
+    1.5e-3         # stand_calcsize_aero_model_wetdep_ts_379
    )
 
+set(TestLabel "calcsize")
 
 foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
   # copy the baseline file into place.
@@ -68,10 +69,13 @@ foreach(input tol IN ZIP_LISTS TEST_LIST ERROR_THRESHOLDS)
 
   # add a test to run the skywalker driver
   add_test(run_${input} calcsize_driver ${CALCSIZE_VALIDATION_DIR}/${input}.yaml)
+  set_tests_properties(run_${input} PROPERTIES LABELS ${TestLabel})
 
   # add a test to validate mam4xx's results against the baseline.
   # Select a threshold error slightly bigger than the largest relative error for the threshold error.
   # compare_mam4xx_mam4.py <module1.py> <module2.py> <check_norms> <threshold error>
   add_test(validate_${input} python3 compare_mam4xx_mam4.py mam4xx_${input}.py mam_${input}.py True ${tol})
+  set_property(TEST validate_${input} PROPERTY SKIP_REGULAR_EXPRESSION "regex_fail_rel_tol")
   set_tests_properties(validate_${input} PROPERTIES DEPENDS run_${input})
+  set_tests_properties(validate_${input} PROPERTIES LABELS ${TestLabel})
 endforeach()

src/validation/coagulation/CMakeLists.txt

@@ -12,7 +12,7 @@ add_executable(coagulation_driver
                coag_1subarea.cpp
                getcoags.cpp
                getcoags_wrapper_f.cpp
-               coag_aer_update.cpp 
+               coag_aer_update.cpp
                coag_num_update.cpp )
 target_link_libraries(coagulation_driver skywalker;validation;haero)
 
@@ -28,6 +28,8 @@ endforeach()
 
 # Run the driver in several configurations to produce datasets.
 
+set(TestLabel "coagulation")
+
 # Aging
 foreach (input
          coag_1subarea
@@ -46,9 +48,9 @@ foreach (input
 
   # add a test to run the skywalker driver
   add_test(run_${input} coagulation_driver ${COAGULATION_VALIDATION_DIR}/${input}.yaml)
-  
+  set_tests_properties(run_${input} PROPERTIES LABELS ${TestLabel})
+
   # add a test to validate mam4xx's results against the baseline.
   add_test(validate_${input} python3 compare_coag_validation.py mam4xx_${input}.py mam_${input}.py)
-  endforeach() 
-
-
+  set_tests_properties(validate_${input} PROPERTIES LABELS ${TestLabel})
+  endforeach()

src/validation/convproc/CMakeLists.txt

@@ -47,6 +47,8 @@ endforeach()
 
 # Run the driver in several configurations to produce datasets.
 
+set(TestLabel "convproc")
+
 # convproc
 foreach (input
          update_tendency_final
@@ -87,22 +89,25 @@ foreach (input
 
   # add a test to run the skywalker driver
   add_test(run_${input} convproc_driver ${CONVPROC_VALIDATION_DIR}/${input}.yaml)
+  set_tests_properties(run_${input} PROPERTIES LABELS ${TestLabel})
 
   # add a test to validate mam4xx's results against the baseline.
   # Select a threshold error slightly bigger than the largest relative error for the threshold error.
   # Conservative approach: At the time of this commit, threshold error=2e-6.
-  # compare_mam4xx_mam4.py <module1.py> <module2.py> <check_norms> <threshold error> 
-  if (${input} STREQUAL compute_activation_tend_do_act_true) 
+  # compare_mam4xx_mam4.py <module1.py> <module2.py> <check_norms> <threshold error>
+  if (${input} STREQUAL compute_activation_tend_do_act_true)
     set(TOL 6e-6)
-  elseif (${input} STREQUAL ma_activate_convproc_54) 
+  elseif (${input} STREQUAL ma_activate_convproc_54)
     set(TOL 6e-5)
-  elseif (${input} STREQUAL ma_convproc_tend) 
+  elseif (${input} STREQUAL ma_convproc_tend)
     set(TOL 1e-4)
-  elseif (${input} STREQUAL compute_updraft_mixing_ratio) 
+  elseif (${input} STREQUAL compute_updraft_mixing_ratio)
     set(TOL 2e-4)
   else ()
     set(TOL 2e-6)
   endif ()
   add_test(validate_${input} python3 compare_mam4xx_mam4.py mam4xx_${input}.py mam_${input}.py True ${TOL})
+  set_property(TEST validate_${input} PROPERTY SKIP_REGULAR_EXPRESSION "regex_fail_rel_tol")
   set_tests_properties(validate_${input} PROPERTIES DEPENDS run_${input})
+  set_tests_properties(validate_${input} PROPERTIES LABELS ${TestLabel})
 endforeach()

src/validation/drydep/CMakeLists.txt

@@ -31,6 +31,7 @@ target_link_libraries(drydep_driver skywalker;validation;haero)
      COPYONLY
    )
 
+set(TestLabel "drydep")
 # Run the driver in several configurations to produce datasets.
 function(add_drydep_test input_name)
   string(REPLACE "_input_" "_output_"  output_name ${input_name})
@@ -39,21 +40,24 @@ function(add_drydep_test input_name)
   configure_file(${DRYDEP_VALIDATION_DIR}/${output_name}.py ${CMAKE_CURRENT_BINARY_DIR}/${output_name}.py COPYONLY)
   # add a test to run the skywalker driver
   add_test(run_${test_name} drydep_driver ${DRYDEP_VALIDATION_DIR}/${input_name}.yaml)
+  set_tests_properties(run_${test_name} PROPERTIES LABELS "${TestLabel}")
   # add a test to validate mam4xx's results against the baseline.
   add_test(validate_${test_name} python3 compare_drydep.py mam4xx_${input_name}.py ${output_name}.py)
+  set_property(TEST validate_${test_name} PROPERTY SKIP_REGULAR_EXPRESSION "regex_fail_rel_tol")
   set_tests_properties(validate_${test_name} PROPERTIES DEPENDS run_${test_name})
+  set_tests_properties(validate_${test_name} PROPERTIES LABELS ${TestLabel})
 endfunction()
 
 foreach (input
-          gravit_settling_velocity
-          air_dynamic_viscosity
-          air_kinematic_viscosity
-          radius_for_moment
-          schmidt_number
-          slip_correction_factor
-          )
+         gravit_settling_velocity
+         air_dynamic_viscosity
+         air_kinematic_viscosity
+         radius_for_moment
+         schmidt_number
+         slip_correction_factor
+         )
   add_drydep_test(${input})
-endforeach() 
+endforeach()
 
 
 foreach (test_name
@@ -62,8 +66,8 @@ foreach (test_name
     depvel_part
     )
     foreach (ts
-	#1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 
-       1393 #1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 
+	     #1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
+       1393 #1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
        #1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416
         )
         foreach (jvlc 3 4)
@@ -72,36 +76,34 @@ foreach (test_name
             set(imode No_mode)
             set(input modal_aero_${test_name}_input_ts_${ts}_jvlc_${jvlc}_imnt_${imnt}_imode_${imode})
 	    add_drydep_test(${input})
-        endforeach() 
+        endforeach()
 
         foreach (jvlc 1 2)
 	    math(EXPR imnt "3 * ( ${jvlc} - 1)")
             foreach (imode 1 2 3 4)
                 set(input modal_aero_${test_name}_input_ts_${ts}_jvlc_${jvlc}_imnt_${imnt}_imode_${imode})
 	        add_drydep_test(${input})
-            endforeach() 
-        endforeach() 
-    endforeach() 
-endforeach() 
+            endforeach()
+        endforeach()
+    endforeach()
+endforeach()
 
 foreach (ts 1400 1401 1402 1403)# 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416)
     set(input calcram_input_ts_${ts})
     add_drydep_test(${input})
-endforeach() 
+endforeach()
 
 foreach (ts
-    1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 
-#   1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 
+    1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392
+#   1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
 #   1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416
           )
     set(test_name drydep)
     set(input aero_model_${test_name}_input_ts_${ts})
     add_drydep_test(${input})
-endforeach() 
+endforeach()
 
 set(test_name drydep)
 set(ts compute_tendencies)
 set(input aero_model_${test_name}_input_${ts})
 add_drydep_test(${input})
-
-