Automate testing of specific cases

tests/integration/template_min_case_add_dependencies.cfg

@@ -14,12 +14,12 @@ environment_commands = "#expand environment_commands#"
 input = #expand user_input_v3#/E3SMv3/#expand case_name#
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-output = "#expand user_output#zppy_min_case_add_dependencies_output/#expand unique_id#/#expand case_name#"
+output = "#expand output_prefix#min_case_add_dependencies#expand output_suffix#"
 partition = "#expand partition_short#"
 qos = "#expand qos_short#"
 # ts is in 5 year increments
 ts_num_years = 5
-www = "#expand user_www#zppy_min_case_add_dependencies_www/#expand unique_id#"
+www = "#expand www_prefix#min_case_add_dependencies#expand www_suffix#"
 # We want to produce diagnostics for 10 years.
 years = "1985:1995:10",
 

tests/integration/test_min_cases.py

@@ -0,0 +1,81 @@
+import os
+from typing import Dict, List
+
+from tests.integration.utils import TestResults, check_mismatched_images, get_expansions
+
+V3_CASE_NAME = "v3.LR.historical_0051"
+V2_CASE_NAME = "v2.LR.historical_0201"
+
+
+def check_images(test_name: str, case_name: str, subdir: List[str]) -> TestResults:
+    # See docs/source/dev_guide/testing.rst for steps to run before running this test.
+    expansions = get_expansions()
+    # TODO: Add expected results to expected_dir_min_case
+    expected_dir = expansions["expected_dir_min_case"]
+    user_www = expansions["user_www"]
+    unique_id = expansions["unique_id"]
+    # TODO: Create a script that will launch ALL the min case cfgs, similar to https://github.com/E3SM-Project/zppy/pull/520
+    actual_images_dir = (
+        f"{user_www}/zppy_test_results/{unique_id}/zppy_{test_name}_www/{case_name}/"
+    )
+
+    # The expected_images_file lists all images we expect to compare.
+    expected_images_file = f"{expected_dir}image_list_expected_{test_name}.txt"
+    expected_images_dir = f"{expected_dir}expected_{test_name}"
+
+    # The directory to place differences in.
+    diff_dir = f"{actual_images_dir}image_check_failures_{test_name}"
+
+    test_results = check_mismatched_images(
+        actual_images_dir,
+        expected_images_file,
+        expected_images_dir,
+        diff_dir,
+        subdir,
+    )
+    if os.path.exists(f"{diff_dir}/missing_images.txt"):
+        os.remove(f"{diff_dir}/missing_images.txt")
+    if os.path.exists(f"{diff_dir}/mismatched_images.txt"):
+        os.remove(f"{diff_dir}/mismatched_images.txt")
+    for missing_image in test_results.file_list_missing:
+        with open(f"{diff_dir}/missing_images.txt", "a") as f:
+            f.write(f"{missing_image}\n")
+    for mismatched_image in test_results.file_list_mismatched:
+        with open(f"{diff_dir}/mismatched_images.txt", "a") as f:
+            f.write(f"{mismatched_image}\n")
+    return test_results
+
+
+def construct_markdown_summary_table(
+    test_results_dict: Dict[str, TestResults], output_file_path: str
+):
+    with open(output_file_path, "w") as f:
+        f.write("# Summary of test results\n\n")
+        f.write(
+            "| Test name | Total images | Missing images | Mismatched images | Correct images |\n"
+        )
+        f.write("| --- | --- | --- | --- | --- |\n")
+        for test_name, test_results in test_results_dict.items():
+            f.write(
+                f"| {test_name} | {test_results.image_count_total} | {test_results.image_count_missing} | {test_results.image_count_mismatched} | {test_results.image_count_correct} |\n"
+            )
+
+
+# Run with:
+# pytest tests/integration/test_min_cases.py
+def test_all_min_cases():
+    test_results_dict: Dict[str, TestResults] = dict()
+
+    # TODO: add test lines for ALL min case cfgs
+    test_results = check_images(
+        "min_case_add_dependencies",
+        V3_CASE_NAME,
+        ["e3sm_diags", "global_time_series", "ilamb"],
+    )
+    test_results_dict["min_case_add_dependencies"] = test_results
+
+    construct_markdown_summary_table(test_results_dict, "min_case_summary.md")
+    print("Copy output of min_case_summary.md to a PR comment.")
+    # TODO: create an image diff grid, as in https://github.com/E3SM-Project/zppy/pull/685, for each min case
+    for tr in test_results_dict.values():
+        assert tr.image_count_total == tr.image_count_correct

tests/integration/utils.py

@@ -12,6 +12,26 @@
 # Image checking ##########################################################
 
 
+# Originally in https://github.com/E3SM-Project/zppy/pull/695
+class TestResults(object):
+    def __init__(
+        self,
+        diff_dir: str,
+        image_count_total: int,
+        file_list_missing: List[str],
+        file_list_mismatched: List[str],
+    ):
+        self.diff_dir = diff_dir
+        self.image_count_total = image_count_total
+        self.image_count_missing = len(file_list_missing)
+        self.image_count_mismatched = len(file_list_mismatched)
+        self.image_count_correct = (
+            image_count_total - len(file_list_missing) - len(file_list_mismatched)
+        )
+        self.file_list_missing = sorted(file_list_missing)
+        self.file_list_mismatched = sorted(file_list_mismatched)
+
+
 # Copied from E3SM Diags
 def compare_images(
     missing_images,
@@ -92,12 +112,12 @@ def compare_images(
 
 
 def check_mismatched_images(
-    actual_images_dir,
-    expected_images_file,
-    expected_images_dir,
-    diff_dir,
-    subdirs_to_check,
-):
+    actual_images_dir: str,
+    expected_images_file: str,
+    expected_images_dir: str,
+    diff_dir: str,
+    subdirs_to_check: List[str],
+) -> TestResults:
     missing_images: List[str] = []
     mismatched_images: List[str] = []
 
@@ -142,6 +162,7 @@ def check_mismatched_images(
     print(
         f"Number of correct images: {counter - len(missing_images) - len(mismatched_images)}"
     )
+    test_results = TestResults(diff_dir, counter, missing_images, mismatched_images)
 
     # Make diff_dir readable
     if os.path.exists(diff_dir):
@@ -151,8 +172,7 @@ def check_mismatched_images(
         # That is, if we're in this case, we expect the following:
         assert len(missing_images) == counter
 
-    assert missing_images == []
-    assert mismatched_images == []
+    return test_results
 
 
 # Multi-machine testing ##########################################################
@@ -175,6 +195,7 @@ def get_chyrsalis_expansions(config):
         "diags_walltime": "5:00:00",
         "environment_commands_test": "",
         "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/",
+        "expected_dir_min_case": "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_resources/",
         "global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
         "mpas_analysis_walltime": "00:30:00",
         "partition_long": "compute",
@@ -204,6 +225,7 @@ def get_compy_expansions(config):
         "diags_walltime": "03:00:00",
         "environment_commands_test": "",
         "expected_dir": "/compyfs/www/zppy_test_resources/",
+        "expected_dir_min_case": "",
         "global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
         "mpas_analysis_walltime": "00:30:00",
         "partition_long": "slurm",
@@ -233,6 +255,7 @@ def get_perlmutter_expansions(config):
         "diags_walltime": "6:00:00",
         "environment_commands_test": "",
         "expected_dir": "/global/cfs/cdirs/e3sm/www/zppy_test_resources/",
+        "expected_dir_min_case": "",
         "global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
         "mpas_analysis_walltime": "01:00:00",
         "partition_long": "",
@@ -276,6 +299,15 @@ def substitute_expansions(expansions, file_in, file_out):
                 while match_object is not None:
                     expansion_indicator = match_object.group(0)
                     expansion_name = match_object.group(1)
+                    # TODO: add these prefixes and suffixes to ALL min case cfg's
+                    if expansion_name == "output_prefix":
+                        expansion = f"{expansions['user_output']}zppy_test_results/{expansions['unique_id']}/zppy_"
+                    elif expansion_name == "output_suffix":
+                        expansion = f"_output/{expansions['case_name']}"
+                    elif expansion_name == "www_prefix":
+                        expansion = f"{expansions['user_www']}zppy_test_results/{expansions['unique_id']}/zppy_"
+                    else:
+                        expansion = "_www/"  # No case_name here
                     expansion = expansions[expansion_name]
                     line = line.replace(expansion_indicator, expansion)
                     match_object = re.search("#expand ([^#]*)#", line)