diff --git a/tests/integration/template_min_case_add_dependencies.cfg b/tests/integration/template_min_case_add_dependencies.cfg index 6f1a8e15..52436c2b 100644 --- a/tests/integration/template_min_case_add_dependencies.cfg +++ b/tests/integration/template_min_case_add_dependencies.cfg @@ -14,12 +14,12 @@ environment_commands = "#expand environment_commands#" input = #expand user_input_v3#/E3SMv3/#expand case_name# input_subdir = archive/atm/hist mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -output = "#expand user_output#zppy_min_case_add_dependencies_output/#expand unique_id#/#expand case_name#" +output = "#expand output_prefix#min_case_add_dependencies#expand output_suffix#" partition = "#expand partition_short#" qos = "#expand qos_short#" # ts is in 5 year increments ts_num_years = 5 -www = "#expand user_www#zppy_min_case_add_dependencies_www/#expand unique_id#" +www = "#expand www_prefix#min_case_add_dependencies#expand www_suffix#" # We want to produce diagnostics for 10 years. years = "1985:1995:10", diff --git a/tests/integration/test_min_cases.py b/tests/integration/test_min_cases.py new file mode 100644 index 00000000..9b041c89 --- /dev/null +++ b/tests/integration/test_min_cases.py @@ -0,0 +1,81 @@ +import os +from typing import Dict, List + +from tests.integration.utils import TestResults, check_mismatched_images, get_expansions + +V3_CASE_NAME = "v3.LR.historical_0051" +V2_CASE_NAME = "v2.LR.historical_0201" + + +def check_images(test_name: str, case_name: str, subdir: List[str]) -> TestResults: + # See docs/source/dev_guide/testing.rst for steps to run before running this test. + expansions = get_expansions() + # TODO: Add expected results to expected_dir_min_case + expected_dir = expansions["expected_dir_min_case"] + user_www = expansions["user_www"] + unique_id = expansions["unique_id"] + # TODO: Create a script that will launch ALL the min case cfgs, similar to https://github.com/E3SM-Project/zppy/pull/520 + actual_images_dir = ( + f"{user_www}/zppy_test_results/{unique_id}/zppy_{test_name}_www/{case_name}/" + ) + + # The expected_images_file lists all images we expect to compare. + expected_images_file = f"{expected_dir}image_list_expected_{test_name}.txt" + expected_images_dir = f"{expected_dir}expected_{test_name}" + + # The directory to place differences in. + diff_dir = f"{actual_images_dir}image_check_failures_{test_name}" + + test_results = check_mismatched_images( + actual_images_dir, + expected_images_file, + expected_images_dir, + diff_dir, + subdir, + ) + if os.path.exists(f"{diff_dir}/missing_images.txt"): + os.remove(f"{diff_dir}/missing_images.txt") + if os.path.exists(f"{diff_dir}/mismatched_images.txt"): + os.remove(f"{diff_dir}/mismatched_images.txt") + for missing_image in test_results.file_list_missing: + with open(f"{diff_dir}/missing_images.txt", "a") as f: + f.write(f"{missing_image}\n") + for mismatched_image in test_results.file_list_mismatched: + with open(f"{diff_dir}/mismatched_images.txt", "a") as f: + f.write(f"{mismatched_image}\n") + return test_results + + +def construct_markdown_summary_table( + test_results_dict: Dict[str, TestResults], output_file_path: str +): + with open(output_file_path, "w") as f: + f.write("# Summary of test results\n\n") + f.write( + "| Test name | Total images | Missing images | Mismatched images | Correct images |\n" + ) + f.write("| --- | --- | --- | --- | --- |\n") + for test_name, test_results in test_results_dict.items(): + f.write( + f"| {test_name} | {test_results.image_count_total} | {test_results.image_count_missing} | {test_results.image_count_mismatched} | {test_results.image_count_correct} |\n" + ) + + +# Run with: +# pytest tests/integration/test_min_cases.py +def test_all_min_cases(): + test_results_dict: Dict[str, TestResults] = dict() + + # TODO: add test lines for ALL min case cfgs + test_results = check_images( + "min_case_add_dependencies", + V3_CASE_NAME, + ["e3sm_diags", "global_time_series", "ilamb"], + ) + test_results_dict["min_case_add_dependencies"] = test_results + + construct_markdown_summary_table(test_results_dict, "min_case_summary.md") + print("Copy output of min_case_summary.md to a PR comment.") + # TODO: create an image diff grid, as in https://github.com/E3SM-Project/zppy/pull/685, for each min case + for tr in test_results_dict.values(): + assert tr.image_count_total == tr.image_count_correct diff --git a/tests/integration/utils.py b/tests/integration/utils.py index d2613d70..702e7ef9 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -12,6 +12,26 @@ # Image checking ########################################################## +# Originally in https://github.com/E3SM-Project/zppy/pull/695 +class TestResults(object): + def __init__( + self, + diff_dir: str, + image_count_total: int, + file_list_missing: List[str], + file_list_mismatched: List[str], + ): + self.diff_dir = diff_dir + self.image_count_total = image_count_total + self.image_count_missing = len(file_list_missing) + self.image_count_mismatched = len(file_list_mismatched) + self.image_count_correct = ( + image_count_total - len(file_list_missing) - len(file_list_mismatched) + ) + self.file_list_missing = sorted(file_list_missing) + self.file_list_mismatched = sorted(file_list_mismatched) + + # Copied from E3SM Diags def compare_images( missing_images, @@ -92,12 +112,12 @@ def compare_images( def check_mismatched_images( - actual_images_dir, - expected_images_file, - expected_images_dir, - diff_dir, - subdirs_to_check, -): + actual_images_dir: str, + expected_images_file: str, + expected_images_dir: str, + diff_dir: str, + subdirs_to_check: List[str], +) -> TestResults: missing_images: List[str] = [] mismatched_images: List[str] = [] @@ -142,6 +162,7 @@ def check_mismatched_images( print( f"Number of correct images: {counter - len(missing_images) - len(mismatched_images)}" ) + test_results = TestResults(diff_dir, counter, missing_images, mismatched_images) # Make diff_dir readable if os.path.exists(diff_dir): @@ -151,8 +172,7 @@ def check_mismatched_images( # That is, if we're in this case, we expect the following: assert len(missing_images) == counter - assert missing_images == [] - assert mismatched_images == [] + return test_results # Multi-machine testing ########################################################## @@ -175,6 +195,7 @@ def get_chyrsalis_expansions(config): "diags_walltime": "5:00:00", "environment_commands_test": "", "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/", + "expected_dir_min_case": "/lcrc/group/e3sm/ac.forsyth2/zppy_min_case_resources/", "global_time_series_environment_commands": "source /conda.sh; conda activate ", "mpas_analysis_walltime": "00:30:00", "partition_long": "compute", @@ -204,6 +225,7 @@ def get_compy_expansions(config): "diags_walltime": "03:00:00", "environment_commands_test": "", "expected_dir": "/compyfs/www/zppy_test_resources/", + "expected_dir_min_case": "", "global_time_series_environment_commands": "source /conda.sh; conda activate ", "mpas_analysis_walltime": "00:30:00", "partition_long": "slurm", @@ -233,6 +255,7 @@ def get_perlmutter_expansions(config): "diags_walltime": "6:00:00", "environment_commands_test": "", "expected_dir": "/global/cfs/cdirs/e3sm/www/zppy_test_resources/", + "expected_dir_min_case": "", "global_time_series_environment_commands": "source /conda.sh; conda activate ", "mpas_analysis_walltime": "01:00:00", "partition_long": "", @@ -276,6 +299,15 @@ def substitute_expansions(expansions, file_in, file_out): while match_object is not None: expansion_indicator = match_object.group(0) expansion_name = match_object.group(1) + # TODO: add these prefixes and suffixes to ALL min case cfg's + if expansion_name == "output_prefix": + expansion = f"{expansions['user_output']}zppy_test_results/{expansions['unique_id']}/zppy_" + elif expansion_name == "output_suffix": + expansion = f"_output/{expansions['case_name']}" + elif expansion_name == "www_prefix": + expansion = f"{expansions['user_www']}zppy_test_results/{expansions['unique_id']}/zppy_" + else: + expansion = "_www/" # No case_name here expansion = expansions[expansion_name] line = line.replace(expansion_indicator, expansion) match_object = re.search("#expand ([^#]*)#", line)