[Feature]: Adapt CDAT Migration regression testing scripts as integration tests to replace image checker

.coveragerc

@@ -0,0 +1,3 @@
+[report]
+exclude_also =
+    if TYPE_CHECKING:

.github/workflows/build_workflow.yml

@@ -5,7 +5,7 @@ on:
     branches: [main]
 
   pull_request:
-    branches: [main]
+    branches: [main, cdat-migration-fy24]
 
   workflow_dispatch:
 
@@ -29,10 +29,10 @@ jobs:
         uses: actions/checkout@v3
 
       - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
-        name: Set up Python 3.10
+        name: Set up Python 3.11
         uses: actions/setup-python@v3
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
         # Run all pre-commit hooks on all the files.
@@ -50,7 +50,7 @@ jobs:
         shell: bash -l {0}
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@master

.gitignore

@@ -110,6 +110,7 @@ ENV/
 
 # NetCDF files needed
 !e3sm_diags/driver/acme_ne30_ocean_land_mask.nc
+!auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/*.nc
 
 # Folder for storing quality assurance files and notes
 qa/

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 exclude: "docs|node_modules|migrations|.git|.tox|examples|analysis_data_preprocess|auxiliary_tools|conda/meta.yaml|e3sm_diags/driver/utils/zwf_functions.py"
-default_stages: [commit]
+default_stages: [pre-commit]
 fail_fast: true
 
 repos:
@@ -34,4 +34,5 @@ repos:
     hooks:
       - id: mypy
         args: [--config=pyproject.toml]
-        additional_dependencies: [dask, numpy>=1.23.0, types-PyYAML]
+        additional_dependencies:
+          [dask, numpy>=1.23.0, xarray>=2023.3.0, types-PyYAML]

.vscode/e3sm_diags.code-workspace

@@ -58,7 +58,7 @@
     "configurations": [
       {
         "name": "Python: Current File",
-        "type": "python",
+        "type": "debugpy",
         "request": "launch",
         "program": "${file}",
         "console": "integratedTerminal",

auxiliary_tools/aerosol_budget.py

@@ -1,3 +1,5 @@
+# NOTE: This module uses the deprecated e3sm_diags.driver.utils.dataset.Dataset
+# class, which was replaced by e3sm_diags.driver.utils.dataset_xr.Dataset.
 import e3sm_diags
 from e3sm_diags.driver import utils
 import cdms2
@@ -12,11 +14,12 @@
 
 
 def global_integral(var, area_m2):
-    """ Compute global integral of 2 dimentional properties"""
-    return numpy.sum(numpy.sum(abs(var)*area_m2,axis = 0), axis=0)
+    """Compute global integral of 2 dimentional properties"""
+    return numpy.sum(numpy.sum(abs(var) * area_m2, axis=0), axis=0)
+
 
 def calc_column_integral(data, aerosol, season):
-    """ Calculate column integrated mass """
+    """Calculate column integrated mass"""
 
     # take aerosol and change it to the appropriate string
     # ncl -> SEASALT, dst -> DUST, rest1 -> REST1
@@ -32,129 +35,129 @@ def calc_column_integral(data, aerosol, season):
         burden = data.get_climo_variable(f"ABURDEN{aerosol_name}", season)
     except RuntimeError:
         # if not, use the Mass_ terms and integrate over the column
-        mass = data.get_climo_variable(f'Mass_{aerosol}', season)
+        mass = data.get_climo_variable(f"Mass_{aerosol}", season)
         hyai, hybi, ps = data.get_extra_variables_only(
-            f'Mass_{aerosol}', season, extra_vars=["hyai", "hybi", "PS"]
+            f"Mass_{aerosol}", season, extra_vars=["hyai", "hybi", "PS"]
         )
 
         p0 = 100000.0  # Pa
-        ps = ps   # Pa
-        pressure_levs = cdutil.vertical.reconstructPressureFromHybrid(ps, hyai, hybi, p0)
+        ps = ps  # Pa
+        pressure_levs = cdutil.vertical.reconstructPressureFromHybrid(
+            ps, hyai, hybi, p0
+        )
 
-        #(72,lat,lon)
-        delta_p = numpy.diff(pressure_levs,axis = 0)
-        mass_3d = mass*delta_p/9.8 #mass density * mass air   kg/m2
-        burden = numpy.nansum(mass_3d,axis = 0)   #kg/m2
+        # (72,lat,lon)
+        delta_p = numpy.diff(pressure_levs, axis=0)
+        mass_3d = mass * delta_p / 9.8  # mass density * mass air   kg/m2
+        burden = numpy.nansum(mass_3d, axis=0)  # kg/m2
     return burden
-
+
+
 def generate_metrics_dic(data, aerosol, season):
     metrics_dict = {}
-    wetdep = data.get_climo_variable(f'{aerosol}_SFWET', season)
-    drydep = data.get_climo_variable(f'{aerosol}_DDF', season)
-    srfemis = data.get_climo_variable(f'SF{aerosol}', season)
-    area = data.get_extra_variables_only(
-                f'{aerosol}_DDF', season, extra_vars=["area"]
-            )
+    wetdep = data.get_climo_variable(f"{aerosol}_SFWET", season)
+    drydep = data.get_climo_variable(f"{aerosol}_DDF", season)
+    srfemis = data.get_climo_variable(f"SF{aerosol}", season)
+    area = data.get_extra_variables_only(f"{aerosol}_DDF", season, extra_vars=["area"])
     area_m2 = area * REARTH**2
 
     burden = calc_column_integral(data, aerosol, season)
-    burden_total= global_integral(burden, area_m2)*1e-9 # kg to Tg
-    print(f'{aerosol} Burden (Tg): ',f'{burden_total:.3f}')
-    sink = global_integral((drydep-wetdep),area_m2)*UNITS_CONV
-    drydep = global_integral(drydep,area_m2)*UNITS_CONV
-    wetdep = global_integral(wetdep,area_m2)*UNITS_CONV
-    srfemis = global_integral(srfemis,area_m2)*UNITS_CONV
-    print(f'{aerosol} Sink (Tg/year): ',f'{sink:.3f}')
-    print(f'{aerosol} Lifetime (days): ',f'{burden_total/sink*365:.3f}')
+    burden_total = global_integral(burden, area_m2) * 1e-9  # kg to Tg
+    print(f"{aerosol} Burden (Tg): ", f"{burden_total:.3f}")
+    sink = global_integral((drydep - wetdep), area_m2) * UNITS_CONV
+    drydep = global_integral(drydep, area_m2) * UNITS_CONV
+    wetdep = global_integral(wetdep, area_m2) * UNITS_CONV
+    srfemis = global_integral(srfemis, area_m2) * UNITS_CONV
+    print(f"{aerosol} Sink (Tg/year): ", f"{sink:.3f}")
+    print(f"{aerosol} Lifetime (days): ", f"{burden_total/sink*365:.3f}")
     metrics_dict = {
-    "Surface Emission (Tg/yr)": f'{srfemis:.3f}',
-    "Sink (Tg/yr)": f'{sink:.3f}',
-    "Dry Deposition (Tg/yr)": f'{drydep:.3f}',
-    "Wet Deposition (Tg/yr)": f'{wetdep:.3f}',
-    "Burden (Tg)": f'{burden_total:.3f}',
-    "Lifetime (Days)": f'{burden_total/sink*365:.3f}',
+        "Surface Emission (Tg/yr)": f"{srfemis:.3f}",
+        "Sink (Tg/yr)": f"{sink:.3f}",
+        "Dry Deposition (Tg/yr)": f"{drydep:.3f}",
+        "Wet Deposition (Tg/yr)": f"{wetdep:.3f}",
+        "Burden (Tg)": f"{burden_total:.3f}",
+        "Lifetime (Days)": f"{burden_total/sink*365:.3f}",
     }
     return metrics_dict
 
+
 param = CoreParameter()
-param.test_name = 'v2.LR.historical_0101'
-param.test_name = 'F2010.PD.NGD_v3atm.0096484.compy'
-param.test_data_path = '/Users/zhang40/Documents/ACME_simulations/'
-param.test_data_path = '/compyfs/mahf708/E3SMv3_dev/F2010.PD.NGD_v3atm.0096484.compy/post/atm/180x360_aave/clim/10yr'
+param.test_name = "v2.LR.historical_0101"
+param.test_name = "F2010.PD.NGD_v3atm.0096484.compy"
+param.test_data_path = "/Users/zhang40/Documents/ACME_simulations/"
+param.test_data_path = "/compyfs/mahf708/E3SMv3_dev/F2010.PD.NGD_v3atm.0096484.compy/post/atm/180x360_aave/clim/10yr"
 test_data = utils.dataset.Dataset(param, test=True)
 
-#rearth = 6.37122e6 #km
-#UNITS_CONV = 86400.0*365.0*1e-9 # kg/s to Tg/yr
-REARTH = 6.37122e6 #km
-UNITS_CONV = 86400.0*365.0*1e-9 # kg/s to Tg/yr
+# rearth = 6.37122e6 #km
+# UNITS_CONV = 86400.0*365.0*1e-9 # kg/s to Tg/yr
+REARTH = 6.37122e6  # km
+UNITS_CONV = 86400.0 * 365.0 * 1e-9  # kg/s to Tg/yr
 # TODO:
 # Convert so4 unit to TgS
-#mwso4 = 115.0
-#mws = 32.066
-#UNITS_CONV_S = UNITS_CONV/mwso4*mws # kg/s to TgS/yr
+# mwso4 = 115.0
+# mws = 32.066
+# UNITS_CONV_S = UNITS_CONV/mwso4*mws # kg/s to TgS/yr
 
 
-species = ["bc", "dst", "mom", "ncl","pom","so4","soa"]
-SPECIES_NAMES = {"bc": "Black Carbon",
+species = ["bc", "dst", "mom", "ncl", "pom", "so4", "soa"]
+SPECIES_NAMES = {
+    "bc": "Black Carbon",
     "dst": "Dust",
     "mom": "Marine Organic Matter",
     "ncl": "Sea Salt",
     "pom": "Primary Organic Matter",
     "so4": "Sulfate",
-    "soa": "Secondary Organic Aerosol"}
+    "soa": "Secondary Organic Aerosol",
+}
 MISSING_VALUE = 999.999
 metrics_dict = {}
 metrics_dict_ref = {}
 
 seasons = ["ANN"]
 
 ref_data_path = os.path.join(
-        e3sm_diags.INSTALL_PATH,
-        "control_runs",
-        "aerosol_global_metrics_benchmarks.json",
-    )
+    e3sm_diags.INSTALL_PATH,
+    "control_runs",
+    "aerosol_global_metrics_benchmarks.json",
+)
 
-with open(ref_data_path, 'r') as myfile:
-    ref_file=myfile.read()
+with open(ref_data_path, "r") as myfile:
+    ref_file = myfile.read()
 
 metrics_ref = json.loads(ref_file)
 
 for season in seasons:
     for aerosol in species:
-        print(f'Aerosol species: {aerosol}')
+        print(f"Aerosol species: {aerosol}")
         metrics_dict[aerosol] = generate_metrics_dic(test_data, aerosol, season)
         metrics_dict_ref[aerosol] = metrics_ref[aerosol]
-        #metrics_dict_ref[aerosol] = {
+        # metrics_dict_ref[aerosol] = {
         #    "Surface Emission (Tg/yr)": f'{MISSING_VALUE:.3f}',
         #    "Sink (Tg/yr)": f'{MISSING_VALUE:.3f}',
         #    "Dry Deposition (Tg/yr)": f'{MISSING_VALUE:.3f}',
         #    "Wet Deposition (Tg/yr)": f'{MISSING_VALUE:.3f}',
         #    "Burden (Tg)": f'{MISSING_VALUE:.3f}',
         #    "Lifetime (Days)": f'{MISSING_VALUE:.3f}',
         #    }
-    
-        with open(f'aerosol_table_{season}.csv', "w") as table_csv:
+
+        with open(f"aerosol_table_{season}.csv", "w") as table_csv:
             writer = csv.writer(
                 table_csv,
                 delimiter=",",
                 quotechar="'",
                 quoting=csv.QUOTE_MINIMAL,
-                lineterminator='\n',
+                lineterminator="\n",
             )
-            #writer.writerow([" ", "test","ref",])
+            # writer.writerow([" ", "test","ref",])
             for key, values in metrics_dict.items():
                 writer.writerow([SPECIES_NAMES[key]])
-                print('key',key, values)
+                print("key", key, values)
                 for value in values:
                     print(value)
                     line = []
                     line.append(value)
                     line.append(values[value])
                     line.append(metrics_dict_ref[key][value])
-                    print(line, 'line')
+                    print(line, "line")
                     writer.writerows([line])
                 writer.writerows([""])
-
-
-
-