diff --git a/tests/test_sections.py b/tests/test_sections.py index df49f4a9..d81c2b7c 100644 --- a/tests/test_sections.py +++ b/tests/test_sections.py @@ -90,7 +90,13 @@ def test_sections(): "reservation": "", "qos": "regular", "templateDir": "zppy/templates", + "ts_atm_grid": "180x360_aave", + "ts_atm_subsection": "", + "ts_grid": "180x360_aave", + "ts_land_grid": "180x360_aave", + "ts_land_subsection": "", "ts_num_years": 5, + "ts_subsection": "", "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U", "walltime": "02:00:00", "www": "WWWW", @@ -149,7 +155,13 @@ def test_sections(): "subsection": None, "templateDir": "zppy/templates", "tpd": 1, + "ts_atm_grid": "180x360_aave", + "ts_atm_subsection": "", + "ts_grid": "180x360_aave", + "ts_land_grid": "180x360_aave", + "ts_land_subsection": "", "ts_num_years": 5, + "ts_subsection": "", "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", "walltime": "02:00:00", "www": "WWWW", @@ -204,7 +216,13 @@ def test_sections(): "reservation": "", "subsection": None, "templateDir": "zppy/templates", + "ts_atm_grid": "180x360_aave", + "ts_atm_subsection": "", + "ts_grid": "180x360_aave", + "ts_land_grid": "180x360_aave", + "ts_land_subsection": "", "ts_num_years": 5, + "ts_subsection": "", "vars": "", "walltime": "02:00:00", "www": "WWWW", @@ -259,7 +277,13 @@ def test_subsections(): "qos": "regular", "reservation": "", "templateDir": "zppy/templates", + "ts_atm_grid": "180x360_aave", + "ts_atm_subsection": "", + "ts_grid": "180x360_aave", + "ts_land_grid": "180x360_aave", + "ts_land_subsection": "", "ts_num_years": 5, + "ts_subsection": "", "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U", "walltime": "02:00:00", "www": "WWWW", @@ -334,7 +358,13 @@ def test_subsections(): "subsection": "ts_grid1", "templateDir": "zppy/templates", "tpd": 1, + "ts_atm_grid": "180x360_aave", + "ts_atm_subsection": "", + "ts_grid": "180x360_aave", + "ts_land_grid": "180x360_aave", + "ts_land_subsection": "", "ts_num_years": 5, + "ts_subsection": "", "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", "walltime": "02:00:00", "www": "WWWW", @@ -375,7 +405,13 @@ def test_subsections(): "subsection": "ts_grid2", "templateDir": "zppy/templates", "tpd": 1, + "ts_atm_grid": "180x360_aave", + "ts_atm_subsection": "", + "ts_grid": "180x360_aave", + "ts_land_grid": "180x360_aave", + "ts_land_subsection": "", "ts_num_years": 5, + "ts_subsection": "", "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT", "walltime": "02:00:00", "www": "WWWW", @@ -446,7 +482,13 @@ def test_subsections(): "reservation": "", "subsection": "climo_grid1", "templateDir": "zppy/templates", + "ts_atm_grid": "180x360_aave", + "ts_atm_subsection": "", + "ts_grid": "180x360_aave", + "ts_land_grid": "180x360_aave", + "ts_land_subsection": "", "ts_num_years": 5, + "ts_subsection": "", "vars": "", "walltime": "02:00:00", "www": "WWWW", @@ -484,7 +526,13 @@ def test_subsections(): "qos": "regular", "subsection": "climo_grid2", "templateDir": "zppy/templates", + "ts_atm_grid": "180x360_aave", + "ts_atm_subsection": "", + "ts_grid": "180x360_aave", + "ts_land_grid": "180x360_aave", + "ts_land_subsection": "", "ts_num_years": 5, + "ts_subsection": "", "vars": "", "walltime": "02:00:00", "www": "WWWW", diff --git a/tests/test_zppy_e3sm_to_cmip.py b/tests/test_zppy_e3sm_to_cmip.py new file mode 100644 index 00000000..1e796003 --- /dev/null +++ b/tests/test_zppy_e3sm_to_cmip.py @@ -0,0 +1,194 @@ +import pytest + +from zppy.e3sm_to_cmip import check_and_define_parameters, check_parameters_for_bash +from zppy.utils import ParameterNotProvidedError + + +def test_check_parameters_for_bash(): + + # 1. ts_grid is set explictily + c = {"ts_grid": "grid"} + check_parameters_for_bash(c) + + # 2. ts_grid can't be set because an empty string is provided + c = {"ts_grid": ""} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + # 3. ts_grid is set via ts_atm_grid + c = {"component": "atm", "ts_atm_grid": "atm_grid"} + check_parameters_for_bash(c) + # 4. ts_grid is set via ts_land_grid + c = {"component": "lnd", "ts_land_grid": "land_grid"} + check_parameters_for_bash(c) + + # 5. ts_grid can't be set via ts_land_grid since component is atm + c = {"component": "atm", "ts_land_grid": "land_grid"} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + # 6. ts_grid can't be set via ts_atm_grid since component is lnd + c = {"component": "lnd", "ts_atm_grid": "atm_grid"} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + # 7. ts_grid can't be set via ts_atm_grid since component isn't specified + c = {"ts_atm_grid": "atm_grid"} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + # 8. ts_grid can't be set via ts_land_grid since component isn't specified + c = {"ts_land_grid": "land_grid"} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + # 9. ts_grid can't be set because ts_atm_grid is set to the empty string + c = {"component": "atm", "ts_atm_grid": ""} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + # 10. ts_grid can't be set because ts_land_grid is set to the empty string + c = {"component": "lnd", "ts_land_grid": ""} + with pytest.raises(ParameterNotProvidedError): + check_parameters_for_bash(c) + + +def test_check_and_define_parameters(): + sub = "name_of_this_subsection" + + # Guess the subsection #################################################### + # 1. ts_subsection is set explictily + c = {"ts_subsection": "subsection", "guess_section_parameters": True} + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "subsection" + + # 2. ts_subsection is set via sub because it is initially set to the empty string + c = {"ts_subsection": "", "guess_section_parameters": True} + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "name_of_this_subsection" + + # 3. ts_subsection is set via ts_atm_subsection + c = { + "component": "atm", + "ts_atm_subsection": "atm_subsection", + "guess_section_parameters": True, + } + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "atm_subsection" + # 4. ts_subsection is set via ts_land_subsection + c = { + "component": "lnd", + "ts_land_subsection": "land_subsection", + "guess_section_parameters": True, + } + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "land_subsection" + + # 5. ts_subsection can't be set via ts_land_subsection since component is atm + c = { + "component": "atm", + "ts_land_subsection": "land_subsection", + "guess_section_parameters": True, + } + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "name_of_this_subsection" + # 6. ts_subsection can't be set via ts_atm_subsection since component is lnd + c = { + "component": "lnd", + "ts_atm_subsection": "atm_subsection", + "guess_section_parameters": True, + } + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "name_of_this_subsection" + + # 7. ts_subsection can't be set via ts_atm_subsection since component isn't specified + c = {"ts_atm_subsection": "atm_subsection", "guess_section_parameters": True} + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "name_of_this_subsection" + # 8. ts_subsection can't be set via ts_atm_subsection since component isn't specified + c = {"ts_land_subsection": "land_subsection", "guess_section_parameters": True} + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "name_of_this_subsection" + + # 9. ts_subsection is set via sub because it is initially not provided + c = {"guess_section_parameters": True} + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "name_of_this_subsection" + + # 10. ts_subsection is set via sub because it is initially not provided and component isn't specified (required to use ts_atm_subsection) + c = {"ts_atm_subsection": "", "guess_section_parameters": True} + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "name_of_this_subsection" + + # 11. ts_subsection is set via sub because it is initially not provided and component isn't specified (required to use ts_land_subsection) + c = {"ts_land_subsection": "", "guess_section_parameters": True} + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "name_of_this_subsection" + + # Don't guess the subsection ############################################## + # Repeat above cases, but with guess_section_parameters set to False + + # 1 + c = {"ts_subsection": "subsection", "guess_section_parameters": False} + check_and_define_parameters(c, sub) + assert c["ts_subsection"] == "subsection" + + # 2 + c = {"ts_subsection": "", "guess_section_parameters": False} + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + + # 3 + c = { + "component": "atm", + "ts_atm_subsection": "atm_subsection", + "guess_section_parameters": False, + } + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + # 4 + c = { + "component": "lnd", + "ts_land_subsection": "land_subsection", + "guess_section_parameters": False, + } + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + + # 5 + c = { + "component": "atm", + "ts_land_subsection": "land_subsection", + "guess_section_parameters": False, + } + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + # 6 + c = { + "component": "lnd", + "ts_atm_subsection": "atm_subsection", + "guess_section_parameters": False, + } + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + + # 7 + c = {"ts_atm_subsection": "atm_subsection", "guess_section_parameters": False} + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + # 8 + c = {"ts_land_subsection": "land_subsection", "guess_section_parameters": False} + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + + # 9 + c = {"guess_section_parameters": False} + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + + # 10 + c = {"ts_atm_subsection": "", "guess_section_parameters": False} + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) + + # 11 + c = {"ts_land_subsection": "", "guess_section_parameters": False} + with pytest.raises(ParameterNotProvidedError): + check_and_define_parameters(c, sub) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index f62a66d8..74c90b5c 100644 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -57,8 +57,28 @@ plugins = force_list(default=list()) qos = string(default="regular") # Reservation -- if you have access to a node reservation, specify it with this parameter. reservation = string(default="") +# Use for e3sm_to_cmip and/or ilamb tasks. +# Name of the grid used by the relevant `[ts]` atm subtask +ts_atm_grid = string(default="180x360_aave") +# Use for e3sm_to_cmip and/or ilamb tasks. +# Name of the `[ts]` atm subtask to depend on +ts_atm_subsection = string(default="") +# Use for e3sm_to_cmip task (but NOT the ilamb task) -- you can either set this, or +# both ts_atm_grid and ts_land_grid +# Name of the grid used by the relevant `[ts]` task +ts_grid = string(default="180x360_aave") +# Use for e3sm_to_cmip and/or ilamb tasks. +# Name of the grid used by the relevant `[ts]` land subtask +ts_land_grid = string(default="180x360_aave") +# Use for e3sm_to_cmip and/or ilamb tasks. +# Name of the `[ts]` land subtask to depend on +ts_land_subsection = string(default="") # The years increment from `years` in `[ts]` ts_num_years = integer(default=5) +# Use for e3sm_to_cmip task (but NOT the ilamb task) -- you can either set this, or +# both ts_atm_subsection and ts_land_subsection +# Name of the `[ts]` subtask to depend on +ts_subsection = string(default="") # scriptDir -- NOTE: this parameter is created internally # templateDir -- NOTE: this parameter is created internally # The variables to process @@ -123,17 +143,11 @@ cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json") cmip_vars = string(default="") # Model component having generated input files (eam, eamxx, elm, mosart, ...) input_component = string(default="") -# Name of the grid used by the relevant `[ts]` task -ts_grid = string(default="180x360_aave") -# Name of the `[ts]` subtask to depend on -ts_subsection = string(default="") [[__many__]] cmip_metadata = string(default=None) cmip_vars = string(default=None) input_component = string(default=None) - ts_grid = string(default=None) - ts_subsection = string(default=None) [tc_analysis] # NOTE: always overrides value in [default] @@ -339,11 +353,3 @@ e3sm_to_cmip_land_subsection = string(default="") ilamb_obs = string(default="") # for land_only run land_only = boolean(default=False) -# Name of the grid used by the relevant `[ts]` atm subtask -ts_atm_grid = string(default="180x360_aave") -# Name of the `[ts]` atm subtask to depend on -ts_atm_subsection = string(default="") -# Name of the grid used by the relevant `[ts]` land subtask -ts_land_grid = string(default="180x360_aave") -# Name of the `[ts]` land subtask to depend on -ts_land_subsection = string(default="") diff --git a/zppy/e3sm_to_cmip.py b/zppy/e3sm_to_cmip.py index 38decbe5..168a9b7f 100644 --- a/zppy/e3sm_to_cmip.py +++ b/zppy/e3sm_to_cmip.py @@ -5,11 +5,12 @@ from zppy.bundle import handle_bundles from zppy.utils import ( ParameterGuessType, + ParameterNotProvidedError, add_dependencies, check_status, define_or_guess, - define_or_guess2, get_file_names, + get_guess_type_parameter, get_tasks, get_years, initialize_template, @@ -34,6 +35,7 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f for c in tasks: dependencies: List[str] = [] set_component_and_prc_typ(c) + check_parameters_for_bash(c) c["cmor_tables_prefix"] = c["diagnostics_base_path"] year_sets: List[Tuple[int, int]] = get_years(c["years"]) # Loop over year sets @@ -52,7 +54,9 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f # Run default variables if none are specified if c["cmip_vars"] == "": if c["component"] == "atm": - c["cmip_vars"] = "pr, tas, rsds, rlds, rsus" + c["cmip_vars"] = ( + "tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin" + ) elif c["component"] == "lnd": c["cmip_vars"] = ( "mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai, cLitter, cProduct, cSoilFast, cSoilMedium, cSoilSlow, fFire, fHarvest, cVeg, nbp, gpp, ra, rh" @@ -68,8 +72,7 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f with open(bash_file, "w") as f: f.write(template.render(**c)) make_executable(bash_file) - # Default to the name of this task if ts_subsection is not defined - define_or_guess2(c, "ts_subsection", sub, ParameterGuessType.SECTION_GUESS) + check_and_define_parameters(c, sub) add_dependencies( dependencies, script_dir, @@ -106,3 +109,56 @@ def e3sm_to_cmip(config: ConfigObj, script_dir: str, existing_bundles, job_ids_f print(f" environment_commands={c['environment_commands']}") return existing_bundles + + +def check_parameters_for_bash(c: Dict[str, Any]) -> None: + # Check parameters that aren't used until e3sm_diags.bash is run + parameter = "ts_grid" + if (parameter not in c.keys()) or (c[parameter] == ""): + if "component" in c.keys(): + # NOTE: unlike ts_subsection, ts_grid (and its component equivalents) + # are not defaulted to an empty string. + if ( + (c["component"] == "atm") + and ("ts_atm_grid" in c.keys()) + and (c["ts_atm_grid"] != "") + ): + c[parameter] = c["ts_atm_grid"] + elif ( + (c["component"] == "lnd") + and ("ts_land_grid" in c.keys()) + and (c["ts_land_grid"] != "") + ): + c[parameter] = c["ts_land_grid"] + else: + raise ParameterNotProvidedError(parameter) + else: + raise ParameterNotProvidedError(parameter) + + +def check_and_define_parameters(c: Dict[str, Any], sub: str) -> None: + parameter = "ts_subsection" + if (parameter not in c.keys()) or (c[parameter] == ""): + guess_type_parameter: str = get_guess_type_parameter( + ParameterGuessType.SECTION_GUESS + ) + if c[guess_type_parameter]: + if "component" in c.keys(): + if ( + (c["component"] == "atm") + and ("ts_atm_subsection" in c.keys()) + and (c["ts_atm_subsection"] != "") + ): + c[parameter] = c["ts_atm_subsection"] + elif ( + (c["component"] == "lnd") + and ("ts_land_subsection" in c.keys()) + and (c["ts_land_subsection"] != "") + ): + c[parameter] = c["ts_land_subsection"] + else: + c[parameter] = sub + else: + c[parameter] = sub + else: + raise ParameterNotProvidedError(parameter)