diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 62997e3..46fd6be 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -65,8 +65,3 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} verbose: true - - - name: Run pylint - if: ${{ always() }} - run: | - pylint --rcfile pylintrc fremorizer/ diff --git a/.gitignore b/.gitignore index df2b827..ab880fc 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,10 @@ cython_debug/ *.nc tmp/ +# experiment-config JSONs are generated by conftest fixtures at test time +fremorizer/tests/test_files/CMOR_input_example.json +fremorizer/tests/test_files/CMOR_CMIP7_input_example.json + # QA results wcrp_compliance_reports/ diff --git a/README.md b/README.md index ff3ce16..568b4c9 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![readthedocs](https://app.readthedocs.org/projects/fremorizer/badge/?version=latest&style=flat)](https://fremorizer.readthedocs.io/en/latest/) [![pylint](https://github.com/ilaflott/fremorizer/actions/workflows/pylint.yml/badge.svg?branch=main)](https://github.com/ilaflott/fremorizer/actions/workflows/pylint.yml) -[![pylint](https://img.shields.io/badge/pylint-%E2%89%A59.6-brightgreen)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) +[![pylint](https://img.shields.io/badge/pylint-%E2%89%A59.7-brightgreen)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) [![codecov](https://codecov.io/gh/ilaflott/fremorizer/branch/main/graph/badge.svg)](https://codecov.io/gh/ilaflott/fremorizer) diff --git a/fremorizer/__init__.py b/fremorizer/__init__.py index 273a2d5..5dc5282 100644 --- a/fremorizer/__init__.py +++ b/fremorizer/__init__.py @@ -8,7 +8,7 @@ fre_logger = logging.getLogger(__name__) -FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s" +FORMAT = '[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s' logging.basicConfig( level = logging.WARNING, format = FORMAT, filename = None, diff --git a/fremorizer/cli.py b/fremorizer/cli.py index 594bb46..5db3762 100644 --- a/fremorizer/cli.py +++ b/fremorizer/cli.py @@ -20,16 +20,16 @@ fre_logger = logging.getLogger(__name__) -OPT_VAR_NAME_HELP="optional, specify a variable name to specifically process only filenames " + \ - "matching that variable name. I.e., this string help target local_vars, not " + \ - "target_vars." -VARLIST_HELP="path pointing to a json file containing directory of key/value pairs. " + \ - "the keys are the \'local\' names used in the filename, and the values " + \ - "pointed to by those keys are strings representing the name of the variable " + \ - "contained in targeted files. the key and value are often the same, " + \ - "but it is not required." -RUN_ONE_HELP="process only one file, then exit. mostly for debugging and isolating issues." -DRY_RUN_HELP="don't call the cmor_mixer subtool, just printout what would be called and move on until natural exit" +OPT_VAR_NAME_HELP='optional, specify a variable name to specifically process only filenames ' + \ + 'matching that variable name. I.e., this string help target local_vars, not ' + \ + 'target_vars.' +VARLIST_HELP='path pointing to a json file containing directory of key/value pairs. ' + \ + 'the keys are the \'local\' names used in the filename, and the values ' + \ + 'pointed to by those keys are strings representing the name of the variable ' + \ + 'contained in targeted files. the key and value are often the same, ' + \ + 'but it is not required.' +RUN_ONE_HELP='process only one file, then exit. mostly for debugging and isolating issues.' +DRY_RUN_HELP='don\'t call the cmor_mixer subtool, just printout what would be called and move on until natural exit' START_YEAR_HELP = 'string representing the minimum calendar year CMOR should start processing for. ' + \ 'currently, only YYYY format is supported.' STOP_YEAR_HELP = 'string representing the maximum calendar year CMOR should stop processing for. ' + \ @@ -37,28 +37,28 @@ @click.version_option( - package_name = "fremorizer", + package_name = 'fremorizer', version = version ) @click.group( help = click.style( - "'fremor' is the main CLI for fremorizer. it houses the cmor subcommands.", + 'fremor is the main CLI for fremorizer. it houses the cmor subcommands.', fg = 'cyan') ) @click.option( '-v', '--verbose', default = 0, required = False, count = True, type = int, - help = "Increment logging verbosity from default (logging.WARNING) to logging.INFO. " + \ - "use -vv for logging.DEBUG. will be overridden by -q/--quiet" ) + help = 'Increment logging verbosity from default (logging.WARNING) to logging.INFO. ' + \ + 'use -vv for logging.DEBUG. will be overridden by -q/--quiet' ) @click.option( '-q', '--quiet', default = False, required = False, is_flag = True, type = bool, - help = "Set logging verbosity from default (logging.WARNING) to logging.ERROR, printing " + \ - "less output to screen. overrides -v[v]/--verbose" ) + help = 'Set logging verbosity from default (logging.WARNING) to logging.ERROR, printing ' + \ + 'less output to screen. overrides -v[v]/--verbose' ) @click.option( '-l', '--log_file', default = None, required = False, type = str, help = 'Path to log file for all fremor calls, the output to screen will still print with the ' + \ 'path specified. If the log file already exists, it is appended to.' ) def fremor(verbose = 0, quiet = False, log_file = None): - ''' + """ entry point function to subgroup functions, setting global verbosity/logging formats that all other routines will utilize - ''' + """ log_level = logging.WARNING # default if verbose == 1: log_level = logging.INFO # -v, more verbose than default @@ -91,20 +91,20 @@ def fremor(verbose = 0, quiet = False, log_file = None): @fremor.command() -@click.option("-y", "--yamlfile", type = str, +@click.option('-y', '--yamlfile', type = str, help = 'YAML file to be used for parsing', required = True ) -@click.option("-e", "--experiment", type = str, - help = "Experiment name", +@click.option('-e', '--experiment', type = str, + help = 'Experiment name', required = True ) -@click.option("-p", "--platform", type = str, - help = "Platform name", +@click.option('-p', '--platform', type = str, + help = 'Platform name', required = True ) -@click.option("-t", "--target", type = str, - help = "Target name", +@click.option('-t', '--target', type = str, + help = 'Target name', required = True ) -@click.option("-o", "--output", type = str, default = None, - help = "Output file if desired", required = False) +@click.option('-o', '--output', type = str, default = None, + help = 'Output file if desired', required = False) @click.option('--run_one', is_flag = True, default = False, help=RUN_ONE_HELP, required = False) @@ -142,22 +142,22 @@ def yaml(yamlfile, experiment, target, platform, output, run_one, dry_run, start @fremor.command() -@click.option("-l", "--varlist", type = str, +@click.option('-l', '--varlist', type = str, help=VARLIST_HELP, required=False) -@click.option("-r", "--table_config_dir", type = str, - help="directory holding MIP tables to search for variables in var list", +@click.option('-r', '--table_config_dir', type = str, + help='directory holding MIP tables to search for variables in var list', required=True) -@click.option('-v', "--opt_var_name", type = str, +@click.option('-v', '--opt_var_name', type = str, help=OPT_VAR_NAME_HELP, required=False) def find(varlist, table_config_dir, opt_var_name): #uncovered - ''' + """ loop over json table files in config_dir and show which tables contain variables in var list/ the tool will also print what that table entry is expecting of that variable as well. if given an opt_var_name in addition to varlist, only that variable name will be printed out. accepts 3 arguments, two of the three required. - ''' + """ cmor_find_subtool( json_var_list = varlist, json_table_config_dir = table_config_dir, @@ -166,44 +166,44 @@ def find(varlist, table_config_dir, opt_var_name): #uncovered @fremor.command() -@click.option("-d", "--indir", type = str, - help="directory containing netCDF files. keys specified in json_var_list are local " + \ - "variable names used for targeting specific files in this directory", +@click.option('-d', '--indir', type = str, + help='directory containing netCDF files. keys specified in json_var_list are local ' + \ + 'variable names used for targeting specific files in this directory', required=True) -@click.option("-l", "--varlist", type = str, +@click.option('-l', '--varlist', type = str, help=VARLIST_HELP, required=True) -@click.option("-r", "--table_config", type = str, - help="json file containing CMIP-compliant per-variable/metadata for specific " + \ - "MIP table. The MIP table can generally be identified by the specific " + \ - "filename (e.g. \'Omon\')", +@click.option('-r', '--table_config', type = str, + help='json file containing CMIP-compliant per-variable/metadata for specific ' + \ + 'MIP table. The MIP table can generally be identified by the specific ' + \ + 'filename (e.g. \'Omon\')', required=True) -@click.option("-p", "--exp_config", type = str, - help="json file containing metadata dictionary for CMORization. this metadata is " + \ - "effectively appended to the final output file's header", +@click.option('-p', '--exp_config', type = str, + help='json file containing metadata dictionary for CMORization. this metadata is ' + \ + 'effectively appended to the final output file\'s header', required=True) -@click.option("-o", "--outdir", type = str, - help="directory root that will contain the full output and output directory " + \ - "structure generated by the cmor module upon request.", +@click.option('-o', '--outdir', type = str, + help='directory root that will contain the full output and output directory ' + \ + 'structure generated by the cmor module upon request.', required=True) @click.option('--run_one', is_flag = True, default = False, help=RUN_ONE_HELP, required = False) -@click.option('-v', "--opt_var_name", type = str, default = None, +@click.option('-v', '--opt_var_name', type = str, default = None, help=OPT_VAR_NAME_HELP, required=False) @click.option('-g', '--grid_label', type = str, default = None, - help = 'label representing grid type of input data, e.g. "gn" for native or "gr" for regridded, ' + \ - 'replaces the "grid_label" field in the CMOR experiment configuration file. The label must ' + \ + help = 'label representing grid type of input data, e.g. gn for native or gr for regridded, ' + \ + 'replaces the grid_label field in the CMOR experiment configuration file. The label must ' + \ 'be one of the entries in the MIP controlled-vocab file.', required = False) @click.option('--grid_desc', type = str, default = None, - help = 'description of grid indicated by grid label, replaces the "grid" field in the CMOR ' + \ + help = 'description of grid indicated by grid label, replaces the grid field in the CMOR ' + \ 'experiment configuration file.', required = False) @click.option('--nom_res', type = str, default = None, - help = 'nominal resolution indicated by grid and/or grid label, replaces the "nominal_resolution", ' + \ - 'replaces the "grid" field in the CMOR experiment configuration file. The entered string ' + \ + help = 'nominal resolution indicated by grid and/or grid label, replaces the nominal_resolution, ' + \ + 'replaces the grid field in the CMOR experiment configuration file. The entered string ' + \ 'must be one of the entries in the MIP controlled-vocab file.', required = False) @click.option('--start', type=str, default=None, @@ -238,12 +238,12 @@ def run(indir, varlist, table_config, exp_config, outdir, run_one, opt_var_name, ) -@fremor.command() -@click.option("-d", "--dir_targ", type=str, required=True, help="Target directory") -@click.option("-o", "--output_variable_list", type=str, required=True, help="Output variable list file") -@click.option("-t", "--mip_table", type=str, required=False, default=None, - help="Target MIP table for making variable list") -def varlist(dir_targ, output_variable_list, mip_table): +@fremor.command('varlist') +@click.option('-d', '--dir_targ', type=str, required=True, help='Target directory') +@click.option('-o', '--output_variable_list', type=str, required=True, help='Output variable list file') +@click.option('-t', '--mip_table', type=str, required=False, default=None, + help='Target MIP table for making variable list') +def varlist_(dir_targ, output_variable_list, mip_table): """ Create a simple variable list from netCDF files in the target directory. """ @@ -253,30 +253,30 @@ def varlist(dir_targ, output_variable_list, mip_table): @fremor.command() -@click.option("-p", "--pp_dir", type=str, required=True, - help="Root post-processing directory containing per-component subdirectories.") -@click.option("-t", "--mip_tables_dir", type=str, required=True, - help="Directory containing MIP table JSON files.") -@click.option("-m", "--mip_era", type=str, required=True, - help="MIP era identifier, e.g. 'cmip6' or 'cmip7'.") -@click.option("-e", "--exp_config", type=str, required=True, - help="Path to JSON experiment/input configuration file expected by CMOR.") -@click.option("-o", "--output_yaml", type=str, required=True, - help="Path for the output CMOR YAML configuration file.") -@click.option("-d", "--output_dir", type=str, required=True, - help="Root output directory for CMORized data.") -@click.option("-l", "--varlist_dir", type=str, required=True, - help="Directory in which per-component variable list JSON files are written.") -@click.option("--freq", type=str, default="monthly", - help="Temporal frequency string, e.g. 'monthly', 'daily'. Default 'monthly'.") -@click.option("--chunk", type=str, default="5yr", - help="Time chunk string, e.g. '5yr', '10yr'. Default '5yr'.") -@click.option("--grid", type=str, default="g999", - help="Grid label anchor name, e.g. 'g999', 'gn'. Default 'g999'.") -@click.option("--overwrite", is_flag=True, default=False, - help="Overwrite existing variable list files.") -@click.option("--calendar", type=str, default="noleap", - help="Calendar type, e.g. 'noleap', '360_day'. Default 'noleap'.") +@click.option('-p', '--pp_dir', type=str, required=True, + help='Root post-processing directory containing per-component subdirectories.') +@click.option('-t', '--mip_tables_dir', type=str, required=True, + help='Directory containing MIP table JSON files.') +@click.option('-m', '--mip_era', type=str, required=True, + help='MIP era identifier, e.g. cmip6 or cmip7.') +@click.option('-e', '--exp_config', type=str, required=True, + help='Path to JSON experiment/input configuration file expected by CMOR.') +@click.option('-o', '--output_yaml', type=str, required=True, + help='Path for the output CMOR YAML configuration file.') +@click.option('-d', '--output_dir', type=str, required=True, + help='Root output directory for CMORized data.') +@click.option('-l', '--varlist_dir', type=str, required=True, + help='Directory in which per-component variable list JSON files are written.') +@click.option('--freq', type=str, default='monthly', + help='Temporal frequency string, e.g. monthly, daily. Default monthly.') +@click.option('--chunk', type=str, default='5yr', + help='Time chunk string, e.g. 5yr, 10yr. Default 5yr.') +@click.option('--grid', type=str, default='g999', + help='Grid label anchor name, e.g. g999, gn. Default g999.') +@click.option('--overwrite', is_flag=True, default=False, + help='Overwrite existing variable list files.') +@click.option('--calendar', type=str, default='noleap', + help='Calendar type, e.g. noleap, 360_day. Default noleap.') def config(pp_dir, mip_tables_dir, mip_era, exp_config, output_yaml, output_dir, varlist_dir, freq, chunk, grid, overwrite, calendar): """ @@ -301,20 +301,20 @@ def config(pp_dir, mip_tables_dir, mip_era, exp_config, output_yaml, @fremor.command() -@click.option("-m", "--mip_era", type=click.Choice(['cmip6', 'cmip7'], case_sensitive=False), +@click.option('-m', '--mip_era', type=click.Choice(['cmip6', 'cmip7'], case_sensitive=False), required=True, - help="MIP era for the template: 'cmip6' or 'cmip7'.") -@click.option("-e", "--exp_config", type=str, default=None, - help="Output path for the template experiment-config JSON file. " - "When omitted and --tables_dir is also omitted, a default " - "filename is used.") -@click.option("-t", "--tables_dir", type=str, default=None, - help="Directory into which MIP tables will be fetched from " - "trusted sources. Omit to skip table retrieval.") -@click.option("--tag", type=str, default=None, - help="Specific git tag or release for the MIP tables repository.") -@click.option("--fast", is_flag=True, default=False, - help="Use curl to download a tarball instead of git clone.") + help='MIP era for the template: cmip6 or cmip7.') +@click.option('-e', '--exp_config', type=str, default=None, + help='Output path for the template experiment-config JSON file. ' + 'When omitted and --tables_dir is also omitted, a default ' + 'filename is used.') +@click.option('-t', '--tables_dir', type=str, default=None, + help='Directory into which MIP tables will be fetched from ' + 'trusted sources. Omit to skip table retrieval.') +@click.option('--tag', type=str, default=None, + help='Specific git tag or release for the MIP tables repository.') +@click.option('--fast', is_flag=True, default=False, + help='Use curl to download a tarball instead of git clone.') def init(mip_era, exp_config, tables_dir, tag, fast): """ Initialise CMOR resources: write an empty experiment-config JSON template diff --git a/fremorizer/cmor_config.py b/fremorizer/cmor_config.py index 952347a..aa51552 100644 --- a/fremorizer/cmor_config.py +++ b/fremorizer/cmor_config.py @@ -215,7 +215,7 @@ def cmor_config_subtool( lines.append(f" - component_name: '{component_name}'") lines.append(f" variable_list: '{variable_list}'") lines.append(" data_series_type: 'ts'") - lines.append(" chunk: *PP_CMIP_CHUNK") + lines.append(' chunk: *PP_CMIP_CHUNK') # ---- write output YAML ---- diff --git a/fremorizer/cmor_constants.py b/fremorizer/cmor_constants.py index 5b6ed29..1fa6e80 100644 --- a/fremorizer/cmor_constants.py +++ b/fremorizer/cmor_constants.py @@ -28,21 +28,21 @@ # Vertical-coordinate classification (used by cmor_mixer) # --------------------------------------------------------------------------- ACCEPTED_VERT_DIMS = [ - "z_l", "landuse", - "plev39", "plev30", "plev19", "plev8", - "height2m", - "level", "lev", "levhalf", + 'z_l', 'landuse', + 'plev39', 'plev30', 'plev19', 'plev8', + 'height2m', + 'level', 'lev', 'levhalf', ] NON_HYBRID_SIGMA_COORDS = [ - "landuse", - "plev39", "plev30", "plev19", "plev8", - "height2m", + 'landuse', + 'plev39', 'plev30', 'plev19', 'plev8', + 'height2m', ] -ALT_HYBRID_SIGMA_COORDS = ["level", "lev", "levhalf"] +ALT_HYBRID_SIGMA_COORDS = ['level', 'lev', 'levhalf'] -DEPTH_COORDS = ["z_l"] +DEPTH_COORDS = ['z_l'] # --------------------------------------------------------------------------- @@ -62,10 +62,10 @@ # equivalents. Dimensions whose names already match (e.g. plev39, height2m) # need no entry; the look-up falls back to using the input name directly. INPUT_TO_MIP_VERT_DIM = { - "z_l": "olevel", - "level": "alevel", - "lev": "alevel", - "levhalf": "alevhalf", + 'z_l': 'olevel', + 'level': 'alevel', + 'lev': 'alevel', + 'levhalf': 'alevhalf', } diff --git a/fremorizer/cmor_finder.py b/fremorizer/cmor_finder.py index 9a003b8..4975cf5 100644 --- a/fremorizer/cmor_finder.py +++ b/fremorizer/cmor_finder.py @@ -53,9 +53,9 @@ def print_var_content(table_config_file: IO[str], table_name = None try: - table_name = proj_table_vars["Header"].get('table_id').split(' ')[1] + table_name = proj_table_vars['Header'].get('table_id').split(' ')[1] except KeyError: - fre_logger.warning("couldn't get header and table_name field") + fre_logger.warning('couldn\'t get header and table_name field') if table_name is not None: fre_logger.info('looking for %s data in table %s!', var_name, table_name) @@ -63,7 +63,7 @@ def print_var_content(table_config_file: IO[str], fre_logger.info('looking for %s data in table %s, but could not find its table_name!', var_name, table_config_file.name) - var_content = proj_table_vars.get("variable_entry", {}).get(var_name) + var_content = proj_table_vars.get('variable_entry', {}).get(var_name) if var_content is None: fre_logger.debug('variable %s not found in %s, moving on!', var_name, Path(table_config_file.name).name) return @@ -107,7 +107,7 @@ def cmor_find_subtool( json_var_list: Optional[str] = None, var_list = None if json_var_list is not None: - with open(json_var_list, "r", encoding="utf-8") as var_list_file: + with open(json_var_list, 'r', encoding='utf-8') as var_list_file: var_list = json.load(var_list_file) if opt_var_name is None and var_list is None: @@ -116,14 +116,14 @@ def cmor_find_subtool( json_var_list: Optional[str] = None, if opt_var_name is not None: fre_logger.info('opt_var_name is not None: looking for only ONE variables worth of info!') for json_table_config in json_table_configs: - with open(json_table_config, "r", encoding="utf-8") as table_config_file: + with open(json_table_config, 'r', encoding='utf-8') as table_config_file: print_var_content(table_config_file, opt_var_name) elif var_list is not None: fre_logger.info('opt_var_name is None, and var_list is not None, looking for many variables worth of info!') for var in var_list: for json_table_config in json_table_configs: - with open(json_table_config, "r", encoding="utf-8") as table_config_file: + with open(json_table_config, 'r', encoding='utf-8') as table_config_file: print_var_content(table_config_file, str(var_list[var])) @@ -132,7 +132,7 @@ def make_simple_varlist( dir_targ: str, json_mip_table: Optional[str] = None) -> Optional[Dict[str, str]]: """ Generate a JSON file containing a list of variable names from NetCDF files in a specified directory. - This function searches for NetCDF files in the given directory, or a subdirectory, "ts/monthly/5yr", + This function searches for NetCDF files in the given directory, or a subdirectory, 'ts/monthly/5yr', if not already included. It then extracts variable names from the filenames, and writes these variable names to a JSON file. @@ -154,22 +154,22 @@ def make_simple_varlist( dir_targ: str, """ # if the variable is in the filename, it's likely delimited by another period. - all_nc_files = glob.glob(os.path.join(dir_targ, "*.*.nc")) + all_nc_files = glob.glob(os.path.join(dir_targ, '*.*.nc')) if not all_nc_files: - fre_logger.error("No files found in the directory.") #uncovered + fre_logger.error('No files found in the directory.') #uncovered return None if len(all_nc_files) == 1: - fre_logger.warning("Warning: Only one file found matching the pattern.") + fre_logger.warning('Warning: Only one file found matching the pattern.') - fre_logger.info("Files found matching pattern. Number of files: %d", len(all_nc_files)) + fre_logger.info('Files found matching pattern. Number of files: %d', len(all_nc_files)) mip_vars = None if json_mip_table is not None: try: # read in mip vars to check against later fre_logger.debug('attempting to read in variable entries in specified mip table') - full_mip_vars_list=get_json_file_data(json_mip_table)["variable_entry"].keys() + full_mip_vars_list=get_json_file_data(json_mip_table)['variable_entry'].keys() except Exception as exc: raise Exception( 'problem opening mip table and getting variable entry data.' diff --git a/fremorizer/cmor_helpers.py b/fremorizer/cmor_helpers.py index acc78f4..ea3ab8b 100644 --- a/fremorizer/cmor_helpers.py +++ b/fremorizer/cmor_helpers.py @@ -60,9 +60,9 @@ # CF calendar aliases mapped to their canonical equivalents CF_CALENDAR_ALIASES = { - "noleap": "365_day", - "all_leap": "366_day", - "standard": "gregorian", + 'noleap': '365_day', + 'all_leap': '366_day', + 'standard': 'gregorian', } @@ -106,13 +106,13 @@ def get_time_calendar_value(time_var) -> Optional[str]: try: calendar_val = str(time_var.calendar).lower() except Exception: - fre_logger.debug("could not find calendar attribute on time axis. moving on.") + fre_logger.debug('could not find calendar attribute on time axis. moving on.') if calendar_val is None: try: calendar_val = str(time_var.calendar_type).lower() except Exception: - fre_logger.debug("could not find calendar_type attribute on time axis. moving on.") + fre_logger.debug('could not find calendar_type attribute on time axis. moving on.') return normalize_calendar(calendar_val) @@ -276,7 +276,7 @@ def find_statics_file( bronx_file_path: str) -> Optional[str]: statics_path = '/'.join(bronx_file_path_elem) fre_logger.debug('going to glob the following path for a statics file: \n%s\n', statics_path) fre_logger.debug('the call is going to be:') - fre_logger.debug("\n glob.glob(%s) \n", statics_path+'/*static*.nc') + fre_logger.debug('\n glob.glob(%s) \n', statics_path+'/*static*.nc') statics_file_glob = glob.glob(statics_path+'/*static*.nc') # update to use component TODO fre_logger.debug('the output glob looks like: %s', statics_file_glob) @@ -355,7 +355,7 @@ def get_iso_datetime_ranges( var_filenames: List[str], for filename in var_filenames: fre_logger.debug('filename = %s', filename) - iso_daterange = filename.split(".")[-3] # '????????-????????' + iso_daterange = filename.split('.')[-3] # '????????-????????' fre_logger.debug('iso_daterange = %s', iso_daterange) if start_stop_filter: @@ -387,13 +387,13 @@ def check_dataset_for_ocean_grid( ds: Dataset) -> bool: .. note:: Logs a warning if an ocean grid is detected. """ ds_var_keys = list(ds.variables.keys()) - uses_ocean_grid = any(["xh" in ds_var_keys, "yh" in ds_var_keys]) + uses_ocean_grid = any(['xh' in ds_var_keys, 'yh' in ds_var_keys]) if uses_ocean_grid: fre_logger.warning( - "\n----------------------------------------------------------------------------------\n" - " 'xh' found in var_list: ocean grid req'd\n" - " sometimes i don't cmorize right! check me!\n" - "----------------------------------------------------------------------------------\n" + '\n----------------------------------------------------------------------------------\n' + ' \'xh\' found in var_list: ocean grid req\'d\n' + ' sometimes i don\'t cmorize right! check me!\n' + '----------------------------------------------------------------------------------\n' ) return uses_ocean_grid @@ -421,7 +421,7 @@ def get_vertical_dimension( ds: Dataset, if dim.lower() == 'landuse': vert_dim = dim break - if not (ds[dim].axis and ds[dim].axis == "Z"): + if not (ds[dim].axis and ds[dim].axis == 'Z'): continue vert_dim = dim return vert_dim @@ -434,24 +434,24 @@ def create_tmp_dir( outdir: str, :param outdir: Base output directory. :type outdir: str - :param json_exp_config: Path to a JSON config file with an "outpath" key. + :param json_exp_config: Path to a JSON config file with an 'outpath' key. :type json_exp_config: str, optional :raises OSError: If the temporary directory cannot be created. :return: Path to the created temporary directory. :rtype: str - .. note:: If json_exp_config is provided and contains "outpath", a subdirectory is also created. + .. note:: If json_exp_config is provided and contains 'outpath', a subdirectory is also created. """ outdir_from_exp_config = None if json_exp_config is not None: - with open(json_exp_config, "r", encoding="utf-8") as table_config_file: + with open(json_exp_config, 'r', encoding='utf-8') as table_config_file: try: - outdir_from_exp_config = json.load(table_config_file)["outpath"] + outdir_from_exp_config = json.load(table_config_file)['outpath'] except Exception: fre_logger.warning( 'could not read outdir from json_exp_config. the cmor module will throw a toothless warning') - tmp_dir = str(Path(f"{outdir}/CMOR_tmp/").resolve()) + tmp_dir = str(Path(f'{outdir}/CMOR_tmp/').resolve()) try: os.makedirs(tmp_dir, exist_ok=True) if outdir_from_exp_config is not None: @@ -478,7 +478,7 @@ def get_json_file_data( json_file_path: Optional[str] = None) -> dict: :rtype: dict """ try: - with open(json_file_path, "r", encoding="utf-8") as json_config_file: + with open(json_file_path, 'r', encoding='utf-8') as json_config_file: return json.load(json_config_file) except Exception as exc: raise FileNotFoundError( @@ -493,15 +493,15 @@ def update_grid_and_label( json_file_path: str, new_nom_res: str, output_file_path: Optional[str] = None) -> None: """ - Update the "grid_label", "grid", and "nominal_resolution" fields in a JSON experiment config. + Update the 'grid_label', 'grid', and 'nominal_resolution' fields in a JSON experiment config. :param json_file_path: Path to the input JSON file. :type json_file_path: str - :param new_grid_label: New value for the "grid_label" field. + :param new_grid_label: New value for the 'grid_label' field. :type new_grid_label: str - :param new_grid: New value for the "grid" field. + :param new_grid: New value for the 'grid' field. :type new_grid: str - :param new_nom_res: New value for the "nominal_resolution" field. + :param new_nom_res: New value for the 'nominal_resolution' field. :type new_nom_res: str :param output_file_path: Path to save the updated JSON file. If None, overwrites the original file. :type output_file_path: str, optional @@ -521,48 +521,48 @@ def update_grid_and_label( json_file_path: str, raise ValueError try: - with open(json_file_path, "r", encoding="utf-8") as file: + with open(json_file_path, 'r', encoding='utf-8') as file: data = json.load(file) try: - fre_logger.info('Original "grid": %s', data["grid"]) - data["grid"] = new_grid - fre_logger.info('Updated "grid": %s', data["grid"]) + fre_logger.info('Original grid: %s', data['grid']) + data['grid'] = new_grid + fre_logger.info('Updated grid: %s', data['grid']) except KeyError as e: - fre_logger.error("Failed to update 'grid': %s", e) - raise KeyError("Error while updating 'grid'. Ensure the field exists and is modifiable.") from e + fre_logger.error('Failed to update grid: %s', e) + raise KeyError('Error while updating grid. Ensure the field exists and is modifiable.') from e try: - fre_logger.info('Original "grid_label": %s', data["grid_label"]) - data["grid_label"] = new_grid_label - fre_logger.info('Updated "grid_label": %s', data["grid_label"]) + fre_logger.info('Original grid_label: %s', data['grid_label']) + data['grid_label'] = new_grid_label + fre_logger.info('Updated grid_label: %s', data['grid_label']) except KeyError as e: - fre_logger.error("Failed to update 'grid_label': %s", e) - raise KeyError("Error while updating 'grid_label'. Ensure the field exists and is modifiable.") from e + fre_logger.error('Failed to update grid_label: %s', e) + raise KeyError('Error while updating grid_label. Ensure the field exists and is modifiable.') from e try: - fre_logger.info('Original "nominal_resolution": %s', data["nominal_resolution"]) - data["nominal_resolution"] = new_nom_res - fre_logger.info('Updated "nominal_resolution": %s', data["nominal_resolution"]) + fre_logger.info('Original nominal_resolution: %s', data['nominal_resolution']) + data['nominal_resolution'] = new_nom_res + fre_logger.info('Updated nominal_resolution: %s', data['nominal_resolution']) except KeyError as e: - fre_logger.error("Failed to update 'nominal_resolution': %s", e) - raise KeyError("Error updating 'nominal_resolution'. Ensure the field exists and is modifiable.") from e + fre_logger.error('Failed to update nominal_resolution: %s', e) + raise KeyError('Error updating nominal_resolution. Ensure the field exists and is modifiable.') from e output_file_path = output_file_path or json_file_path - with open(output_file_path, "w", encoding="utf-8") as file: + with open(output_file_path, 'w', encoding='utf-8') as file: json.dump(data, file, indent=4) fre_logger.info('Successfully updated fields and saved to %s', output_file_path) except FileNotFoundError: - fre_logger.error("The file '%s' does not exist.", json_file_path) + fre_logger.error('The file %s does not exist.', json_file_path) raise except json.JSONDecodeError: - fre_logger.error("Failed to decode JSON from the file '%s'.", json_file_path) + fre_logger.error('Failed to decode JSON from the file %s.', json_file_path) raise except Exception as e: - fre_logger.error("An unexpected error occurred: %s", e) + fre_logger.error('An unexpected error occurred: %s', e) raise @@ -570,23 +570,23 @@ def update_calendar_type( json_file_path: str, new_calendar_type: str, output_file_path: Optional[str] = None) -> None: """ - Update the "calendar" field in a JSON experiment config file. + Update the 'calendar' field in a JSON experiment config file. :param json_file_path: Path to the input JSON file. :type json_file_path: str - :param new_calendar_type: New value for the "calendar" field. + :param new_calendar_type: New value for the 'calendar' field. :type new_calendar_type: str :param output_file_path: Path to save the updated JSON file. If None, overwrites the original file. :type output_file_path: str, optional :raises FileNotFoundError: If the input JSON file does not exist. - :raises KeyError: If the "calendar" field is not found in the JSON file. + :raises KeyError: If the 'calendar' field is not found in the JSON file. :raises ValueError: If new_calendar_type is None. :raises json.JSONDecodeError: If the JSON file cannot be decoded. :return: None :rtype: None .. note:: The function logs before and after values, and overwrites the input file unless an output path is given. - CF calendar aliases (e.g., "noleap") are normalized to their canonical CF names (e.g., "365_day"). + CF calendar aliases (e.g., 'noleap') are normalized to their canonical CF names (e.g., '365_day'). """ if new_calendar_type is None: fre_logger.error( @@ -597,35 +597,35 @@ def update_calendar_type( json_file_path: str, normalized_calendar_type = normalize_calendar(new_calendar_type) try: - with open(json_file_path, "r", encoding="utf-8") as file: + with open(json_file_path, 'r', encoding='utf-8') as file: data = json.load(file) try: - fre_logger.info('Original "calendar": %s', data["calendar"]) + fre_logger.info('Original calendar: %s', data['calendar']) if normalized_calendar_type != str(new_calendar_type).lower(): - fre_logger.info('Normalizing calendar alias "%s" to "%s"', + fre_logger.info('Normalizing calendar alias %s to %s', new_calendar_type, normalized_calendar_type) - data["calendar"] = normalized_calendar_type - fre_logger.info('Updated "calendar": %s', data["calendar"]) + data['calendar'] = normalized_calendar_type + fre_logger.info('Updated calendar: %s', data['calendar']) except KeyError as e: - fre_logger.error("Failed to update 'calendar': %s", e) - raise KeyError("Error while updating 'calendar'. Ensure the field exists and is modifiable.") from e + fre_logger.error('Failed to update calendar: %s', e) + raise KeyError('Error while updating calendar. Ensure the field exists and is modifiable.') from e output_file_path = output_file_path or json_file_path - with open(output_file_path, "w", encoding="utf-8") as file: + with open(output_file_path, 'w', encoding='utf-8') as file: json.dump(data, file, indent=4) fre_logger.info('Successfully updated fields and saved to %s', output_file_path) except FileNotFoundError: - fre_logger.error("The file '%s' does not exist.", json_file_path) + fre_logger.error('The file %s does not exist.', json_file_path) raise except json.JSONDecodeError: - fre_logger.error("Failed to decode JSON from the file '%s'.", json_file_path) + fre_logger.error('Failed to decode JSON from the file %s.', json_file_path) raise except Exception as e: - fre_logger.error("An unexpected error occurred: %s", e) + fre_logger.error('An unexpected error occurred: %s', e) raise def check_path_existence(some_path: str): @@ -668,28 +668,28 @@ def conv_mip_to_bronx_freq(cmor_table_freq: str) -> Optional[str]: :rtype: str or None """ cmor_to_bronx_dict = { - "1hr" : "1hr", - "1hrCM" : None, - "1hrPt" : None, - "3hr" : "3hr", - "3hrPt" : None, - "6hr" : "6hr", - "6hrPt" : None, - "day" : "daily", - "dec" : None, - "fx" : None, - "mon" : "monthly", - "monC" : None, - "monPt" : None, - "subhrPt": None, - "yr" : "annual", - "yrPt" : None + '1hr' : '1hr', + '1hrCM' : None, + '1hrPt' : None, + '3hr' : '3hr', + '3hrPt' : None, + '6hr' : '6hr', + '6hrPt' : None, + 'day' : 'daily', + 'dec' : None, + 'fx' : None, + 'mon' : 'monthly', + 'monC' : None, + 'monPt' : None, + 'subhrPt': None, + 'yr' : 'annual', + 'yrPt' : None } bronx_freq = cmor_to_bronx_dict.get(cmor_table_freq) if bronx_freq is None: fre_logger.warning('MIP table frequency = %s does not have a FRE-bronx equivalent', cmor_table_freq) - if cmor_table_freq not in cmor_to_bronx_dict.keys(): - raise KeyError(f'MIP table frequency = "{cmor_table_freq}" is not a valid MIP frequency') + if cmor_table_freq not in cmor_to_bronx_dict: + raise KeyError(f'MIP table frequency = {cmor_table_freq} is not a valid MIP frequency') return bronx_freq def get_bronx_freq_from_mip_table(json_table_config: str) -> str: @@ -719,7 +719,7 @@ def get_bronx_freq_from_mip_table(json_table_config: str) -> str: # outpath: str, # output_file_path: Optional[str] = None) -> None: # """ -# Update the "outpath" field in a JSON experiment config file. +# Update the 'outpath' field in a JSON experiment config file. # # :param json_file_path: Path to the input JSON file. # :type json_file_path: str @@ -736,32 +736,32 @@ def get_bronx_freq_from_mip_table(json_table_config: str) -> str: # raise ValueError # # try: -# with open(json_file_path, "r", encoding="utf-8") as file: +# with open(json_file_path, 'r', encoding='utf-8') as file: # data = json.load(file) # # try: -# fre_logger.info('Original "outpath": %s', data["outpath"]) -# data["outpath"] = outpath -# fre_logger.info('Updated "outpath": %s', data["outpath"]) +# fre_logger.info('Original 'outpath': %s', data['outpath']) +# data['outpath'] = outpath +# fre_logger.info('Updated 'outpath': %s', data['outpath']) # except KeyError as e: -# fre_logger.error("Failed to update 'outpath': %s", e) -# raise KeyError("Error while updating 'outpath'. Ensure the field exists and is modifiable.") from e +# fre_logger.error('Failed to update 'outpath': %s', e) +# raise KeyError('Error while updating 'outpath'. Ensure the field exists and is modifiable.') from e # # output_file_path = output_file_path or json_file_path # -# with open(output_file_path, "w", encoding="utf-8") as file: +# with open(output_file_path, 'w', encoding='utf-8') as file: # json.dump(data, file, indent=4) # # fre_logger.info('Successfully updated fields and saved to %s', output_file_path) # # except FileNotFoundError: -# fre_logger.error("The file '%s' does not exist.", json_file_path) +# fre_logger.error('The file %s does not exist.', json_file_path) # raise # except json.JSONDecodeError: -# fre_logger.error("Failed to decode JSON from the file '%s'.", json_file_path) +# fre_logger.error('Failed to decode JSON from the file %s.', json_file_path) # raise # except Exception as e: -# fre_logger.error("An unexpected error occurred: %s", e) +# fre_logger.error('An unexpected error occurred: %s', e) # raise @@ -789,7 +789,7 @@ def filter_brands( brands: list, :param target_var: The base variable name (before the brand suffix). :type target_var: str :param mip_var_cfgs: The full MIP table config dict (must contain - ``"variable_entry"``). + ``'variable_entry'``). :type mip_var_cfgs: dict :param has_time_bnds: Whether the input dataset contains ``time_bnds``. :type has_time_bnds: bool @@ -809,7 +809,7 @@ def filter_brands( brands: list, filtered_brands = [] for brand in brands: mip_key = f'{target_var}_{brand}' - mip_dims = mip_var_cfgs["variable_entry"][mip_key]["dimensions"] + mip_dims = mip_var_cfgs['variable_entry'][mip_key]['dimensions'] # time filter if has_time_bnds and 'time1' in mip_dims: diff --git a/fremorizer/cmor_init.py b/fremorizer/cmor_init.py index 5352d95..7ae2b2b 100644 --- a/fremorizer/cmor_init.py +++ b/fremorizer/cmor_init.py @@ -50,56 +50,56 @@ def _cmip6_exp_config_template(): """Return an ordered dict-like structure for an empty CMIP6 experiment config.""" return { - "#note": " **** CMIP6 experiment configuration template – fill in values below ****", - "source_type": "", - "experiment_id": "", - "activity_id": "", - "sub_experiment_id": "none", - "realization_index": "1", - "initialization_index": "1", - "physics_index": "1", - "forcing_index": "1", - "run_variant": "", - "parent_experiment_id": "no parent", - "parent_activity_id": "no parent", - "parent_source_id": "no parent", - "parent_variant_label": "no parent", - "parent_time_units": "no parent", - "branch_method": "no parent", - "branch_time_in_child": 0.0, - "branch_time_in_parent": 0.0, - "institution_id": "", - "source_id": "", - "calendar": "", - "grid": "", - "grid_label": "", - "nominal_resolution": "", - "license": "", - "outpath": "", - "contact": "", - "history": "", - "comment": "", - "references": "", - "sub_experiment": "none", - "institution": "", - "source": "", - "_controlled_vocabulary_file": "CMIP6_CV.json", - "_AXIS_ENTRY_FILE": "CMIP6_coordinate.json", - "_FORMULA_VAR_FILE": "CMIP6_formula_terms.json", - "_cmip6_option": "CMIP6", - "mip_era": "CMIP6", - "parent_mip_era": "no parent", - "tracking_prefix": "hdl:21.14100", - "_history_template": ( - "%s ;rewrote data to be consistent with " - " for variable found in table ." + '#note': ' **** CMIP6 experiment configuration template – fill in values below ****', + 'source_type': '', + 'experiment_id': '', + 'activity_id': '', + 'sub_experiment_id': 'none', + 'realization_index': '1', + 'initialization_index': '1', + 'physics_index': '1', + 'forcing_index': '1', + 'run_variant': '', + 'parent_experiment_id': 'no parent', + 'parent_activity_id': 'no parent', + 'parent_source_id': 'no parent', + 'parent_variant_label': 'no parent', + 'parent_time_units': 'no parent', + 'branch_method': 'no parent', + 'branch_time_in_child': 0.0, + 'branch_time_in_parent': 0.0, + 'institution_id': '', + 'source_id': '', + 'calendar': '', + 'grid': '', + 'grid_label': '', + 'nominal_resolution': '', + 'license': '', + 'outpath': '', + 'contact': '', + 'history': '', + 'comment': '', + 'references': '', + 'sub_experiment': 'none', + 'institution': '', + 'source': '', + '_controlled_vocabulary_file': 'CMIP6_CV.json', + '_AXIS_ENTRY_FILE': 'CMIP6_coordinate.json', + '_FORMULA_VAR_FILE': 'CMIP6_formula_terms.json', + '_cmip6_option': 'CMIP6', + 'mip_era': 'CMIP6', + 'parent_mip_era': 'no parent', + 'tracking_prefix': 'hdl:21.14100', + '_history_template': ( + '%s ;rewrote data to be consistent with ' + ' for variable found in table .' ), - "output_path_template": ( - "" - "<_member_id>" + 'output_path_template': ( + '' + '<_member_id>
' ), - "output_file_template": ( - "
<_member_id>" + 'output_file_template': ( + '
<_member_id>' ), } @@ -107,57 +107,57 @@ def _cmip6_exp_config_template(): def _cmip7_exp_config_template(): """Return an ordered dict-like structure for an empty CMIP7 experiment config.""" return { - "#note": " **** CMIP7 experiment configuration template – fill in values below ****", - "contact": "MIP participant mipmember@foobar.c.om", - "comment": "additional important information not fitting into other fields can be placed here", - "license": "", - "references": "", - "drs_specs": "MIP-DRS7", - "archive_id": "WCRP", - "license_id": "CC-BY-4-0", - "tracking_prefix": "hdl:21.14107", - "_cmip7_option": 1, - "mip_era": "CMIP7", - "activity_id": "CMIP", - "institution": "", - "institution_id": "", - "source": "", - "source_id": "", - "source_type": "", - "experiment_id": "", - "sub_experiment": "none", - "sub_experiment_id": "none", - "realization_index": "r1", - "initialization_index": "i1", - "physics_index": "p1", - "forcing_index": "f1", - "run_variant": "", - "branch_method": "no parent", - "branch_time_in_child": 0.0, - "branch_time_in_parent": 0.0, - "calendar": "", - "grid": "PLACEHOLD", - "grid_label": "g999", - "frequency": "", - "region": "", - "nominal_resolution": "", - "history": "", - "_history_template": ( - "%s ;rewrote data to be consistent with " - " for variable found in table ." + '#note': ' **** CMIP7 experiment configuration template – fill in values below ****', + 'contact': 'MIP participant mipmember@foobar.c.om', + 'comment': 'additional important information not fitting into other fields can be placed here', + 'license': '', + 'references': '', + 'drs_specs': 'MIP-DRS7', + 'archive_id': 'WCRP', + 'license_id': 'CC-BY-4-0', + 'tracking_prefix': 'hdl:21.14107', + '_cmip7_option': 1, + 'mip_era': 'CMIP7', + 'activity_id': 'CMIP', + 'institution': '', + 'institution_id': '', + 'source': '', + 'source_id': '', + 'source_type': '', + 'experiment_id': '', + 'sub_experiment': 'none', + 'sub_experiment_id': 'none', + 'realization_index': 'r1', + 'initialization_index': 'i1', + 'physics_index': 'p1', + 'forcing_index': 'f1', + 'run_variant': '', + 'branch_method': 'no parent', + 'branch_time_in_child': 0.0, + 'branch_time_in_parent': 0.0, + 'calendar': '', + 'grid': 'PLACEHOLD', + 'grid_label': 'g999', + 'frequency': '', + 'region': '', + 'nominal_resolution': '', + 'history': '', + '_history_template': ( + '%s ;rewrote data to be consistent with ' + ' for variable found in table .' ), - "outpath": ".", - "output_path_template": ( - "" - "" + 'outpath': '.', + 'output_path_template': ( + '' + '' ), - "output_file_template": ( - "" - "" + 'output_file_template': ( + '' + '' ), - "_controlled_vocabulary_file": "../tables-cvs/cmor-cvs.json", - "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", - "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json", + '_controlled_vocabulary_file': '../tables-cvs/cmor-cvs.json', + '_AXIS_ENTRY_FILE': 'CMIP7_coordinate.json', + '_FORMULA_VAR_FILE': 'CMIP7_formula_terms.json', } @@ -273,7 +273,7 @@ def cmor_init_subtool( """ mip_era_lower = mip_era.lower() if mip_era_lower not in ('cmip6', 'cmip7'): - raise ValueError(f"mip_era must be 'cmip6' or 'cmip7', got '{mip_era}'") + raise ValueError(f'mip_era must be cmip6 or cmip7, got {mip_era}') result = {'exp_config': None, 'tables_dir': None} diff --git a/fremorizer/cmor_mixer.py b/fremorizer/cmor_mixer.py index 091269b..e83b012 100644 --- a/fremorizer/cmor_mixer.py +++ b/fremorizer/cmor_mixer.py @@ -86,12 +86,12 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, .. note:: This function performs extensive setup of axes and metadata, and conditionally handles tripolar ocean grids. """ - fre_logger.info("input data:") - fre_logger.info(" local_var = %s", local_var) - fre_logger.info(" target_var = %s", target_var) + fre_logger.info('input data:') + fre_logger.info(' local_var = %s', local_var) + fre_logger.info(' target_var = %s', target_var) # open the input file - fre_logger.info("opening %s", netcdf_file) + fre_logger.info('opening %s', netcdf_file) ds = nc.Dataset(netcdf_file, 'r+') # read the input variable data @@ -106,7 +106,7 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, # grab var_dim var_dim = len(var.shape) - fre_logger.info("var_dim = %d, local_var = %s", var_dim, local_var) + fre_logger.info('var_dim = %d, local_var = %s', var_dim, local_var) # CMORizing ocean grids are implemented only for scalar quantities valued at the central T/h-point of the grid cell. # https://en.wikipedia.org/wiki/Arakawa_grids heavily consulted for this work. @@ -127,9 +127,9 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, exp_cfg_mip_era = get_json_file_data(json_exp_config)['mip_era'].upper() if exp_cfg_mip_era == 'CMIP7': brands = [] - for mip_var in mip_var_cfgs["variable_entry"].keys(): + for mip_var in mip_var_cfgs['variable_entry'].keys(): if all([ target_var == mip_var.split('_')[0], - var_dim == len(mip_var_cfgs["variable_entry"][mip_var]['dimensions']) ]): + var_dim == len(mip_var_cfgs['variable_entry'][mip_var]['dimensions']) ]): brands.append(mip_var.split('_')[1]) if len(brands)>0: @@ -155,9 +155,9 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, expected_mip_coord_dims = None try: if exp_cfg_mip_era == 'CMIP7': - expected_mip_coord_dims = mip_var_cfgs["variable_entry"][f'{target_var}_{var_brand}']["dimensions"] + expected_mip_coord_dims = mip_var_cfgs['variable_entry'][f'{target_var}_{var_brand}']['dimensions'] else: - expected_mip_coord_dims = mip_var_cfgs["variable_entry"][target_var]["dimensions"] + expected_mip_coord_dims = mip_var_cfgs['variable_entry'][target_var]['dimensions'] fre_logger.info( 'I am hoping to find data for the following coordinate dimensions:\n' @@ -173,39 +173,39 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, # Attempt to read lat/lon coordinates and bnds. will check for none later fre_logger.info('attempting to read coordinate, lat') - lat = from_dis_gimme_dis(from_dis=ds, gimme_dis="lat") + lat = from_dis_gimme_dis(from_dis=ds, gimme_dis='lat') fre_logger.info('attempting to read coordinate BNDS, lat_bnds') - lat_bnds = from_dis_gimme_dis(from_dis=ds, gimme_dis="lat_bnds") + lat_bnds = from_dis_gimme_dis(from_dis=ds, gimme_dis='lat_bnds') fre_logger.info('attempting to read coordinate, lon') - lon = from_dis_gimme_dis(from_dis=ds, gimme_dis="lon") + lon = from_dis_gimme_dis(from_dis=ds, gimme_dis='lon') fre_logger.info('attempting to read coordinate BNDS, lon_bnds') - lon_bnds = from_dis_gimme_dis(from_dis=ds, gimme_dis="lon_bnds") + lon_bnds = from_dis_gimme_dis(from_dis=ds, gimme_dis='lon_bnds') # read in time_coords + units fre_logger.info('attempting to read coordinate time, and units...') time_coords = from_dis_gimme_dis(from_dis=ds, gimme_dis='time') - time_coord_units = ds["time"].units - fre_logger.info(" time_coord_units = %s", time_coord_units) + time_coord_units = ds['time'].units + fre_logger.info(' time_coord_units = %s', time_coord_units) # check the calendar of the input netcdf file time coordinate, if present time_coords_calendar = None try: time_coords_calendar = get_time_calendar_value(ds['time']) except Exception: - fre_logger.debug("could not read time variable for calendar detection.") + fre_logger.debug('could not read time variable for calendar detection.') # if it's still None, give a warning and move on. if time_coords_calendar is None: - fre_logger.warning("WARNING input netcdf file's time coordinates do not have a calendar nor calendar_type field" - "this output could have the wrong calendar!") + fre_logger.warning('WARNING input file\'s time coordinates missing calendar and/or calendar_type field' + 'this output could have the wrong calendar!') else: - with open(json_exp_config, "r", encoding="utf-8") as file: + with open(json_exp_config, 'r', encoding='utf-8') as file: exp_cfg_calendar = json.load(file)['calendar'] if not calendars_are_equivalent(time_coords_calendar, exp_cfg_calendar): norm_time = normalize_calendar(time_coords_calendar) norm_cfg = normalize_calendar(exp_cfg_calendar) - raise ValueError(f"data calendar type {norm_time} " - f"does not match input config calendar type: {norm_cfg}") + raise ValueError(f'data calendar type {norm_time} ' + f'does not match input config calendar type: {norm_cfg}') # read in time_bnds, if present fre_logger.info('attempting to read coordinate BNDS, time_bnds') @@ -213,16 +213,16 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, # determine the vertical dimension by looping over netcdf variables vert_dim = get_vertical_dimension(ds, target_var) # returns int(0) if not present - fre_logger.info("Vertical dimension of %s: %s", target_var, vert_dim) + fre_logger.info('Vertical dimension of %s: %s', target_var, vert_dim) # Check var_dim and vert_dim and assign lev if relevant. - lev, lev_units = None, "1" + lev, lev_units = None, '1' lev_bnds = None if vert_dim != 0: if vert_dim.lower() not in ACCEPTED_VERT_DIMS: raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') #uncovered lev = ds[vert_dim] - if vert_dim.lower() != "landuse": + if vert_dim.lower() != 'landuse': lev_units = ds[vert_dim].units process_tripolar_data = all([uses_ocean_grid, lat is None, lon is None]) @@ -253,7 +253,7 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, raise FileNotFoundError('statics file not found.') from exc - fre_logger.info("statics file found.") + fre_logger.info('statics file found.') statics_file_name = Path(statics_file_path).name put_statics_file_here = str(Path(netcdf_file).parent) @@ -272,8 +272,8 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, statics_lon = from_dis_gimme_dis(statics_ds, 'geolon') fre_logger.info('') - print_data_minmax(statics_lat, "statics_lat") - print_data_minmax(statics_lon, "statics_lon") + print_data_minmax(statics_lat, 'statics_lat') + print_data_minmax(statics_lon, 'statics_lon') fre_logger.info('') # spherical lat and lon coords @@ -284,8 +284,8 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, lon[:] = statics_lon[:] fre_logger.info('') - print_data_minmax(lat[:], "lat") - print_data_minmax(lon[:], "lon") + print_data_minmax(lat[:], 'lat') + print_data_minmax(lon[:], 'lon') fre_logger.info('') # grab the corners of the cells, should have shape (yh+1, xh+1) @@ -294,8 +294,8 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, lon_c = from_dis_gimme_dis(statics_ds, 'geolon_c') fre_logger.info('') - print_data_minmax(lat_c, "lat_c") - print_data_minmax(lon_c, "lon_c") + print_data_minmax(lat_c, 'lat_c') + print_data_minmax(lon_c, 'lon_c') fre_logger.info('') # vertex @@ -318,8 +318,8 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, lon_bnds[:, :, 3] = lon_c[:-1, 1:] # SE corner fre_logger.info('') - print_data_minmax(lat_bnds[:], "lat_bnds") - print_data_minmax(lon_bnds[:], "lon_bnds") + print_data_minmax(lat_bnds[:], 'lat_bnds') + print_data_minmax(lon_bnds[:], 'lon_bnds') fre_logger.info('') # grab the h-point lat and lon @@ -328,8 +328,8 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, xh = from_dis_gimme_dis(ds, 'xh') fre_logger.info('') - print_data_minmax(yh[:], "yh") - print_data_minmax(xh[:], "xh") + print_data_minmax(yh[:], 'yh') + print_data_minmax(xh[:], 'xh') fre_logger.info('') yh_dim = len(yh) @@ -341,8 +341,8 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, xq = from_dis_gimme_dis(statics_ds, 'xq') fre_logger.info('') - print_data_minmax(yq, "yq") - print_data_minmax(xq, "xq") + print_data_minmax(yq, 'yq') + print_data_minmax(xq, 'xq') fre_logger.info('') xq_dim = len(xq) @@ -377,8 +377,8 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, fre_logger.info('type(xh_bnds[%d][1]) = %s', i, type(xh_bnds[i][1])) fre_logger.info('') - print_data_minmax(yh_bnds[:], "yh_bnds") - print_data_minmax(xh_bnds[:], "xh_bnds") + print_data_minmax(yh_bnds[:], 'yh_bnds') + print_data_minmax(xh_bnds[:], 'xh_bnds') fre_logger.info('') # now we set up the cmor module object @@ -392,11 +392,11 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, ) # read experiment configuration file - fre_logger.info("cmor is opening: json_exp_config = %s", json_exp_config) + fre_logger.info('cmor is opening: json_exp_config = %s', json_exp_config) cmor.dataset_json(json_exp_config) # load CMOR table - fre_logger.info("cmor is loading+setting json_table_config = %s", json_table_config) + fre_logger.info('cmor is loading+setting json_table_config = %s', json_table_config) loaded_cmor_table_cfg = cmor.load_table(json_table_config) cmor.set_table(loaded_cmor_table_cfg) @@ -412,30 +412,30 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, cmor_y = None if process_tripolar_data: fre_logger.warning('calling cmor.axis for a projected y coordinate!!') - cmor_y = cmor.axis("y_deg", coord_vals=yh[:], cell_bounds=yh_bnds[:], units="degrees") + cmor_y = cmor.axis('y_deg', coord_vals=yh[:], cell_bounds=yh_bnds[:], units='degrees') elif lat is None: fre_logger.warning('lat or lat_bnds is None, skipping assigning cmor_y') else: fre_logger.info('assigning cmor_y') if lat_bnds is None: - cmor_y = cmor.axis("latitude", coord_vals=lat[:], units="degrees_N") #uncovered + cmor_y = cmor.axis('latitude', coord_vals=lat[:], units='degrees_N') #uncovered else: - cmor_y = cmor.axis("latitude", coord_vals=lat[:], cell_bounds=lat_bnds, units="degrees_N") + cmor_y = cmor.axis('latitude', coord_vals=lat[:], cell_bounds=lat_bnds, units='degrees_N') fre_logger.info('DONE assigning cmor_y') # setup cmor longitude axis if relevant cmor_x = None if process_tripolar_data: fre_logger.warning('calling cmor.axis for a projected x coordinate!!') - cmor_x = cmor.axis("x_deg", coord_vals=xh[:], cell_bounds=xh_bnds[:], units="degrees") + cmor_x = cmor.axis('x_deg', coord_vals=xh[:], cell_bounds=xh_bnds[:], units='degrees') elif lon is None: fre_logger.warning('lon or lon_bnds is None, skipping assigning cmor_x') else: fre_logger.info('assigning cmor_x') if lon_bnds is None: - cmor_x = cmor.axis("longitude", coord_vals=lon[:], units="degrees_E") #uncovered + cmor_x = cmor.axis('longitude', coord_vals=lon[:], units='degrees_E') #uncovered else: - cmor_x = cmor.axis("longitude", coord_vals=lon[:], cell_bounds=lon_bnds, units="degrees_E") + cmor_x = cmor.axis('longitude', coord_vals=lon[:], cell_bounds=lon_bnds, units='degrees_E') fre_logger.info('DONE assigning cmor_x') cmor_grid = None @@ -456,14 +456,14 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, try: fre_logger.info('assigning cmor_time using time_bnds...') ntimes_passed=len(time_coords) - fre_logger.debug("Executing: \n" - "cmor.axis('time', \n" - " coord_vals = %s, \n" - " length = %s, \n" - " cell_bounds = %s, units = %s)", + fre_logger.debug('Executing: \n' + 'cmor.axis(\'time\', \n' + ' coord_vals = %s, \n' + ' length = %s, \n' + ' cell_bounds = %s, units = %s)', time_coords, ntimes_passed, time_bnds, time_coord_units ) - cmor_time = cmor.axis("time", + cmor_time = cmor.axis('time', units=time_coord_units, length=ntimes_passed, coord_vals=time_coords, @@ -473,14 +473,14 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, fre_logger.error('exc is %s', str(exc)) fre_logger.info('assigning cmor_time WITHOUT time_bnds...') ntimes_passed=len(time_coords) - fre_logger.debug("Executing: \n" - "cmor_time = cmor.axis('time', \n" - " coord_vals = %s, \n" - " length = %s, \n" - " cell_bounds = None, units = %s)", + fre_logger.debug('Executing: \n' + 'cmor_time = cmor.axis(\'time\', \n' + ' coord_vals = %s, \n' + ' length = %s, \n' + ' cell_bounds = None, units = %s)', time_coords, ntimes_passed, time_coord_units ) - cmor_time = cmor.axis("time", + cmor_time = cmor.axis('time', units=time_coord_units, length=ntimes_passed, coord_vals=time_coords, @@ -500,13 +500,13 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, if vert_dim.lower() in NON_HYBRID_SIGMA_COORDS: fre_logger.info('non-hybrid sigma coordinate case') - if vert_dim.lower() != "landuse": + if vert_dim.lower() != 'landuse': cmor_vert_dim_name = vert_dim cmor_z = cmor.axis(cmor_vert_dim_name, coord_vals=lev[:], units=lev_units) else: landuse_str_list = ['primary_and_secondary_land', 'pastures', 'crops', 'urban'] - cmor_vert_dim_name = "landUse" + cmor_vert_dim_name = 'landUse' cmor_z = cmor.axis(cmor_vert_dim_name, coord_vals=np.array( landuse_str_list, @@ -519,8 +519,8 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, lev_bnds = create_lev_bnds(bound_these=lev, with_these=ds['z_i']) fre_logger.info('created lev_bnds...') except Exception as exc: - fre_logger.error("the cmor module always requires vertical levels to have bounds.") - raise KeyError("CMOR requires the input data have vertical level boundaries (bnds)") from exc + fre_logger.error('the cmor module always requires vertical levels to have bounds.') + raise KeyError('CMOR requires the input data have vertical level boundaries (bnds)') from exc fre_logger.info('lev_bnds = \n%s', lev_bnds) cmor_z = cmor.axis('depth_coord', @@ -535,37 +535,37 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, ps = from_dis_gimme_dis(ds_ps, 'ps') # assign lev_half specifics - if vert_dim == "levhalf": - cmor_z = cmor.axis("alternate_hybrid_sigma_half", + if vert_dim == 'levhalf': + cmor_z = cmor.axis('alternate_hybrid_sigma_half', coord_vals=lev[:], units=lev_units) ierr_ap = cmor.zfactor(zaxis_id=cmor_z, - zfactor_name="ap_half", + zfactor_name='ap_half', axis_ids=[cmor_z, ], - zfactor_values=ds["ap_bnds"][:], - units=ds["ap_bnds"].units) + zfactor_values=ds['ap_bnds'][:], + units=ds['ap_bnds'].units) ierr_b = cmor.zfactor(zaxis_id=cmor_z, - zfactor_name="b_half", + zfactor_name='b_half', axis_ids=[cmor_z, ], - zfactor_values=ds["b_bnds"][:], - units=ds["b_bnds"].units) + zfactor_values=ds['b_bnds'][:], + units=ds['b_bnds'].units) else: - cmor_z = cmor.axis("alternate_hybrid_sigma", + cmor_z = cmor.axis('alternate_hybrid_sigma', coord_vals=lev[:], units=lev_units, - cell_bounds=ds[vert_dim + "_bnds"]) + cell_bounds=ds[vert_dim + '_bnds']) ierr_ap = cmor.zfactor(zaxis_id=cmor_z, - zfactor_name="ap", + zfactor_name='ap', axis_ids=[cmor_z, ], - zfactor_values=ds["ap"][:], - zfactor_bounds=ds["ap_bnds"][:], - units=ds["ap"].units) + zfactor_values=ds['ap'][:], + zfactor_bounds=ds['ap_bnds'][:], + units=ds['ap'].units) ierr_b = cmor.zfactor(zaxis_id=cmor_z, - zfactor_name="b", + zfactor_name='b', axis_ids=[cmor_z, ], - zfactor_values=ds["b"][:], - zfactor_bounds=ds["b_bnds"][:], - units=ds["b"].units) + zfactor_values=ds['b'][:], + zfactor_bounds=ds['b_bnds'][:], + units=ds['b'].units) fre_logger.info('ierr_ap after calling cmor_zfactor: %s\n', ierr_ap) fre_logger.info('ierr_b after calling cmor_zfactor: %s', ierr_b) @@ -586,9 +586,9 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, fre_logger.info('axis_ids now = %s', axis_ids) ips = cmor.zfactor(zaxis_id=cmor_z, - zfactor_name="ps", + zfactor_name='ps', axis_ids=axis_ids, - units="Pa") + units='Pa') save_ps = True @@ -618,19 +618,19 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, fre_logger.info('axes now = %s', axes) # read positive/units attribute and create cmor_var - #units = mip_var_cfgs["variable_entry"][target_var]["units"] + #units = mip_var_cfgs['variable_entry'][target_var]['units'] if exp_cfg_mip_era == 'CMIP7': - units = mip_var_cfgs["variable_entry"][f'{target_var}_{var_brand}']["units"] + units = mip_var_cfgs['variable_entry'][f'{target_var}_{var_brand}']['units'] else: - units = mip_var_cfgs["variable_entry"][target_var]["units"] - fre_logger.info("units = %s", units) + units = mip_var_cfgs['variable_entry'][target_var]['units'] + fre_logger.info('units = %s', units) - #positive = mip_var_cfgs["variable_entry"][target_var]["positive"] + #positive = mip_var_cfgs['variable_entry'][target_var]['positive'] if exp_cfg_mip_era == 'CMIP7': - positive = mip_var_cfgs["variable_entry"][f'{target_var}_{var_brand}']["positive"] + positive = mip_var_cfgs['variable_entry'][f'{target_var}_{var_brand}']['positive'] else: - positive = mip_var_cfgs["variable_entry"][target_var]["positive"] - fre_logger.info("positive = %s", positive) + positive = mip_var_cfgs['variable_entry'][target_var]['positive'] + fre_logger.info('positive = %s', positive) if exp_cfg_mip_era == 'CMIP7': fre_logger.info('cmor.variable call: for cmip7_target_var = %s ', f'{target_var}_{var_brand}') @@ -648,24 +648,24 @@ def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, # Write the output to disk #fre_logger.debug('var is: %s', var) - fre_logger.info("cmor.write call: for var data into cmor_var") + fre_logger.info('cmor.write call: for var data into cmor_var') cmor.write(cmor_var, var) - fre_logger.info("DONE cmor.write call: for var data into cmor_var") + fre_logger.info('DONE cmor.write call: for var data into cmor_var') if save_ps: if any([ips is None, ps is None]): fre_logger.warning('ps or ips is None!, but save_ps is True!\n' #uncovered 'ps = %s, ips = %s\n' 'skipping ps writing!', ps, ips) else: - fre_logger.info("cmor.write call: for interp-pressure data (ips)") + fre_logger.info('cmor.write call: for interp-pressure data (ips)') cmor.write(ips, ps, store_with=cmor_var, ntimes_passed=ntimes_passed) - fre_logger.info("DONE cmor.write call: for interp-pressure data (ips)") + fre_logger.info('DONE cmor.write call: for interp-pressure data (ips)') - fre_logger.info("cmor.close call: for cmor_var") + fre_logger.info('cmor.close call: for cmor_var') filename = cmor.close(cmor_var, file_name=True, preserve=False) - fre_logger.info("DONE cmor.close call: for cmor_var") + fre_logger.info('DONE cmor.close call: for cmor_var') filename = str( Path(filename).resolve() ) - fre_logger.info("returned by cmor.close: filename = %s", filename) + fre_logger.info('returned by cmor.close: filename = %s', filename) fre_logger.info('closing netcdf4 dataset... ds') ds.close() fre_logger.info('tearing-down the cmor module instance') @@ -717,10 +717,10 @@ def cmorize_target_var_files(indir: str = None, .. note:: Copies files to a temporary directory, runs CMORization, moves results to output, cleans up temp files. """ - fre_logger.info("local_var = %s to be used for file-targeting.\n" - "target_var = %s to be used for reading the data \n" - "from the file\n" - "outdir = %s", local_var, target_var, outdir) + fre_logger.info('local_var = %s to be used for file-targeting.\n' + 'target_var = %s to be used for reading the data \n' + 'from the file\n' + 'outdir = %s', local_var, target_var, outdir) # determine a tmp dir for working on files. tmp_dir = create_tmp_dir(outdir, json_exp_config) + '/' @@ -730,27 +730,27 @@ def cmorize_target_var_files(indir: str = None, nc_fls = {} for i, iso_datetime in enumerate(iso_datetime_range_arr): # why is nc_fls a filled list/array/object thingy here? see above line - nc_fls[i] = f"{indir}/{name_of_set}.{iso_datetime}.{local_var}.nc" + nc_fls[i] = f'{indir}/{name_of_set}.{iso_datetime}.{local_var}.nc' - fre_logger.info("input file = %s", nc_fls[i]) + fre_logger.info('input file = %s', nc_fls[i]) if not Path(nc_fls[i]).exists(): - fre_logger.warning("input file not found, omitting: %s", nc_fls[i]) + fre_logger.warning('input file not found, omitting: %s', nc_fls[i]) continue if not Path(nc_fls[i]).is_absolute(): nc_fls[i]=str(Path(nc_fls[i]).resolve()) # create a copy of the input file with local var name into the work directory - nc_file_work = f"{tmp_dir}{name_of_set}.{iso_datetime}.{local_var}.nc" + nc_file_work = f'{tmp_dir}{name_of_set}.{iso_datetime}.{local_var}.nc' - fre_logger.info("nc_file_work = %s", nc_file_work) + fre_logger.info('nc_file_work = %s', nc_file_work) shutil.copy(nc_fls[i], nc_file_work) # if the ps file exists, we'll copy it to the work directory too nc_ps_file = nc_fls[i].replace(f'.{local_var}.nc', '.ps.nc') nc_ps_file_work = nc_file_work.replace(f'.{local_var}.nc', '.ps.nc') if Path(nc_ps_file).exists(): - fre_logger.info("nc_ps_file_work = %s", nc_ps_file_work) + fre_logger.info('nc_ps_file_work = %s', nc_ps_file_work) shutil.copy(nc_ps_file, nc_ps_file_work) # TODO think of better way to write this kind of conditional data movement... @@ -762,12 +762,12 @@ def cmorize_target_var_files(indir: str = None, gotta_go_back_here = os.getcwd() try: - fre_logger.warning("changing directory to: \n%s", make_cmor_write_here) + fre_logger.warning('changing directory to: \n%s', make_cmor_write_here) os.chdir(make_cmor_write_here) except Exception as exc: #uncovered raise OSError(f'(cmorize_target_var_files) could not chdir to {make_cmor_write_here}') from exc - fre_logger.info("calling rewrite_netcdf_file_var") + fre_logger.info('calling rewrite_netcdf_file_var') try: local_file_name = rewrite_netcdf_file_var(mip_var_cfgs, local_var, @@ -785,7 +785,7 @@ def cmorize_target_var_files(indir: str = None, fre_logger.warning('finally, changing directory to: \n%s', gotta_go_back_here) os.chdir(gotta_go_back_here) -# assert False, "made it to break-point for current work, good job" +# assert False, 'made it to break-point for current work, good job' # now that CMOR has rewritten things... we can take our post-rewriting actions # first, remove /CMOR_tmp/ from the output path. @@ -795,30 +795,30 @@ def cmorize_target_var_files(indir: str = None, fre_logger.info('local_file_name = %s', local_file_name) filename = local_file_name.replace('/CMOR_tmp/','/') - fre_logger.info("filename = %s", filename) + fre_logger.info('filename = %s', filename) # the final output file directory will be... filedir = Path(filename).parent - fre_logger.info("FINAL OUTPUT FILE DIR WILL BE filedir = %s", filedir) + fre_logger.info('FINAL OUTPUT FILE DIR WILL BE filedir = %s', filedir) try: fre_logger.info('ATTEMPTING TO CREATE filedir=%s', filedir) os.makedirs(filedir) except FileExistsError: fre_logger.warning('directory %s already exists!', filedir) - mv_cmd = f"mv {local_file_name} {filedir}" - fre_logger.info("moving files...\n%s", mv_cmd) + mv_cmd = f'mv {local_file_name} {filedir}' + fre_logger.info('moving files...\n%s', mv_cmd) subprocess.run(mv_cmd, shell=True, check=True) # ------ refactor this into function? #TODO # ------ what is the use case for this logic really?? - filename_no_nc = filename[:filename.rfind(".nc")] + filename_no_nc = filename[:filename.rfind('.nc')] chunk_str = filename_no_nc[-6:] if not chunk_str.isdigit(): fre_logger.warning('chunk_str is not a digit: chunk_str = %s', chunk_str) #uncovered - filename_corr = f"{filename[:filename.rfind('.nc')]}_{iso_datetime}.nc" - mv_cmd = f"mv {filename} {filename_corr}" - fre_logger.warning("moving files, strange chunkstr logic...\n%s", mv_cmd) + filename_corr = f'{filename[:filename.rfind(".nc")]}_{iso_datetime}.nc' + mv_cmd = f'mv {filename} {filename_corr}' + fre_logger.warning('moving files, strange chunkstr logic...\n%s', mv_cmd) subprocess.run(mv_cmd, shell=True, check=True) # ------ end refactor this into function? @@ -875,7 +875,7 @@ def cmorize_all_variables_in_dir(vars_to_run: Dict[str, Any], return_status = -1 omissions = [] for local_var in vars_to_run: - # if the target-variable is "good", get the name of the data inside the netcdf file. + # if the target-variable is 'good', get the name of the data inside the netcdf file. target_var = vars_to_run[local_var] # often equiv to local_var but not necessarily. if local_var != target_var: fre_logger.warning('local_var == %s != %s == target_var\n' @@ -974,14 +974,14 @@ def cmor_run_subtool(indir: str = None, if None in [indir, json_var_list, json_table_config, json_exp_config, outdir]: raise ValueError('the following input arguments are required:\n' '[indir, json_var_list, json_table_config, json_exp_config, outdir] = \n' - '[%s, %s, %s, %s, %s]', indir, json_var_list, json_table_config, json_exp_config, outdir) + f'[{indir}, {json_var_list}, {json_table_config}, {json_exp_config}, {outdir}]') # CHECK existence of the exp-specific metadata file if Path(json_exp_config).exists(): json_exp_config = str(Path(json_exp_config).resolve()) else: raise FileNotFoundError('ERROR: json_exp_config file cannot be opened.\n' - 'json_exp_config = %s', json_exp_config) + f'json_exp_config = {json_exp_config}') # CHECK mip_era entry of exp config exists, needed ? try: @@ -1027,7 +1027,7 @@ def cmor_run_subtool(indir: str = None, f' experiment mip_era: {exp_cfg_mip_era}\n' f' table format detected in {json_table_config}: {table_mip_era}\n' ' supply a MIP table that matches the experiment mip_era.') - mip_fullvar_list = mip_var_cfgs["variable_entry"].keys() + mip_fullvar_list = mip_var_cfgs['variable_entry'].keys() fre_logger.debug('the following variables were read from the table: %s', mip_fullvar_list) # make the TABLE's variable list, and brand list (if CMIP7) @@ -1039,7 +1039,7 @@ def cmor_run_subtool(indir: str = None, mip_var_brand_list = [ var.split('_')[1] for var in mip_fullvar_list ] if len(mip_var_list) != len(mip_var_brand_list): raise ValueError('the number of brands is not one-to-one with the number of variables. check config.') - elif exp_cfg_mip_era == "CMIP6": + elif exp_cfg_mip_era == 'CMIP6': mip_var_list = mip_fullvar_list fre_logger.debug('list of table variables we will process = \n %s', mip_var_list) @@ -1062,7 +1062,7 @@ def cmor_run_subtool(indir: str = None, if opt_var_name is not None and opt_var_name in mip_var_list: vars_to_run[opt_var_name] = opt_var_name break - if var_list[local_var] not in mip_var_list: #mip_var_cfgs["variable_entry"]: + if var_list[local_var] not in mip_var_list: #mip_var_cfgs['variable_entry']: fre_logger.warning('skipping local_var = %s /\n' 'target_var = %s\n' 'target_var not found in CMOR variable group', local_var, var_list[local_var]) @@ -1077,10 +1077,10 @@ def cmor_run_subtool(indir: str = None, raise ValueError('runnable variable list is of length 0 ' 'this means no variables in input variable list are in ' 'the mip table configuration, so there\'s nothing to process!') - if all([opt_var_name is not None, opt_var_name not in list(vars_to_run.keys())]): - raise ValueError('opt_var_name is not None! (== %s)' - '... but the variable is not contained in the target mip table' - '... there\'s nothing to process, exit', opt_var_name) + if all([opt_var_name is not None, opt_var_name not in list(vars_to_run)]): + raise ValueError(f'opt_var_name is not None! (== {opt_var_name})' + '... but the variable is not contained in the target mip table' + '... there\'s nothing to process, exit') fre_logger.info('runnable variable list formed, it is vars_to_run=\n%s', vars_to_run) @@ -1090,11 +1090,11 @@ def cmor_run_subtool(indir: str = None, indir_filenames = glob.glob(f'{indir}/*.nc') indir_filenames.sort() if len(indir_filenames) == 0: - raise ValueError('no files in input target directory = indir = \n%s', indir) + raise ValueError(f'no files in input target directory = indir = \n{indir}') fre_logger.debug('found %s filenames', len(indir_filenames)) # name_of_set == component label - name_of_set = Path(indir_filenames[0]).name.split(".")[0] + name_of_set = Path(indir_filenames[0]).name.split('.')[0] fre_logger.info('setting name_of_set = %s', name_of_set) # make list of iso-datetimes here diff --git a/fremorizer/cmor_yamler.py b/fremorizer/cmor_yamler.py index 9c31c38..db029e7 100644 --- a/fremorizer/cmor_yamler.py +++ b/fremorizer/cmor_yamler.py @@ -93,11 +93,11 @@ def cmor_yaml_subtool( yamlfile: str = None, fre_logger.info('calling consolidate yamls to create a combined cmor-yaml dictionary') if consolidate_yamls is None: raise ImportError( - "the 'fremor yaml' command requires fre-cli's yamltools module.\n" - "install it with: pip install fre-cli") + 'the \'fremor yaml\' command requires fre-cli\'s yamltools module.\n' + 'install it with: pip install fre-cli') cmor_yaml_dict = consolidate_yamls(yamlfile=yamlfile, experiment=exp_name, platform=platform, target=target, - use="cmor", output=output)['cmor'] + use='cmor', output=output)['cmor'] fre_logger.debug('consolidate_yamls produced the following dictionary of cmor-settings from yamls: \n%s', pprint.pformat(cmor_yaml_dict) ) diff --git a/fremorizer/tests/conftest.py b/fremorizer/tests/conftest.py index cf4b94e..c3c7633 100644 --- a/fremorizer/tests/conftest.py +++ b/fremorizer/tests/conftest.py @@ -1,8 +1,9 @@ -''' +""" Shared fixtures for fremorizer/tests CLI integration tests. -''' +""" from datetime import date +import json from pathlib import Path import shutil import subprocess @@ -30,10 +31,170 @@ YYYYMMDD = date.today().strftime('%Y%m%d') +# ── raw experiment-config contents (kept in-code for fixture use) ─────────── +# pylint: disable=line-too-long +_CMIP6_EXP_CONFIG_DATA = { + '#note': ' **** The following are set correctly for CMIP6 and should not normally need editing', + 'source_type': 'AOGCM ISM AER', + 'experiment_id': 'piControl-withism', + 'activity_id': 'ISMIP6', + 'sub_experiment_id': 'none', + 'realization_index': '3', + 'initialization_index': '1', + 'physics_index': '1', + 'forcing_index': '1', + 'run_variant': '3rd realization', + 'parent_experiment_id': 'no parent', + 'parent_activity_id': 'no parent', + 'parent_source_id': 'no parent', + 'parent_variant_label': 'no parent', + 'parent_time_units': 'no parent', + 'branch_method': 'no parent', + 'branch_time_in_child': 59400.0, + 'branch_time_in_parent': 0.0, + 'institution_id': 'PCMDI', + 'source_id': 'PCMDI-test-1-0', + 'calendar': 'julian', + 'grid': 'FOO_BAR_PLACEHOLD', + 'grid_label': 'gr', + 'nominal_resolution': '10000 km', + 'license': 'CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.', + '#output': 'Root directory for output (can be either a relative or full path)', + 'outpath': 'CMIP6', + 'contact ': 'Python Coder (coder@a.b.c.com)', + 'history': 'Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.', + 'comment': '', + 'references': 'Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html. The ssp245 simulation is described in Dorkey et al. \'(Clim. Dyn., 2003, 323-357.)\'', + 'sub_experiment': 'none', + 'institution': '', + 'source': 'PCMDI-test 1.0 (1989)', + '_controlled_vocabulary_file': 'CMIP6_CV.json', + '_AXIS_ENTRY_FILE': 'CMIP6_coordinate.json', + '_FORMULA_VAR_FILE': 'CMIP6_formula_terms.json', + '_cmip6_option': 'CMIP6', + 'mip_era': 'CMIP6', + 'parent_mip_era': 'no parent', + 'tracking_prefix': 'hdl:21.14100', + '_history_template': '%s ;rewrote data to be consistent with for variable found in table .', + '#output_path_template': 'Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax', + 'output_path_template': '<_member_id>
', + 'output_file_template': '
<_member_id>' +} + +_CMIP7_EXP_CONFIG_DATA = { + '#_TESTING_ONLY': ' ***** This is for unit-test functionality of NOAA-GDFL\'s fremorizer module for CMIP7, they do not reflect values used in actual production *****', + 'contact ': 'MIP participant mipmember@foobar.c.om', + 'comment': 'additional important information not fitting into other fields can be placed here', + 'license_id': 'CC-BY-4.0', + 'license_url': 'https://creativecommons.org/licenses/by/4.0', + 'license': '; CMIP7 data produced by is licensed under a License (). Consult https://wcrp-cmip.github.io/cmip7-guidance/docs/CMIP7/Guidance_for_users/#2-terms-of-use-and-citations-requirements for terms of use governing CMIP7 output, including citation requirements and proper acknowledgment. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.', + '#_TRACKING': '***** anything to do with citing this data, accreditation, licensing, and references go here *****', + 'references': 'Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html. The ssp245 simulation is described in Dorkey et al. \'(Clim. Dyn., 2003, 323-357.)\'', + 'drs_specs': 'MIP-DRS7', + 'archive_id': 'WCRP', + 'tracking_prefix': 'hdl:21.14107', + '#_MIP_DETAILS': '***** anything to do with identifying the specific MIP activity this configuration file is for *****', + '_cmip7_option': 1, + 'mip_era': 'CMIP7', + 'parent_mip_era': 'CMIP7', + 'activity_id': 'CMIP', + 'parent_activity_id': 'CMIP', + '#_SOURCE_SECTION': '***** anything to do with identifying this experiment, it\'s relationships to other experiments, the producers, and their institution *****', + 'institution': '', + 'institution_id': 'CCCma', + 'source': 'DUMMY-MODEL: aerosol: Dummy Aerosol; atmosphere: Dummy Atmosphere; atmospheric_chemistry: Dummy Atmospheric Chemistry; land_surface: Dummy Land Surface; ocean: Dummy Ocean; ocean_biogeochemistry: Dummy Ocean Biogeochemistry; sea_ice: Dummy Sea Ice', + 'parent_source_id': 'DUMMY-MODEL', + 'source_id': 'DUMMY-MODEL', + 'source_type': 'AOGCM ISM AER', + 'parent_experiment_id': 'piControl', + 'experiment_id': 'historical', + 'sub_experiment': 'none', + 'sub_experiment_id': 'none', + '#_INDICES': '***** changed from ints to strings for CMIP7 *****', + 'realization_index': 'r3', + 'initialization_index': 'i1', + 'physics_index': 'p1', + 'forcing_index': 'f3', + 'run_variant': '3rd realization', + 'parent_variant_label': 'r3i1p1f3', + '#_TEMPORAL_INFO': '***** anything to do with describing temporal aspects of the experiment *****', + 'parent_time_units': 'days since 1850-01-01', + 'branch_method': 'no parent', + 'branch_time_in_child': 59400.0, + 'branch_time_in_parent': 0.0, + 'calendar': 'julian', + '#_SPATIAL_INFO': '***** anything to do with describing physical aspects of the experiment *****', + 'grid': 'FOO_BAR_PLACEHOLD', + 'grid_label': 'g999', + 'frequency': 'mon', + 'region': 'glb', + 'nominal_resolution': '10000 km', + '#_HISTORY_METADATA': 'history attribute string and template, to create history field for output file', + 'history': 'Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.', + '_history_template': '%s ;rewrote data to be consistent with for variable found in table .', + '#_OUTPUT_PATHS': '***** pathing/templates for output files *****', + '#_output_template_NOTE': '***** PCMDI/cmor 4e7f1f3d731077b7f65c188edefac924cc3e2779 Test/test_cmor_CMIP7.py L47 *****', + '#_output': '***** Root directory for output (can be either a relative or full path) *****', + 'outpath': '.', + '#_output_path_template': '***** Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax *****', + 'output_path_template': '', + '#_output_file_template': '***** Template for output filename using tables keys or global attributes, these should follow the relevant data reference syntax *****', + 'output_file_template': '', + '#_INPUT_CONFIG_PATHS': '***** pathing/templates for input configuration files holding controlled vocabularies *****', + '_controlled_vocabulary_file': '../tables-cvs/cmor-cvs.json', + '_AXIS_ENTRY_FILE': 'CMIP7_coordinate.json', + '_FORMULA_VAR_FILE': 'CMIP7_formula_terms.json' +} +# pylint: enable=line-too-long + + +# ── experiment-config fixtures ────────────────────────────────────────────── +@pytest.fixture(autouse=True, scope='session') +def _write_exp_configs(): + """Write both experiment-config JSONs to ROOTDIR at the start of every session. + + The JSON data lives in this module (_CMIP6_EXP_CONFIG_DATA / + _CMIP7_EXP_CONFIG_DATA) so the on-disk files are no longer tracked by git. + This session-scoped autouse fixture materialises fresh copies before any + test that needs them runs, and cleans them up afterwards. + """ + EXP_CONFIG.write_text(json.dumps(_CMIP6_EXP_CONFIG_DATA, indent=4)) + EXP_CONFIG_CMIP7.write_text(json.dumps(_CMIP7_EXP_CONFIG_DATA, indent=4)) + yield + # restore pristine copies so later sessions (or re-runs) start clean + EXP_CONFIG.write_text(json.dumps(_CMIP6_EXP_CONFIG_DATA, indent=4)) + EXP_CONFIG_CMIP7.write_text(json.dumps(_CMIP7_EXP_CONFIG_DATA, indent=4)) + + +@pytest.fixture +def cmip6_exp_config(tmp_path): + """Write the CMIP6 experiment config JSON to a temp file and return its path.""" + path = tmp_path / 'CMOR_input_example.json' + path.write_text(json.dumps(_CMIP6_EXP_CONFIG_DATA, indent=4)) + return str(path) + + +@pytest.fixture +def cmip7_exp_config(tmp_path): + """Write the CMIP7 experiment config JSON to a temp file and return its path.""" + path = tmp_path / 'CMOR_CMIP7_input_example.json' + path.write_text(json.dumps(_CMIP7_EXP_CONFIG_DATA, indent=4)) + return str(path) + + # ── ncgen helper ──────────────────────────────────────────────────────────── -def _ncgen(cdl_name, nc_path): - """Run ncgen3 to convert a CDL file into a NetCDF-4 file.""" - cdl_path = ROOTDIR / 'reduced_ascii_files' / cdl_name +def ncgen(cdl_path, nc_path): + """Run ncgen3 to convert a CDL file into a NetCDF-4 file. + + Parameters + ---------- + cdl_path : str or Path + Full path to the CDL source file. + nc_path : str or Path + Full path where the NetCDF-4 file will be written. + """ + cdl_path = Path(cdl_path) + nc_path = Path(nc_path) assert cdl_path.exists(), f'CDL file not found: {cdl_path}' if nc_path.exists(): @@ -46,6 +207,11 @@ def _ncgen(cdl_name, nc_path): assert nc_path.exists(), f'ncgen3 failed to create {nc_path}' +def _ncgen(cdl_name, nc_path): + """Run ncgen3 for a CDL file under ``ROOTDIR/reduced_ascii_files``.""" + ncgen(ROOTDIR / 'reduced_ascii_files' / cdl_name, nc_path) + + # ── session-scoped fixtures ───────────────────────────────────────────────── @pytest.fixture(scope='session') def cli_sos_nc_file(): diff --git a/fremorizer/tests/test_cli.py b/fremorizer/tests/test_cli.py index 66506a5..cb55907 100644 --- a/fremorizer/tests/test_cli.py +++ b/fremorizer/tests/test_cli.py @@ -1,4 +1,5 @@ -'''CLI Tests for fremor subcommands +""" +CLI Tests for fremor subcommands Tests the command-line-interface calls for the fremor CLI (fremorizer package). Each tool generally gets 3 tests: @@ -12,7 +13,7 @@ command-line args for fremor yaml and fremor run. Migrated from NOAA-GFDL/fre-cli fre/tests/test_fre_cmor_cli.py. -''' +""" import json import os @@ -51,20 +52,20 @@ def test_setup_test_files(cli_sos_nc_file, cli_sosv2_nc_file): # pylint: disable # ── fremor (top-level group) ────────────────────────────────────────────── def test_cli_fremor(): - ''' fremor (no subcommand) ''' + """ fremor (no subcommand) """ result = runner.invoke(fremor, args=[]) assert result.exit_code == 2 def test_cli_fremor_help(): - ''' fremor --help ''' - result = runner.invoke(fremor, args=["--help"]) + """ fremor --help """ + result = runner.invoke(fremor, args=['--help']) assert result.exit_code == 0 def test_cli_fremor_help_and_debuglog(tmp_path): - ''' fremor -vv -l LOG yaml --help (logs created by group callback) ''' + """ fremor -vv -l LOG yaml --help (logs created by group callback) """ log_file = tmp_path / 'TEST_FOO_LOG.log' - result = runner.invoke(fremor, args=["-vv", "-l", str(log_file), "yaml", "--help"]) + result = runner.invoke(fremor, args=['-vv', '-l', str(log_file), 'yaml', '--help']) assert result.exit_code == 0 assert log_file.exists() @@ -73,10 +74,10 @@ def test_cli_fremor_help_and_debuglog(tmp_path): assert LOG_DEBUG_LINE in line_list[1] def test_cli_fremor_help_and_infolog(tmp_path): - ''' fremor -v -l LOG yaml --help ''' + """ fremor -v -l LOG yaml --help """ log_file = tmp_path / 'TEST_FOO_LOG.log' - result = runner.invoke(fremor, args=["-v", "-l", str(log_file), "yaml", "--help"]) + result = runner.invoke(fremor, args=['-v', '-l', str(log_file), 'yaml', '--help']) assert result.exit_code == 0 assert log_file.exists() @@ -84,10 +85,10 @@ def test_cli_fremor_help_and_infolog(tmp_path): assert LOG_INFO_LINE in line_list[0] def test_cli_fremor_help_and_quietlog(tmp_path): - ''' fremor -q -l LOG yaml --help ''' + """ fremor -q -l LOG yaml --help """ log_file = tmp_path / 'TEST_FOO_LOG.log' - result = runner.invoke(fremor, args=["-q", "-l", str(log_file), "yaml", "--help"]) + result = runner.invoke(fremor, args=['-q', '-l', str(log_file), 'yaml', '--help']) assert result.exit_code == 0 assert log_file.exists() @@ -95,31 +96,31 @@ def test_cli_fremor_help_and_quietlog(tmp_path): assert line_list == [] def test_cli_fremor_opt_dne(): - ''' fremor optionDNE ''' - result = runner.invoke(fremor, args=["optionDNE"]) + """ fremor optionDNE """ + result = runner.invoke(fremor, args=['optionDNE']) assert result.exit_code == 2 # ── fremor yaml ─────────────────────────────────────────────────────────── def test_cli_fremor_yaml(): - ''' fremor yaml (no args) ''' - result = runner.invoke(fremor, args=["yaml"]) + """ fremor yaml (no args) """ + result = runner.invoke(fremor, args=['yaml']) assert result.exit_code == 2 def test_cli_fremor_yaml_help(): - ''' fremor yaml --help ''' - result = runner.invoke(fremor, args=["yaml", "--help"]) + """ fremor yaml --help """ + result = runner.invoke(fremor, args=['yaml', '--help']) assert result.exit_code == 0 def test_cli_fremor_yaml_opt_dne(): - ''' fremor yaml optionDNE ''' - result = runner.invoke(fremor, args=["yaml", "optionDNE"]) + """ fremor yaml optionDNE """ + result = runner.invoke(fremor, args=['yaml', 'optionDNE']) assert result.exit_code == 2 @patch('fremorizer.cli.cmor_yaml_subtool') def test_cli_fremor_yaml_case1(mock_subtool, tmp_path): - ''' fremor yaml --dry_run -y YAMLFILE ... --output FOO_cmor.yaml ''' + """ fremor yaml --dry_run -y YAMLFILE ... --output FOO_cmor.yaml """ # use a temporary yaml placeholder file as the model yaml input dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder', encoding='utf-8') @@ -127,19 +128,19 @@ def test_cli_fremor_yaml_case1(mock_subtool, tmp_path): mock_subtool.return_value = None - result = runner.invoke(fremor, args=["-v", "-v", "yaml", "--dry_run", - "-y", str(dummy_yaml), - "-e", "test_experiment", - "-p", "test_platform", - "-t", "test_target", - "--output", str(output_yaml) ]) + result = runner.invoke(fremor, args=['-v', '-v', 'yaml', '--dry_run', + '-y', str(dummy_yaml), + '-e', 'test_experiment', + '-p', 'test_platform', + '-t', 'test_target', + '--output', str(output_yaml) ]) assert result.exit_code == 0 mock_subtool.assert_called_once_with( yamlfile=str(dummy_yaml), - exp_name="test_experiment", - target="test_target", - platform="test_platform", + exp_name='test_experiment', + target='test_target', + platform='test_platform', output=str(output_yaml), run_one_mode=False, dry_run_mode=True, @@ -152,36 +153,36 @@ def test_cli_fremor_yaml_case1(mock_subtool, tmp_path): # ── fremor run ──────────────────────────────────────────────────────────── def test_cli_fremor_run(): - ''' fremor run (no args) ''' - result = runner.invoke(fremor, args=["run"]) + """ fremor run (no args) """ + result = runner.invoke(fremor, args=['run']) assert result.exit_code == 2 def test_cli_fremor_run_help(): - ''' fremor run --help ''' - result = runner.invoke(fremor, args=["run", "--help"]) + """ fremor run --help """ + result = runner.invoke(fremor, args=['run', '--help']) assert result.exit_code == 0 def test_cli_fremor_run_opt_dne(): - ''' fremor run optionDNE ''' - result = runner.invoke(fremor, args=["run", "optionDNE"]) + """ fremor run optionDNE """ + result = runner.invoke(fremor, args=['run', 'optionDNE']) assert result.exit_code == 2 def test_cli_fremor_run_case1(cli_sos_nc_file, tmp_path): - '''fremor run, test-use case: sos → sos (CMIP6)''' + """fremor run, test-use case: sos → sos (CMIP6)""" outdir = str(tmp_path / 'outdir') - result = runner.invoke(fremor, args = [ "-v", "-v", - "run", "--run_one", - "--indir", str(INDIR), - "--varlist", str(VARLIST), - "--table_config", str(CMIP6_TABLE_CONFIG), - "--exp_config", str(EXP_CONFIG), - "--outdir", outdir, - "--calendar", "julian", - "--grid_label", "gr", - "--grid_desc", "FOO_BAR_PLACEHOLD", - "--nom_res", "10000 km" ] ) + result = runner.invoke(fremor, args = [ '-v', '-v', + 'run', '--run_one', + '--indir', str(INDIR), + '--varlist', str(VARLIST), + '--table_config', str(CMIP6_TABLE_CONFIG), + '--exp_config', str(EXP_CONFIG), + '--outdir', outdir, + '--calendar', 'julian', + '--grid_label', 'gr', + '--grid_desc', 'FOO_BAR_PLACEHOLD', + '--nom_res', '10000 km' ] ) assert result.exit_code == 0, f'case1 failed: {result.output}' output_ncs = list(Path(outdir).rglob('sos_Omon_*.nc')) @@ -190,20 +191,20 @@ def test_cli_fremor_run_case1(cli_sos_nc_file, tmp_path): def test_cli_fremor_run_case2(cli_sosv2_nc_file, tmp_path): # pylint: disable=redefined-outer-name - '''fremor run, test-use case 2: sosV2 varlist_diff (CMIP6)''' + """fremor run, test-use case 2: sosV2 varlist_diff (CMIP6)""" outdir = str(tmp_path / 'outdir') - result = runner.invoke(fremor, args = ["-v", "-v", - "run", "--run_one", - "--indir", str(INDIR), - "--varlist", str(VARLIST_DIFF), - "--table_config", str(CMIP6_TABLE_CONFIG), - "--exp_config", str(EXP_CONFIG), - "--outdir", outdir, - "--calendar", "julian", - "--grid_label", "gr", - "--grid_desc", "FOO_BAR_PLACEHOLD", - "--nom_res", "10000 km" ] ) + result = runner.invoke(fremor, args = ['-v', '-v', + 'run', '--run_one', + '--indir', str(INDIR), + '--varlist', str(VARLIST_DIFF), + '--table_config', str(CMIP6_TABLE_CONFIG), + '--exp_config', str(EXP_CONFIG), + '--outdir', outdir, + '--calendar', 'julian', + '--grid_label', 'gr', + '--grid_desc', 'FOO_BAR_PLACEHOLD', + '--nom_res', '10000 km' ] ) assert result.exit_code == 0, f'case2 failed: {result.output}' output_ncs = list(Path(outdir).rglob('sos_Omon_*.nc')) @@ -212,20 +213,20 @@ def test_cli_fremor_run_case2(cli_sosv2_nc_file, tmp_path): # pylint: disable=re def test_cli_fremor_run_cmip7_case1(cli_sos_nc_file, tmp_path): # pylint: disable=redefined-outer-name - '''fremor run, test-use case for cmip7: sos → sos''' + """fremor run, test-use case for cmip7: sos → sos""" outdir = str(tmp_path / 'outdir') - result = runner.invoke(fremor, args = [ "-v", "-v", - "run", "--run_one", - "--indir", str(INDIR), - "--varlist", str(VARLIST), - "--table_config", str(CMIP7_TABLE_CONFIG), - "--exp_config", str(EXP_CONFIG_CMIP7), - "--outdir", outdir, - "--calendar", "julian", - "--grid_label", "g999", - "--grid_desc", "FOO_BAR_PLACEHOLD", - "--nom_res", "10000 km" ] ) + result = runner.invoke(fremor, args = [ '-v', '-v', + 'run', '--run_one', + '--indir', str(INDIR), + '--varlist', str(VARLIST), + '--table_config', str(CMIP7_TABLE_CONFIG), + '--exp_config', str(EXP_CONFIG_CMIP7), + '--outdir', outdir, + '--calendar', 'julian', + '--grid_label', 'g999', + '--grid_desc', 'FOO_BAR_PLACEHOLD', + '--nom_res', '10000 km' ] ) assert result.exit_code == 0, f'cmip7 case1 failed: {result.output}' output_ncs = list(Path(outdir).rglob('sos_*.nc')) @@ -234,20 +235,20 @@ def test_cli_fremor_run_cmip7_case1(cli_sos_nc_file, tmp_path): # pylint: disabl def test_cli_fremor_run_cmip7_case2(cli_sosv2_nc_file, tmp_path): # pylint: disable=redefined-outer-name - '''fremor run, test-use case 2 for cmip7: sosV2 varlist_diff''' + """fremor run, test-use case 2 for cmip7: sosV2 varlist_diff""" outdir = str(tmp_path / 'outdir') - result = runner.invoke(fremor, args = [ "-v", "-v", - "run", "--run_one", - "--indir", str(INDIR), - "--varlist", str(VARLIST_DIFF), - "--table_config", str(CMIP7_TABLE_CONFIG), - "--exp_config", str(EXP_CONFIG_CMIP7), - "--outdir", outdir, - "--calendar", "julian", - "--grid_label", "g999", - "--grid_desc", "FOO_BAR_PLACEHOLD", - "--nom_res", "10000 km" ] ) + result = runner.invoke(fremor, args = [ '-v', '-v', + 'run', '--run_one', + '--indir', str(INDIR), + '--varlist', str(VARLIST_DIFF), + '--table_config', str(CMIP7_TABLE_CONFIG), + '--exp_config', str(EXP_CONFIG_CMIP7), + '--outdir', outdir, + '--calendar', 'julian', + '--grid_label', 'g999', + '--grid_desc', 'FOO_BAR_PLACEHOLD', + '--nom_res', '10000 km' ] ) assert result.exit_code == 0, f'cmip7 case2 failed: {result.output}' output_ncs = list(Path(outdir).rglob('sos_*.nc')) @@ -258,59 +259,59 @@ def test_cli_fremor_run_cmip7_case2(cli_sosv2_nc_file, tmp_path): # pylint: disa # ── fremor find ─────────────────────────────────────────────────────────── def test_cli_fremor_find(): - ''' fremor find (no args) ''' - result = runner.invoke(fremor, args=["find"]) + """ fremor find (no args) """ + result = runner.invoke(fremor, args=['find']) assert result.exit_code == 2 def test_cli_fremor_find_help(): - ''' fremor find --help ''' - result = runner.invoke(fremor, args=["find", "--help"]) + """ fremor find --help """ + result = runner.invoke(fremor, args=['find', '--help']) assert result.exit_code == 0 def test_cli_fremor_find_opt_dne(): - ''' fremor find optionDNE ''' - result = runner.invoke(fremor, args=["find", "optionDNE"]) + """ fremor find optionDNE """ + result = runner.invoke(fremor, args=['find', 'optionDNE']) assert result.exit_code == 2 def test_cli_fremor_find_cmip6_case1(): - ''' fremor find, test-use case searching for variables in cmip6 tables ''' - result = runner.invoke(fremor, args=["-v", "find", - "--varlist", str(VARLIST), - "--table_config_dir", str(CMIP6_TABLE_CONFIG.parent)] ) + """ fremor find, test-use case searching for variables in cmip6 tables """ + result = runner.invoke(fremor, args=['-v', 'find', + '--varlist', str(VARLIST), + '--table_config_dir', str(CMIP6_TABLE_CONFIG.parent)] ) assert result.exit_code == 0 def test_cli_fremor_find_cmip6_case2(): - ''' fremor find, test-use case searching for variables in cmip6 tables ''' - result = runner.invoke(fremor, args=["-v", "find", - "--opt_var_name", "sos", - "--table_config_dir", str(CMIP6_TABLE_CONFIG.parent)] ) + """ fremor find, test-use case searching for variables in cmip6 tables """ + result = runner.invoke(fremor, args=['-v', 'find', + '--opt_var_name', 'sos', + '--table_config_dir', str(CMIP6_TABLE_CONFIG.parent)] ) assert result.exit_code == 0 # ── fremor config ───────────────────────────────────────────────────────── def test_cli_fremor_config(): - ''' fremor config (no args) ''' - result = runner.invoke(fremor, args=["config"]) + """ fremor config (no args) """ + result = runner.invoke(fremor, args=['config']) assert result.exit_code == 2 def test_cli_fremor_config_help(): - ''' fremor config --help ''' - result = runner.invoke(fremor, args=["config", "--help"]) + """ fremor config --help """ + result = runner.invoke(fremor, args=['config', '--help']) assert result.exit_code == 0 def test_cli_fremor_config_opt_dne(): - ''' fremor config optionDNE ''' - result = runner.invoke(fremor, args=["config", "optionDNE"]) + """ fremor config optionDNE """ + result = runner.invoke(fremor, args=['config', 'optionDNE']) assert result.exit_code == 2 def test_cli_fremor_config_case1(cli_sos_nc_file): # pylint: disable=redefined-outer-name - ''' + """ fremor config -- generate a CMOR YAML config from a mock pp directory tree. Uses the ocean_sos_var_file test data with a mock pp layout. - ''' + """ # set up a mock pp directory tree that the writer can scan mock_pp_dir = ROOTDIR / 'mock_pp_writer' comp_ts_dir = mock_pp_dir / 'ocean' / 'ts' / 'monthly' / '5yr' @@ -335,7 +336,7 @@ def test_cli_fremor_config_case1(cli_sos_nc_file): # pylint: disable=redefined-o (mock_pp_dir / 'ocean' / 'ts' / 'annual').mkdir(parents=True, exist_ok=True) # symlink the test nc file into the mock tree - src_nc = INDIR / 'reduced_ocean_monthly_1x1deg.199301-199302.sos.nc' + src_nc = Path(cli_sos_nc_file) dst_nc = comp_ts_dir / src_nc.name if dst_nc.exists() or dst_nc.is_symlink(): dst_nc.unlink() @@ -361,26 +362,26 @@ def test_cli_fremor_config_case1(cli_sos_nc_file): # pylint: disable=redefined-o output_yaml.touch() result = runner.invoke(fremor, args=[ - "-v", "-v", - "config", - "--pp_dir", str(mock_pp_dir), - "--mip_tables_dir", str(CMIP6_TABLE_CONFIG.parent), - "--mip_era", "cmip6", - "--exp_config", str(EXP_CONFIG), - "--output_yaml", str(output_yaml), - "--output_dir", str(output_data_dir), - "--varlist_dir", str(varlist_out_dir), - "--freq", "monthly", - "--chunk", "5yr", - "--grid", "gn", - "--overwrite" + '-v', '-v', + 'config', + '--pp_dir', str(mock_pp_dir), + '--mip_tables_dir', str(CMIP6_TABLE_CONFIG.parent), + '--mip_era', 'cmip6', + '--exp_config', str(EXP_CONFIG), + '--output_yaml', str(output_yaml), + '--output_dir', str(output_data_dir), + '--varlist_dir', str(varlist_out_dir), + '--freq', 'monthly', + '--chunk', '5yr', + '--grid', 'gn', + '--overwrite' ]) assert result.exit_code == 0, f'config failed: {result.output}' assert output_yaml.exists(), 'output YAML was not created' assert (varlist_out_dir / 'CMIP6_CMIP6_Omon_ocean.list').exists(), \ 'CMIP6_CMIP6_Omon_ocean.list was not created for some reason' - # basic sanity: the written file should contain "cmor:" and "table_targets:" + # basic sanity: the written file should contain 'cmor:' and 'table_targets:' yaml_text = output_yaml.read_text(encoding='utf-8') assert 'cmor:' in yaml_text assert 'table_targets:' in yaml_text @@ -397,32 +398,33 @@ def test_cli_fremor_config_case1(cli_sos_nc_file): # pylint: disable=redefined-o # ── fremor varlist ──────────────────────────────────────────────────────── def test_cli_fremor_varlist(): - ''' fremor varlist (no args) ''' - result = runner.invoke(fremor, args=["varlist"]) + """ fremor varlist (no args) """ + result = runner.invoke(fremor, args=['varlist']) assert result.exit_code == 2 def test_cli_fremor_varlist_help(): - ''' fremor varlist --help ''' - result = runner.invoke(fremor, args=["varlist", "--help"]) + """ fremor varlist --help """ + result = runner.invoke(fremor, args=['varlist', '--help']) assert result.exit_code == 0 def test_cli_fremor_varlist_opt_dne(): - ''' fremor varlist optionDNE ''' - result = runner.invoke(fremor, args=["varlist", "optionDNE"]) + """ fremor varlist optionDNE """ + result = runner.invoke(fremor, args=['varlist', 'optionDNE']) assert result.exit_code == 2 -def test_cli_fremor_varlist_no_table_filter(cli_sos_nc_file, cli_sosv2_nc_file, tmp_path): # pylint: disable=redefined-outer-name - '''fremor varlist — no MIP table filter. +def test_cli_fremor_varlist_no_table_filter(tmp_path, cli_sos_nc_file, cli_sosv2_nc_file): # pylint: disable=redefined-outer-name + """fremor varlist — no MIP table filter. Creates a variable list from the ocean_sos_var_file test data without a MIP table, - so both sos and sosV2 should appear.''' + so both sos and sosV2 should appear.""" output_varlist = tmp_path / 'test_varlist_no_filter.json' + assert Path(cli_sos_nc_file).parent == Path(cli_sosv2_nc_file).parent, 'something wrong with input nc files' result = runner.invoke(fremor, args=[ - "-v", "-v", - "varlist", - "--dir_targ", str(INDIR), - "--output_variable_list", str(output_varlist) + '-v', '-v', + 'varlist', + '--dir_targ', str(Path(cli_sos_nc_file).parent), + '--output_variable_list', str(output_varlist) ]) assert result.exit_code == 0, f'varlist failed: {result.output}' assert output_varlist.exists(), 'output variable list was not created' @@ -436,16 +438,17 @@ def test_cli_fremor_varlist_no_table_filter(cli_sos_nc_file, cli_sosv2_nc_file, def test_cli_fremor_varlist_cmip6_table_filter(cli_sos_nc_file, cli_sosv2_nc_file, tmp_path): # pylint: disable=redefined-outer-name - '''fremor varlist — with CMIP6 Omon MIP table filter. - Only sos should survive; sosV2 is not in the CMIP6 Omon table.''' + """fremor varlist — with CMIP6 Omon MIP table filter. + Only sos should survive; sosV2 is not in the CMIP6 Omon table.""" output_varlist = tmp_path / 'test_varlist_cmip6_filter.json' + assert Path(cli_sos_nc_file).parent == Path(cli_sosv2_nc_file).parent, 'something wrong with input nc files' result = runner.invoke(fremor, args=[ - "-v", "-v", - "varlist", - "--dir_targ", str(INDIR), - "--output_variable_list", str(output_varlist), - "--mip_table", str(CMIP6_TABLE_CONFIG) + '-v', '-v', + 'varlist', + '--dir_targ', str(Path(cli_sos_nc_file).parent), + '--output_variable_list', str(output_varlist), + '--mip_table', str(CMIP6_TABLE_CONFIG) ]) assert result.exit_code == 0, f'varlist failed: {result.output}' assert output_varlist.exists(), 'output variable list was not created' @@ -458,16 +461,17 @@ def test_cli_fremor_varlist_cmip6_table_filter(cli_sos_nc_file, cli_sosv2_nc_fil def test_cli_fremor_varlist_cmip7_table_filter(cli_sos_nc_file, cli_sosv2_nc_file, tmp_path): # pylint: disable=redefined-outer-name - '''fremor varlist — with CMIP7 ocean MIP table filter. - sos should survive (sos_tavg-u-hxy-sea splits to sos); sosV2 should not.''' + """fremor varlist — with CMIP7 ocean MIP table filter. + sos should survive (sos_tavg-u-hxy-sea splits to sos); sosV2 should not.""" output_varlist = tmp_path / 'test_varlist_cmip7_filter.json' + assert Path(cli_sos_nc_file).parent == Path(cli_sosv2_nc_file).parent, 'something wrong with input nc files' result = runner.invoke(fremor, args=[ - "-v", "-v", - "varlist", - "--dir_targ", str(INDIR), - "--output_variable_list", str(output_varlist), - "--mip_table", str(CMIP7_TABLE_CONFIG) + '-v', '-v', + 'varlist', + '--dir_targ', str(Path(cli_sos_nc_file).parent), + '--output_variable_list', str(output_varlist), + '--mip_table', str(CMIP7_TABLE_CONFIG) ]) assert result.exit_code == 0, f'varlist failed: {result.output}' assert output_varlist.exists(), 'output variable list was not created' @@ -482,31 +486,31 @@ def test_cli_fremor_varlist_cmip7_table_filter(cli_sos_nc_file, cli_sosv2_nc_fil # ── fremor init ─────────────────────────────────────────────────────────── def test_cli_fremor_init(): - ''' fremor init (no args) ''' - result = runner.invoke(fremor, args=["init"]) + """ fremor init (no args) """ + result = runner.invoke(fremor, args=['init']) assert result.exit_code == 2 def test_cli_fremor_init_help(): - ''' fremor init --help ''' - result = runner.invoke(fremor, args=["init", "--help"]) + """ fremor init --help """ + result = runner.invoke(fremor, args=['init', '--help']) assert result.exit_code == 0 def test_cli_fremor_init_opt_dne(): - ''' fremor init optionDNE ''' - result = runner.invoke(fremor, args=["init", "optionDNE"]) + """ fremor init optionDNE """ + result = runner.invoke(fremor, args=['init', 'optionDNE']) assert result.exit_code == 2 def test_cli_fremor_init_cmip6_exp_config(tmp_path): - ''' + """ fremor init -- generate a CMIP6 experiment config template. - ''' + """ output_path = tmp_path / 'test_cmip6_init_template.json' result = runner.invoke(fremor, args=[ - "init", - "--mip_era", "cmip6", - "--exp_config", str(output_path) + 'init', + '--mip_era', 'cmip6', + '--exp_config', str(output_path) ]) assert result.exit_code == 0, f'init failed: {result.output}' assert output_path.exists(), 'output config was not created' @@ -521,15 +525,15 @@ def test_cli_fremor_init_cmip6_exp_config(tmp_path): def test_cli_fremor_init_cmip7_exp_config(tmp_path): - ''' + """ fremor init -- generate a CMIP7 experiment config template. - ''' + """ output_path = tmp_path / 'test_cmip7_init_template.json' result = runner.invoke(fremor, args=[ - "init", - "--mip_era", "cmip7", - "--exp_config", str(output_path) + 'init', + '--mip_era', 'cmip7', + '--exp_config', str(output_path) ]) assert result.exit_code == 0, f'init failed: {result.output}' assert output_path.exists(), 'output config was not created' @@ -544,15 +548,15 @@ def test_cli_fremor_init_cmip7_exp_config(tmp_path): def test_cli_fremor_init_default_name(tmp_path): - ''' + """ fremor init -- when no --exp_config is given and no --tables_dir, a default-named file should be created in the current directory. - ''' + """ # Use CliRunner's isolated_filesystem to avoid polluting the actual working directory with runner.isolated_filesystem(temp_dir=tmp_path): result = runner.invoke(fremor, args=[ - "init", - "--mip_era", "cmip6" + 'init', + '--mip_era', 'cmip6' ]) assert result.exit_code == 0, f'init failed: {result.output}' @@ -567,20 +571,20 @@ def test_cli_fremor_init_default_name(tmp_path): # ── fremor run: logfile + omission tracking ─────────────────────────────── def test_cli_fremor_run_with_logfile(cli_sos_nc_file, tmp_path): # pylint: disable=redefined-outer-name - ''' + """ fremor -vv -l LOGFILE run ... Runs a real CMOR workflow with the -l flag and verifies that the resulting log file contains log lines from both cli.py (the CLI entry point) and cmor_mixer (the CMOR processing module). - ''' + """ log_path = tmp_path / 'TEST_CMOR_RUN.log' outdir = str(tmp_path / 'outdir') result = runner.invoke(fremor, args=[ '-vv', '-l', str(log_path), 'run', '--run_one', - '--indir', str(INDIR), + '--indir', str(Path(cli_sos_nc_file).parent), '--varlist', str(VARLIST), '--table_config', str(CMIP6_TABLE_CONFIG), '--exp_config', str(EXP_CONFIG), @@ -606,13 +610,13 @@ def test_cli_fremor_run_with_logfile(cli_sos_nc_file, tmp_path): # pylint: disab def test_cli_fremor_run_with_logfile_omission_case(cli_sos_nc_file, cli_sosv2_nc_file, tmp_path): # pylint: disable=redefined-outer-name - ''' + """ fremor -vv -l LOGFILE run ... Uses a varlist where sos->sos succeeds and sosV2->tob fails (tob is a valid CMIP6_Omon variable but the sosV2 file contains sos data, not tob). Verifies the OMISSION LOG appears in the log file with the failed variable info. - ''' + """ log_path = tmp_path / 'TEST_CMOR_RUN_OMISSION.log' outdir = str(tmp_path / 'outdir') @@ -621,11 +625,12 @@ def test_cli_fremor_run_with_logfile_omission_case(cli_sos_nc_file, cli_sosv2_nc varlist_fd, varlist_path = tempfile.mkstemp(suffix='.json') with os.fdopen(varlist_fd, 'w') as f: json.dump(varlist_data, f) + assert Path(cli_sos_nc_file).parent == Path(cli_sosv2_nc_file).parent, 'something wrong with input nc files' result = runner.invoke(fremor, args=[ '-vv', '-l', str(log_path), 'run', - '--indir', str(INDIR), + '--indir', str(Path(cli_sos_nc_file).parent), '--varlist', varlist_path, '--table_config', str(CMIP6_TABLE_CONFIG), '--exp_config', str(EXP_CONFIG), diff --git a/fremorizer/tests/test_cmor_find_subtool.py b/fremorizer/tests/test_cmor_find_subtool.py index 8ffc7c4..018e75a 100644 --- a/fremorizer/tests/test_cmor_find_subtool.py +++ b/fremorizer/tests/test_cmor_find_subtool.py @@ -1,35 +1,27 @@ -''' +""" tests for fremorizer.cmor_finder.cmor_find_subtool, mostly -''' +""" import json -import tempfile from pathlib import Path import pytest from fremorizer.cmor_finder import make_simple_varlist, cmor_find_subtool -@pytest.fixture -def temp_dir(): - ''' simple pytest fixture for providing temp directory ''' - with tempfile.TemporaryDirectory() as tmpdir: - yield tmpdir - - -def test_make_simple_varlist(temp_dir): - ''' +def test_make_simple_varlist(tmp_path): + """ quick tests of make_simple_varlist - ''' + """ # Create some dummy netCDF files - nc_files = ["test.20230101.var1.nc", "test.20230101.var2.nc", "test.20230101.var3.nc"] - assert Path(temp_dir).exists() + nc_files = ['test.20230101.var1.nc', 'test.20230101.var2.nc', 'test.20230101.var3.nc'] + assert Path(tmp_path).exists() for nc_file in nc_files: - Path(temp_dir, nc_file).touch() + Path(tmp_path, nc_file).touch() - output_file = Path(temp_dir, "varlist.json") - make_simple_varlist(temp_dir, output_file) + output_file = Path(tmp_path, 'varlist.json') + make_simple_varlist(tmp_path, output_file) # Check if the output file is created assert output_file.exists() @@ -39,16 +31,18 @@ def test_make_simple_varlist(temp_dir): var_list = json.load(f) expected_var_list = { - "var1": "var1", - "var2": "var2", - "var3": "var3" + 'var1': 'var1', + 'var2': 'var2', + 'var3': 'var3' } assert var_list == expected_var_list -def test_find_subtool_no_json_dir_err(temp_dir): - ''' test json_table_config_dir does not exist error ''' - target_dir_dne = Path(temp_dir) / 'foo' +def test_find_subtool_no_json_dir_err(tmp_path): + """ + test json_table_config_dir does not exist error + """ + target_dir_dne = Path(tmp_path) / 'foo' assert not target_dir_dne.exists(), 'target dir should not exist for this test' with pytest.raises(OSError, match=f'ERROR directory {target_dir_dne} does not exist! exit.'): cmor_find_subtool(json_var_list=None, @@ -56,19 +50,23 @@ def test_find_subtool_no_json_dir_err(temp_dir): opt_var_name=None) -def test_find_subtool_no_json_files_in_dir_err(temp_dir): - ''' test json_table_config_dir has no files in it error ''' - target_dir = Path(temp_dir) / 'foo' +def test_find_subtool_no_json_files_in_dir_err(tmp_path): + """ + test json_table_config_dir has no files in it error + """ + target_dir = Path(tmp_path) / 'foo' target_dir.mkdir(exist_ok=False) - assert target_dir.is_dir() and target_dir.exists(), "temp dir directory creation failed, inspect code" + assert target_dir.is_dir() and target_dir.exists(), 'temp dir directory creation failed, inspect code' with pytest.raises(OSError, match=f'ERROR directory {target_dir} contains no JSON files, exit.'): cmor_find_subtool(json_var_list=None, json_table_config_dir=str(target_dir), opt_var_name=None) -def test_find_subtool_no_varlist_no_optvarname_err(temp_dir): - ''' test no opt_var_name AND no varlist error ''' +def test_find_subtool_no_varlist_no_optvarname_err(): + """ + test no opt_var_name AND no varlist error + """ with pytest.raises(ValueError, match='ERROR: no opt_var_name given but also no content in variable list!!! exit!'): cmor_find_subtool(json_var_list=None, json_table_config_dir='fremorizer/tests/test_files/cmip6-cmor-tables/Tables', diff --git a/fremorizer/tests/test_cmor_finder_make_simple_varlist.py b/fremorizer/tests/test_cmor_finder_make_simple_varlist.py index b8a4d90..456e2e0 100644 --- a/fremorizer/tests/test_cmor_finder_make_simple_varlist.py +++ b/fremorizer/tests/test_cmor_finder_make_simple_varlist.py @@ -1,6 +1,6 @@ -''' +""" tests for fremorizer.cmor_finder.make_simple_varlist -''' +""" import json from unittest.mock import patch @@ -10,16 +10,16 @@ from fremorizer.cmor_finder import make_simple_varlist -@pytest.fixture -def temp_netcdf_dir(tmp_path): +@pytest.fixture(name='netcdf_dir_files') +def temp_netcdf_dir_files(tmp_path): """ Fixture to create a temporary directory with sample NetCDF files. """ # Create sample NetCDF filenames following the expected pattern netcdf_files = [ - "model.19900101.temp.nc", - "model.19900101.salt.nc", - "model.19900101.velocity.nc" + 'model.19900101.temp.nc', + 'model.19900101.salt.nc', + 'model.19900101.velocity.nc' ] for filename in netcdf_files: @@ -29,104 +29,99 @@ def temp_netcdf_dir(tmp_path): return tmp_path -@pytest.fixture -def temp_netcdf_dir_single_file(tmp_path): +@pytest.fixture(name='netcdf_dir_file') +def temp_netcdf_dir_file(tmp_path): """ Fixture to create a temporary directory with a single NetCDF file. """ - file_path = tmp_path / "model.19900101.temp.nc" + file_path = tmp_path / 'model.19900101.temp.nc' file_path.touch() return tmp_path -@pytest.fixture -def empty_dir(tmp_path): - """ - Fixture to create an empty temporary directory. - """ - return tmp_path -def test_make_simple_varlist_success(temp_netcdf_dir, tmp_path): + +def test_make_simple_varlist_success(netcdf_dir_files): """ Test successful creation of variable list from NetCDF files. """ # Arrange - output_file = tmp_path / "varlist.json" + output_file = netcdf_dir_files / 'varlist.json' # Act - result = make_simple_varlist(str(temp_netcdf_dir), str(output_file)) + result = make_simple_varlist(str(netcdf_dir_files), str(output_file)) # Assert assert result is not None assert isinstance(result, dict) - assert "temp" in result - assert "salt" in result - assert "velocity" in result - assert result["temp"] == "temp" - assert result["salt"] == "salt" - assert result["velocity"] == "velocity" + assert 'temp' in result + assert 'salt' in result + assert 'velocity' in result + assert result['temp'] == 'temp' + assert result['salt'] == 'salt' + assert result['velocity'] == 'velocity' # Verify output file was created assert output_file.exists() - with open(output_file, "r", encoding='utf-8') as f: + with open(output_file, 'r', encoding='utf-8') as f: saved_data = json.load(f) assert saved_data == result -def test_make_simple_varlist_return_value_only(temp_netcdf_dir): +def test_make_simple_varlist_return_value_only(netcdf_dir_files): """ Test make_simple_varlist with output_variable_list=None returns var_list. """ # Act - result = make_simple_varlist(str(temp_netcdf_dir), None) + result = make_simple_varlist(str(netcdf_dir_files), None) # Assert assert result is not None assert isinstance(result, dict) - assert "temp" in result - assert "salt" in result - assert "velocity" in result + assert 'temp' in result + assert 'salt' in result + assert 'velocity' in result -def test_make_simple_varlist_single_file_warning(temp_netcdf_dir_single_file, tmp_path): +def test_make_simple_varlist_single_file_warning(netcdf_dir_file): """ Test warning when only one file is found. """ # Arrange - output_file = tmp_path / "varlist.json" + output_file = netcdf_dir_file / 'varlist.json' # Act - result = make_simple_varlist(str(temp_netcdf_dir_single_file), str(output_file)) + result = make_simple_varlist(str(netcdf_dir_file), str(output_file)) # Assert assert result is not None assert isinstance(result, dict) - assert "temp" in result - assert result["temp"] == "temp" + assert 'temp' in result + assert result['temp'] == 'temp' -def test_make_simple_varlist_no_files(empty_dir): +def test_make_simple_varlist_no_files(tmp_path): """ Test behavior when no NetCDF files are found in directory. """ # Act - result = make_simple_varlist(str(empty_dir), None) + result = make_simple_varlist(str(tmp_path), None) # Assert - function should return None when no files found assert result is None -def test_make_simple_varlist_invalid_output_path(temp_netcdf_dir): +def test_make_simple_varlist_invalid_output_path(netcdf_dir_file): """ Test OSError when output file cannot be written. """ # Arrange - try to write to a directory that doesn't exist - invalid_output_path = "/nonexistent_directory/varlist.json" + invalid_output_path = '/nonexistent_directory/varlist.json' # Act & Assert - with pytest.raises(OSError, match="output variable list created but cannot be written"): - make_simple_varlist(str(temp_netcdf_dir), invalid_output_path) + with pytest.raises(OSError, match='output variable list created but cannot be written'): + make_simple_varlist(str(netcdf_dir_file), invalid_output_path) def test_make_simple_varlist_no_matching_pattern(tmp_path): @@ -134,8 +129,8 @@ def test_make_simple_varlist_no_matching_pattern(tmp_path): Test behavior when files exist but don't match the expected pattern. """ # Arrange - create files that don't follow the expected pattern - (tmp_path / "random_file.txt").touch() - (tmp_path / "another_file.nc").touch() # Missing datetime pattern + (tmp_path / 'random_file.txt').touch() + (tmp_path / 'another_file.nc').touch() # Missing datetime pattern # Act result = make_simple_varlist(str(tmp_path), None) @@ -151,15 +146,15 @@ def test_make_simple_varlist_deduplicates(tmp_path): the result should contain the variable only once. Variables that only appear at a single datetime are still included. """ - # Two files with var_name "temp" and one with "salt" - (tmp_path / "model.19900101.temp.nc").touch() - (tmp_path / "model.19900201.temp.nc").touch() # duplicate var_name - (tmp_path / "model.19900101.salt.nc").touch() + # Two files with var_name 'temp' and one with 'salt' + (tmp_path / 'model.19900101.temp.nc').touch() + (tmp_path / 'model.19900201.temp.nc').touch() # duplicate var_name + (tmp_path / 'model.19900101.salt.nc').touch() result = make_simple_varlist(str(tmp_path), None) assert result is not None - assert result == {"temp": "temp", "salt": "salt"} + assert result == {'temp': 'temp', 'salt': 'salt'} # ---- mip-table filtering coverage ---- @@ -169,22 +164,22 @@ def test_make_simple_varlist_mip_table_filter(tmp_path): should appear in the result. """ # create data files - (tmp_path / "model.19900101.sos.nc").touch() - (tmp_path / "model.19900101.notinmip.nc").touch() + (tmp_path / 'model.19900101.sos.nc').touch() + (tmp_path / 'model.19900101.notinmip.nc').touch() - # create a minimal MIP table with only "sos" - mip_table = tmp_path / "Omon.json" + # create a minimal MIP table with only 'sos' + mip_table = tmp_path / 'Omon.json' mip_table.write_text(json.dumps({ - "variable_entry": { - "sos": {"frequency": "mon"} + 'variable_entry': { + 'sos': {'frequency': 'mon'} } })) result = make_simple_varlist(str(tmp_path), None, json_mip_table=str(mip_table)) assert result is not None - assert "sos" in result - assert "notinmip" not in result + assert 'sos' in result + assert 'notinmip' not in result # ---- no files matching search pattern ---- @@ -195,7 +190,7 @@ def test_make_simple_varlist_no_files_matching_pattern(tmp_path): glob.glob to return an empty list. """ # Create a file for the probe to land on (but the patch overrides it) - probe_file = tmp_path / "model.19900101.temp.nc" + probe_file = tmp_path / 'model.19900101.temp.nc' probe_file.touch() # Patch glob.glob to return empty list @@ -212,12 +207,12 @@ def test_make_simple_varlist_single_file_hits_warning(tmp_path): log a warning and still return the variable. Covers the 'elif len(files) == 1' branch. """ - (tmp_path / "model.19900101.salinity.nc").touch() + (tmp_path / 'model.19900101.salinity.nc').touch() result = make_simple_varlist(str(tmp_path), None) assert result is not None - assert result == {"salinity": "salinity"} + assert result == {'salinity': 'salinity'} # ---- duplicate var_name skip with datetime grouping ---- @@ -228,17 +223,17 @@ def test_make_simple_varlist_dedup_across_datetimes(tmp_path): All files across all datetimes are scanned; duplicate var names are collapsed by dict assignment. """ - (tmp_path / "ocean.19900101.tos.nc").touch() - (tmp_path / "ocean.19900201.tos.nc").touch() - (tmp_path / "ocean.19900101.sos.nc").touch() - (tmp_path / "ocean.19900201.sos.nc").touch() + (tmp_path / 'ocean.19900101.tos.nc').touch() + (tmp_path / 'ocean.19900201.tos.nc').touch() + (tmp_path / 'ocean.19900101.sos.nc').touch() + (tmp_path / 'ocean.19900201.sos.nc').touch() # All four files are scanned; tos and sos each appear twice but # dict assignment deduplicates them. result = make_simple_varlist(str(tmp_path), None) assert result is not None - assert result == {"tos": "tos", "sos": "sos"} + assert result == {'tos': 'tos', 'sos': 'sos'} # ---- mip table filtering: no variables match ---- @@ -248,12 +243,12 @@ def test_make_simple_varlist_mip_table_no_match(tmp_path): the result should be an empty dict (quick_vlist stays empty → 'no variables in target mip table found' warning, var_list stays {}). """ - (tmp_path / "model.19900101.fake_var.nc").touch() + (tmp_path / 'model.19900101.fake_var.nc').touch() - mip_table = tmp_path / "table.json" + mip_table = tmp_path / 'table.json' mip_table.write_text(json.dumps({ - "variable_entry": { - "sos": {"frequency": "mon"} + 'variable_entry': { + 'sos': {'frequency': 'mon'} } })) @@ -270,13 +265,13 @@ def test_make_simple_varlist_minority_datetime_var_included(tmp_path): even when most files belong to a different datetime. Scanning all files (not just those at the most-common datetime) guarantees this. """ - # "temp" appears four times at 19900101; "salt" appears only once at 19900201. + # 'temp' appears four times at 19900101; 'salt' appears only once at 19900201. for i in range(1, 5): - (tmp_path / f"model.1990010{i}.temp.nc").touch() - (tmp_path / "model.19900201.salt.nc").touch() + (tmp_path / f'model.1990010{i}.temp.nc').touch() + (tmp_path / 'model.19900201.salt.nc').touch() result = make_simple_varlist(str(tmp_path), None) assert result is not None - assert "temp" in result - assert "salt" in result # must not be silently dropped + assert 'temp' in result + assert 'salt' in result # must not be silently dropped diff --git a/fremorizer/tests/test_cmor_helpers.py b/fremorizer/tests/test_cmor_helpers.py index b2c0a5a..2dd231c 100644 --- a/fremorizer/tests/test_cmor_helpers.py +++ b/fremorizer/tests/test_cmor_helpers.py @@ -1,6 +1,6 @@ -''' +""" tests for fremorizer helper functions in cmor_helpers -''' +""" import json from pathlib import Path @@ -16,13 +16,13 @@ filter_brands ) def test_iso_to_bronx_chunk(): - ''' tests value error raising by iso_to_bronx_chunk ''' + """ tests value error raising by iso_to_bronx_chunk """ with pytest.raises(ValueError, match='problem with converting to bronx chunk from the cmor chunk. check cmor_yamler.py'): iso_to_bronx_chunk('foo') def test_find_statics_file_success(tmp_path): - ''' what happens when no statics file is found given a bronx directory structure ''' + """ what happens when no statics file is found given a bronx directory structure """ mock_root = tmp_path / 'mock_archive' bronx_subpath = ('USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/' 'gfdl.ncrc5-intel23-prod-openmp/pp/ocean_monthly') @@ -43,7 +43,7 @@ def test_find_statics_file_success(tmp_path): def test_find_statics_file_nothing_found(): - ''' what happens when a statics file is found given a bronx directory structure ''' + """ what happens when a statics file is found given a bronx directory structure """ statics_file = find_statics_file( bronx_file_path = 'fremorizer/tests/test_files/ascii_files/' + \ 'mock_archive/USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/' + \ @@ -52,32 +52,32 @@ def test_find_statics_file_nothing_found(): def test_print_data_minmax_no_exception_case1(): - ''' checks to make sure this doesn't raise an exception ''' + """ checks to make sure this doesn't raise an exception """ print_data_minmax(None, None) def test_print_data_minmax_no_exception_case2(): - ''' checks to make sure this doesn't raise an exception ''' + """ checks to make sure this doesn't raise an exception """ print_data_minmax(np.ma.core.MaskedArray( data=(0, 10, 20, 30) ), None) def test_print_data_minmax_no_exception_case3(): - ''' checks to make sure this doesn't raise an exception ''' + """ checks to make sure this doesn't raise an exception """ print_data_minmax(np.ma.core.MaskedArray( data=(0, 10, 20, 30) ), 'foo') # ---- find_gold_ocean_statics_file tests ---- def test_find_gold_ocean_statics_file_none_arg(): - ''' put_copy_here=None should return None immediately ''' + """ put_copy_here=None should return None immediately """ result = find_gold_ocean_statics_file(put_copy_here=None) assert result is None def test_find_gold_ocean_statics_file_archive_missing(tmp_path): - ''' + """ when the archive gold file does not exist on disk (i.e. not at PPAN), the function should create the local directory tree but return None because there's nothing to copy. - ''' + """ from fremorizer.cmor_constants import ARCHIVE_GOLD_DATA_DIR, CMIP7_GOLD_OCEAN_FILE_STUB # pylint: disable=import-outside-toplevel gold_file = f'{ARCHIVE_GOLD_DATA_DIR}/{CMIP7_GOLD_OCEAN_FILE_STUB}' @@ -95,10 +95,10 @@ def test_find_gold_ocean_statics_file_archive_missing(tmp_path): def test_find_gold_ocean_statics_file_mock_copy(tmp_path): - ''' + """ exercise the full copy path by creating a fake archive gold file in tmp_path and monkeypatching ARCHIVE_GOLD_DATA_DIR so the function finds it. - ''' + """ import fremorizer.cmor_helpers as _helpers_mod # pylint: disable=import-outside-toplevel import fremorizer.cmor_constants as _const_mod # pylint: disable=import-outside-toplevel @@ -128,7 +128,7 @@ def test_find_gold_ocean_statics_file_mock_copy(tmp_path): # ---- create_lev_bnds failure case ---- def test_create_lev_bnds_length_mismatch(): - ''' create_lev_bnds should raise ValueError when len(with_these) != len(bound_these)+1 ''' + """ create_lev_bnds should raise ValueError when len(with_these) != len(bound_these)+1 """ bound_these = np.array([10.0, 20.0, 30.0]) with_these = np.array([5.0, 15.0]) # wrong: should be len 4 (=3+1) with pytest.raises(ValueError, match='failed creating bnds'): @@ -136,7 +136,7 @@ def test_create_lev_bnds_length_mismatch(): def test_create_lev_bnds_length_mismatch_too_long(): - ''' same check, but with_these is too long instead of too short ''' + """ same check, but with_these is too long instead of too short """ bound_these = np.array([10.0, 20.0]) with_these = np.array([5.0, 15.0, 25.0, 35.0]) # wrong: should be len 3 (=2+1) with pytest.raises(ValueError, match='failed creating bnds'): @@ -156,7 +156,7 @@ def test_create_lev_bnds_length_mismatch_too_long(): def test_get_iso_datetime_ranges_no_filter(): - ''' all 5 date ranges should appear when start/stop are None ''' + """ all 5 date ranges should appear when start/stop are None """ result = [] get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, iso_daterange_arr=result) @@ -166,7 +166,7 @@ def test_get_iso_datetime_ranges_no_filter(): def test_get_iso_datetime_ranges_with_stop(): - ''' only date ranges whose end-year <= 2004 should survive ''' + """ only date ranges whose end-year <= 2004 should survive """ result = [] get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, iso_daterange_arr=result, @@ -181,7 +181,7 @@ def test_get_iso_datetime_ranges_with_stop(): def test_get_iso_datetime_ranges_with_start(): - ''' only date ranges whose start-year >= 2000 should survive ''' + """ only date ranges whose start-year >= 2000 should survive """ result = [] get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, iso_daterange_arr=result, @@ -195,7 +195,7 @@ def test_get_iso_datetime_ranges_with_start(): def test_get_iso_datetime_ranges_with_start_and_stop(): - ''' start=1995 stop=2004 should give exactly two ranges ''' + """ start=1995 stop=2004 should give exactly two ranges """ result = [] get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, iso_daterange_arr=result, @@ -205,14 +205,14 @@ def test_get_iso_datetime_ranges_with_start_and_stop(): def test_get_iso_datetime_ranges_none_arr_raises(): - ''' passing iso_daterange_arr=None should raise ValueError ''' + """ passing iso_daterange_arr=None should raise ValueError """ with pytest.raises(ValueError, match='requires the list'): get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, iso_daterange_arr=None) def test_get_iso_datetime_ranges_no_matches_raises(): - ''' if the filter excludes everything, ValueError should be raised ''' + """ if the filter excludes everything, ValueError should be raised """ result = [] with pytest.raises(ValueError, match='length 0'): get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, @@ -224,23 +224,23 @@ def test_get_iso_datetime_ranges_no_matches_raises(): # ---- create_tmp_dir tests ---- def test_create_tmp_dir_success(tmp_path): - ''' create_tmp_dir should create a tmp/ subdirectory and return its path ''' + """ create_tmp_dir should create a tmp/ subdirectory and return its path """ result = create_tmp_dir(outdir=str(tmp_path)) assert Path(result).is_dir() assert result.endswith('/CMOR_tmp') def test_create_tmp_dir_with_exp_config(tmp_path): - ''' when json_exp_config has an outpath key, a subdirectory should also be created ''' + """ when json_exp_config has an outpath key, a subdirectory should also be created """ exp_config = tmp_path / 'exp_config.json' - exp_config.write_text(json.dumps({"outpath": "CMIP7/output"})) + exp_config.write_text(json.dumps({'outpath': 'CMIP7/output'})) result = create_tmp_dir(outdir=str(tmp_path), json_exp_config=str(exp_config)) assert Path(result).is_dir() assert Path(result, 'CMIP7/output').is_dir() -def test_create_tmp_dir_oserror(tmp_path): - ''' create_tmp_dir should raise OSError when directory creation fails ''' +def test_create_tmp_dir_oserror(): + """ create_tmp_dir should raise OSError when directory creation fails """ # /dev/null is not a directory; we can't mkdir inside it with pytest.raises(OSError, match='problem creating tmp output directory'): create_tmp_dir(outdir='/dev/null/impossible_path') @@ -249,21 +249,21 @@ def test_create_tmp_dir_oserror(tmp_path): # ---- get_json_file_data tests ---- def test_get_json_file_data_success(tmp_path): - ''' should load and return JSON content ''' + """ should load and return JSON content """ f = tmp_path / 'data.json' - payload = {"key": "value", "num": 42} + payload = {'key': 'value', 'num': 42} f.write_text(json.dumps(payload)) assert get_json_file_data(str(f)) == payload def test_get_json_file_data_nonexistent(): - ''' should raise FileNotFoundError for a missing file ''' + """ should raise FileNotFoundError for a missing file """ with pytest.raises(FileNotFoundError, match='cannot be opened'): get_json_file_data('/nonexistent/path/file.json') def test_get_json_file_data_invalid_json(tmp_path): - ''' should raise FileNotFoundError (wrapping JSONDecodeError) for invalid JSON ''' + """ should raise FileNotFoundError (wrapping JSONDecodeError) for invalid JSON """ f = tmp_path / 'bad.json' f.write_text('NOT JSON {{{{') with pytest.raises(FileNotFoundError, match='cannot be opened'): @@ -273,47 +273,47 @@ def test_get_json_file_data_invalid_json(tmp_path): # ---- update_grid_and_label None-args test ---- def test_update_grid_and_label_none_grid_label(tmp_path): - ''' should raise ValueError when new_grid_label is None ''' + """ should raise ValueError when new_grid_label is None """ f = tmp_path / 'exp.json' - f.write_text(json.dumps({"grid_label": "gr", "grid": "g", "nominal_resolution": "50 km"})) + f.write_text(json.dumps({'grid_label': 'gr', 'grid': 'g', 'nominal_resolution': '50 km'})) with pytest.raises(ValueError): - update_grid_and_label(str(f), None, "new_grid", "100 km") + update_grid_and_label(str(f), None, 'new_grid', '100 km') def test_update_grid_and_label_none_grid(tmp_path): - ''' should raise ValueError when new_grid is None ''' + """ should raise ValueError when new_grid is None """ f = tmp_path / 'exp.json' - f.write_text(json.dumps({"grid_label": "gr", "grid": "g", "nominal_resolution": "50 km"})) + f.write_text(json.dumps({'grid_label': 'gr', 'grid': 'g', 'nominal_resolution': '50 km'})) with pytest.raises(ValueError): - update_grid_and_label(str(f), "gn", None, "100 km") + update_grid_and_label(str(f), 'gn', None, '100 km') def test_update_grid_and_label_none_nom_res(tmp_path): - ''' should raise ValueError when new_nom_res is None ''' + """ should raise ValueError when new_nom_res is None """ f = tmp_path / 'exp.json' - f.write_text(json.dumps({"grid_label": "gr", "grid": "g", "nominal_resolution": "50 km"})) + f.write_text(json.dumps({'grid_label': 'gr', 'grid': 'g', 'nominal_resolution': '50 km'})) with pytest.raises(ValueError): - update_grid_and_label(str(f), "gn", "new_grid", None) + update_grid_and_label(str(f), 'gn', 'new_grid', None) # ---- get_bronx_freq_from_mip_table tests ---- def test_get_bronx_freq_from_mip_table_success(tmp_path): - ''' should return the bronx-equivalent frequency for a valid table ''' + """ should return the bronx-equivalent frequency for a valid table """ table = { - "variable_entry": { - "sos": {"frequency": "mon", "other": "stuff"} + 'variable_entry': { + 'sos': {'frequency': 'mon', 'other': 'stuff'} } } f = tmp_path / 'Omon.json' f.write_text(json.dumps(table)) - assert get_bronx_freq_from_mip_table(str(f)) == "monthly" + assert get_bronx_freq_from_mip_table(str(f)) == 'monthly' def test_get_bronx_freq_from_mip_table_no_freq(tmp_path): - ''' should raise bronx-equivalent frequency for a valid table ''' + """ should raise bronx-equivalent frequency for a valid table """ table = { - "variable_entry": { - "sos": {"other": "stuff"} + 'variable_entry': { + 'sos': {'other': 'stuff'} } } f = tmp_path / 'Omon.json' @@ -323,10 +323,10 @@ def test_get_bronx_freq_from_mip_table_no_freq(tmp_path): get_bronx_freq_from_mip_table(str(f)) def test_get_bronx_freq_from_mip_table_invalid_freq(tmp_path): - ''' should raise KeyError when the table frequency is not a valid MIP frequency ''' + """ should raise KeyError when the table frequency is not a valid MIP frequency """ table = { - "variable_entry": { - "sos": {"frequency": "bogus_freq"} + 'variable_entry': { + 'sos': {'frequency': 'bogus_freq'} } } f = tmp_path / 'Obogus.json' @@ -337,15 +337,15 @@ def test_get_bronx_freq_from_mip_table_invalid_freq(tmp_path): ## ---- update_outpath tests ---- # #def test_update_outpath_none_json_path(): -# ''' should raise ValueError when json_file_path is None ''' +# """ should raise ValueError when json_file_path is None """ # with pytest.raises(ValueError): # update_outpath(None, '/some/output/path') # # #def test_update_outpath_none_output_path(tmp_path): -# ''' should raise ValueError when output_file_path is None ''' +# """ should raise ValueError when output_file_path is None """ # f = tmp_path / 'exp.json' -# f.write_text(json.dumps({"outpath": "/old/path"})) +# f.write_text(json.dumps({'outpath': '/old/path'})) # with pytest.raises(ValueError): # update_outpath(str(f), None) @@ -353,68 +353,68 @@ def test_get_bronx_freq_from_mip_table_invalid_freq(tmp_path): # ---- filter_brands tests ---- def _make_mip_var_cfgs(var_brands_dims): - '''helper: build a minimal mip_var_cfgs dict from {mip_key: dims_string} pairs''' - return {"variable_entry": {k: {"dimensions": v} for k, v in var_brands_dims.items()}} + """helper: build a minimal mip_var_cfgs dict from {mip_key: dims_string} pairs""" + return {'variable_entry': {k: {'dimensions': v} for k, v in var_brands_dims.items()}} def test_filter_brands_time_filter_selects_mean(): - ''' brand with time (mean) should be selected when input has time_bnds ''' + """ brand with time (mean) should be selected when input has time_bnds """ mip = _make_mip_var_cfgs({ - "sos_mean-2d": "longitude latitude time", - "sos_inst-2d": "longitude latitude time1", + 'sos_mean-2d': 'longitude latitude time', + 'sos_inst-2d': 'longitude latitude time1', }) result = filter_brands( - brands=["mean-2d", "inst-2d"], - target_var="sos", + brands=['mean-2d', 'inst-2d'], + target_var='sos', mip_var_cfgs=mip, has_time_bnds=True, input_vert_dim=0, ) - assert result == "mean-2d" + assert result == 'mean-2d' def test_filter_brands_time_filter_selects_inst(): - ''' brand with time1 (instantaneous) should be selected when input lacks time_bnds ''' + """ brand with time1 (instantaneous) should be selected when input lacks time_bnds """ mip = _make_mip_var_cfgs({ - "sos_mean-2d": "longitude latitude time", - "sos_inst-2d": "longitude latitude time1", + 'sos_mean-2d': 'longitude latitude time', + 'sos_inst-2d': 'longitude latitude time1', }) result = filter_brands( - brands=["mean-2d", "inst-2d"], - target_var="sos", + brands=['mean-2d', 'inst-2d'], + target_var='sos', mip_var_cfgs=mip, has_time_bnds=False, input_vert_dim=0, ) - assert result == "inst-2d" + assert result == 'inst-2d' def test_filter_brands_vertical_filter(): - ''' vertical filter should select the brand whose MIP dims contain the expected vert dim ''' + """ vertical filter should select the brand whose MIP dims contain the expected vert dim """ mip = _make_mip_var_cfgs({ - "temp_mean-3d-native-sea": "longitude latitude olevel time", - "temp_mean-2d": "longitude latitude time", + 'temp_mean-3d-native-sea': 'longitude latitude olevel time', + 'temp_mean-2d': 'longitude latitude time', }) result = filter_brands( - brands=["mean-3d-native-sea", "mean-2d"], - target_var="temp", + brands=['mean-3d-native-sea', 'mean-2d'], + target_var='temp', mip_var_cfgs=mip, has_time_bnds=True, - input_vert_dim="z_l", + input_vert_dim='z_l', ) - assert result == "mean-3d-native-sea" + assert result == 'mean-3d-native-sea' def test_filter_brands_all_eliminated(): - ''' should raise ValueError when all brands are filtered out ''' + """ should raise ValueError when all brands are filtered out """ mip = _make_mip_var_cfgs({ - "sos_a": "longitude latitude time1", - "sos_b": "longitude latitude time1", + 'sos_a': 'longitude latitude time1', + 'sos_b': 'longitude latitude time1', }) with pytest.raises(ValueError, match='none survived'): filter_brands( - brands=["a", "b"], - target_var="sos", + brands=['a', 'b'], + target_var='sos', mip_var_cfgs=mip, has_time_bnds=True, input_vert_dim=0, @@ -422,15 +422,15 @@ def test_filter_brands_all_eliminated(): def test_filter_brands_multiple_remain(): - ''' should raise ValueError when multiple brands survive filtering ''' + """ should raise ValueError when multiple brands survive filtering """ mip = _make_mip_var_cfgs({ - "sos_a": "longitude latitude time", - "sos_b": "longitude latitude time", + 'sos_a': 'longitude latitude time', + 'sos_b': 'longitude latitude time', }) with pytest.raises(ValueError, match='remain for sos'): filter_brands( - brands=["a", "b"], - target_var="sos", + brands=['a', 'b'], + target_var='sos', mip_var_cfgs=mip, has_time_bnds=True, input_vert_dim=0, diff --git a/fremorizer/tests/test_cmor_helpers_update_calendar.py b/fremorizer/tests/test_cmor_helpers_update_calendar.py index ffd3566..6b8fc38 100644 --- a/fremorizer/tests/test_cmor_helpers_update_calendar.py +++ b/fremorizer/tests/test_cmor_helpers_update_calendar.py @@ -1,6 +1,6 @@ -''' +""" tests for fremorizer.cmor_helpers.update_calendar_type -''' +""" import json import pytest @@ -25,12 +25,12 @@ def temp_json_file(tmp_path): """ # Sample data for testing test_json_content = { - "calendar": "original_calendar_type", - "other_field": "some_value" + 'calendar': 'original_calendar_type', + 'other_field': 'some_value' } - json_file = tmp_path / "test_file.json" - with open(json_file, "w", encoding="utf-8") as file: + json_file = tmp_path / 'test_file.json' + with open(json_file, 'w', encoding='utf-8') as file: json.dump(test_json_content, file, indent=4) return json_file @@ -39,26 +39,26 @@ def test_update_calendar_type_success(temp_json_file): # pylint: disable=redefin Test successful update of 'grid_label' and 'grid' fields. """ # Arrange - new_calendar_type = "365_day" + new_calendar_type = '365_day' # Act update_calendar_type(temp_json_file, new_calendar_type) # Assert - with open(temp_json_file, "r", encoding="utf-8") as file: + with open(temp_json_file, 'r', encoding='utf-8') as file: data = json.load(file) - assert data["calendar"] == new_calendar_type - assert data["other_field"] == "some_value" + assert data['calendar'] == new_calendar_type + assert data['other_field'] == 'some_value' def test_update_calendar_type_alias_normalized(temp_json_file): # pylint: disable=redefined-outer-name """ Calendar aliases should be normalized when updating the experiment config. """ - update_calendar_type(temp_json_file, "noleap") + update_calendar_type(temp_json_file, 'noleap') - with open(temp_json_file, "r", encoding="utf-8") as file: + with open(temp_json_file, 'r', encoding='utf-8') as file: data = json.load(file) - assert data["calendar"] == "365_day" + assert data['calendar'] == '365_day' def test_update_calendar_type_valerr_raise(temp_json_file): # pylint: disable=redefined-outer-name """ @@ -88,12 +88,12 @@ def temp_keyerr_json_file(tmp_path): """ # Sample data for testing test_json_content = { - "clendar": "original_calendar_type", #oops spelling error # cspell:disable-line - "other_field": "some_value" + 'clendar': 'original_calendar_type', #oops spelling error # cspell:disable-line + 'other_field': 'some_value' } - json_file = tmp_path / "test_file.json" - with open(json_file, "w", encoding="utf-8") as file: + json_file = tmp_path / 'test_file.json' + with open(json_file, 'w', encoding='utf-8') as file: json.dump(test_json_content, file, indent=4) return json_file @@ -109,9 +109,9 @@ def temp_jsondecodeerr_json_file(tmp_path): """ Create a file with invalid JSON content """ - invalid_json_file = tmp_path / "invalid.json" + invalid_json_file = tmp_path / 'invalid.json' invalid_content = '{ "calendar": "original_calendar_type", "other_field": "some_value" ' # missing closing } - with open(invalid_json_file, "w", encoding="utf-8") as f: + with open(invalid_json_file, 'w', encoding='utf-8') as f: f.write(invalid_content) return invalid_json_file @@ -131,71 +131,71 @@ def test_update_calendar_type_json_dne_raise(): def test_normalize_calendar_noleap(): - ''' noleap is a CF alias for 365_day ''' - assert normalize_calendar("noleap") == "365_day" + """ noleap is a CF alias for 365_day """ + assert normalize_calendar('noleap') == '365_day' def test_normalize_calendar_365_day_passthrough(): - ''' 365_day is already canonical ''' - assert normalize_calendar("365_day") == "365_day" + """ 365_day is already canonical """ + assert normalize_calendar('365_day') == '365_day' def test_normalize_calendar_all_leap(): - ''' all_leap is a CF alias for 366_day ''' - assert normalize_calendar("all_leap") == "366_day" + """ all_leap is a CF alias for 366_day """ + assert normalize_calendar('all_leap') == '366_day' def test_normalize_calendar_366_day_passthrough(): - ''' 366_day is already canonical ''' - assert normalize_calendar("366_day") == "366_day" + """ 366_day is already canonical """ + assert normalize_calendar('366_day') == '366_day' def test_normalize_calendar_standard(): - ''' standard is a CF alias for gregorian ''' - assert normalize_calendar("standard") == "gregorian" + """ standard is a CF alias for gregorian """ + assert normalize_calendar('standard') == 'gregorian' def test_normalize_calendar_gregorian_passthrough(): - ''' gregorian is already canonical ''' - assert normalize_calendar("gregorian") == "gregorian" + """ gregorian is already canonical """ + assert normalize_calendar('gregorian') == 'gregorian' def test_normalize_calendar_unknown_passthrough(): - ''' unknown calendars are lowercased and passed through ''' - assert normalize_calendar("proleptic_gregorian") == "proleptic_gregorian" - assert normalize_calendar("julian") == "julian" - assert normalize_calendar("CustomCalendar") == "customcalendar" + """ unknown calendars are lowercased and passed through """ + assert normalize_calendar('proleptic_gregorian') == 'proleptic_gregorian' + assert normalize_calendar('julian') == 'julian' + assert normalize_calendar('CustomCalendar') == 'customcalendar' def test_normalize_calendar_none(): - ''' None input returns None ''' + """ None input returns None """ assert normalize_calendar(None) is None # ---- calendars_are_equivalent tests ---- def test_calendars_are_equivalent_noleap_and_365_day(): - ''' noleap and 365_day are CF aliases for the same calendar ''' + """ noleap and 365_day are CF aliases for the same calendar """ assert calendars_are_equivalent('noleap', '365_day') def test_calendars_are_equivalent_365_day_and_noleap(): - ''' 365_day and noleap are CF aliases for the same calendar (reversed order) ''' + """ 365_day and noleap are CF aliases for the same calendar (reversed order) """ assert calendars_are_equivalent('365_day', 'noleap') def test_calendars_are_equivalent_all_leap_and_366_day(): - ''' all_leap and 366_day are CF aliases for the same calendar ''' + """ all_leap and 366_day are CF aliases for the same calendar """ assert calendars_are_equivalent('all_leap', '366_day') def test_calendars_are_equivalent_standard_and_gregorian(): - ''' standard and gregorian are CF aliases for the same calendar ''' + """ standard and gregorian are CF aliases for the same calendar """ assert calendars_are_equivalent('standard', 'gregorian') def test_calendars_are_equivalent_same_name(): - ''' identical calendar names should be equivalent ''' + """ identical calendar names should be equivalent """ assert calendars_are_equivalent('360_day', '360_day') assert calendars_are_equivalent('julian', 'julian') assert calendars_are_equivalent('proleptic_gregorian', 'proleptic_gregorian') def test_calendars_are_equivalent_case_insensitive(): - ''' comparison is case-insensitive ''' + """ comparison is case-insensitive """ assert calendars_are_equivalent('NoLeap', '365_day') assert calendars_are_equivalent('STANDARD', 'gregorian') def test_calendars_are_equivalent_different_calendars(): - ''' distinct calendars should NOT be equivalent ''' + """ distinct calendars should NOT be equivalent """ assert not calendars_are_equivalent('noleap', '360_day') assert not calendars_are_equivalent('gregorian', '360_day') assert not calendars_are_equivalent('julian', 'noleap') @@ -215,22 +215,22 @@ def __init__(self, calendar=None, calendar_type=None): self.calendar = calendar if calendar_type is not None: self.calendar_type = calendar_type - self.units = "days since 0001-01-01" + self.units = 'days since 0001-01-01' def test_get_time_calendar_prefers_calendar_attr(): - ''' calendar attribute takes priority over calendar_type ''' - fake_time = _FakeTime(calendar="NoLeap", calendar_type="julian") - assert get_time_calendar_value(fake_time) == "365_day" + """ calendar attribute takes priority over calendar_type """ + fake_time = _FakeTime(calendar='NoLeap', calendar_type='julian') + assert get_time_calendar_value(fake_time) == '365_day' def test_get_time_calendar_fallback_calendar_type(): - ''' calendar_type is used when calendar attribute is absent ''' - fake_time = _FakeTime(calendar_type="Standard") - assert get_time_calendar_value(fake_time) == "gregorian" + """ calendar_type is used when calendar attribute is absent """ + fake_time = _FakeTime(calendar_type='Standard') + assert get_time_calendar_value(fake_time) == 'gregorian' def test_get_time_calendar_missing_returns_none(): - ''' None is returned when neither calendar attribute is present ''' + """ None is returned when neither calendar attribute is present """ fake_time = _FakeTime() assert get_time_calendar_value(fake_time) is None diff --git a/fremorizer/tests/test_cmor_helpers_update_grid_label.py b/fremorizer/tests/test_cmor_helpers_update_grid_label.py index ae144c2..866f0b3 100644 --- a/fremorizer/tests/test_cmor_helpers_update_grid_label.py +++ b/fremorizer/tests/test_cmor_helpers_update_grid_label.py @@ -1,6 +1,6 @@ -''' +""" unit tests for cmor_helpers.update_grid_and_label -''' +""" import json @@ -11,13 +11,13 @@ # Sample data for testing TEST_JSON_CONTENT = { - "grid_label": "original_label", - "grid": "original_grid", - "nominal_resolution": "original_nom_res", - "other_field": "some_value" + 'grid_label': 'original_label', + 'grid': 'original_grid', + 'nominal_resolution': 'original_nom_res', + 'other_field': 'some_value' } -@pytest.fixture +@pytest.fixture(name='test_json_file') def temp_json_file(tmp_path): """ Fixture to create a temporary JSON file for testing. @@ -28,106 +28,106 @@ def temp_json_file(tmp_path): Returns: Path to the temporary JSON file. """ - json_file = tmp_path / "test_file.json" - with open(json_file, "w", encoding="utf-8") as file: + json_file = tmp_path / 'test_file.json' + with open(json_file, 'w', encoding='utf-8') as file: json.dump(TEST_JSON_CONTENT, file, indent=4) return json_file -def test_update_grid_label_and_grid_success(temp_json_file): +def test_update_grid_label_and_grid_success(test_json_file): """ Test successful update of 'grid_label' and 'grid' fields. """ # Arrange - new_grid = "updated_grid" - new_grid_label = "updated_label" - new_nom_res = "updated_nom_res" + new_grid = 'updated_grid' + new_grid_label = 'updated_label' + new_nom_res = 'updated_nom_res' # Act - update_grid_and_label(temp_json_file, new_grid_label, new_grid, new_nom_res) + update_grid_and_label(test_json_file, new_grid_label, new_grid, new_nom_res) # Assert - with open(temp_json_file, "r", encoding="utf-8") as file: + with open(test_json_file, 'r', encoding='utf-8') as file: data = json.load(file) - assert data["grid"] == new_grid - assert data["grid_label"] == new_grid_label - assert data["nominal_resolution"] == new_nom_res - assert data["other_field"] == "some_value" # Ensure other fields are untouched + assert data['grid'] == new_grid + assert data['grid_label'] == new_grid_label + assert data['nominal_resolution'] == new_nom_res + assert data['other_field'] == 'some_value' # Ensure other fields are untouched -def test_missing_nom_res_field(temp_json_file): +def test_missing_nom_res_field(test_json_file): """ Test behavior when the 'nominal_resolution' field is missing in the JSON file. """ # Arrange - with open(temp_json_file, "r+", encoding="utf-8") as file: + with open(test_json_file, 'r+', encoding='utf-8') as file: data = json.load(file) - del data["nominal_resolution"] # Remove the 'nominal_resolution' field + del data['nominal_resolution'] # Remove the 'nominal_resolution' field file.seek(0) json.dump(data, file, indent=4) file.truncate() - new_grid_label = "updated_label" - new_grid = "updated_grid" - new_nom_res = "updated_nom_res" + new_grid_label = 'updated_label' + new_grid = 'updated_grid' + new_nom_res = 'updated_nom_res' # Act & Assert with pytest.raises( KeyError, - match='"Error updating \'nominal_resolution\'. Ensure the field exists and is modifiable."' + match='Error updating nominal_resolution. Ensure the field exists and is modifiable.' ): - update_grid_and_label(temp_json_file, new_grid_label, new_grid, new_nom_res) + update_grid_and_label(test_json_file, new_grid_label, new_grid, new_nom_res) -def test_missing_grid_label_field(temp_json_file): +def test_missing_grid_label_field(test_json_file): """ Test behavior when the 'grid_label' field is missing in the JSON file. """ # Arrange - with open(temp_json_file, "r+", encoding="utf-8") as file: + with open(test_json_file, 'r+', encoding='utf-8') as file: data = json.load(file) - del data["grid_label"] # Remove the 'grid_label' field + del data['grid_label'] # Remove the 'grid_label' field file.seek(0) json.dump(data, file, indent=4) file.truncate() - new_grid_label = "updated_label" - new_grid = "updated_grid" - new_nom_res = "updated_nom_res" + new_grid_label = 'updated_label' + new_grid = 'updated_grid' + new_nom_res = 'updated_nom_res' # Act & Assert - with pytest.raises(KeyError, match="Error while updating 'grid_label'"): - update_grid_and_label(temp_json_file, new_grid_label, new_grid, new_nom_res) + with pytest.raises(KeyError, match='Error while updating grid_label. Ensure the field exists and is modifiable.'): + update_grid_and_label(test_json_file, new_grid_label, new_grid, new_nom_res) -def test_missing_grid_field(temp_json_file): +def test_missing_grid_field(test_json_file): """ Test behavior when the 'grid' field is missing in the JSON file. """ # Arrange - with open(temp_json_file, "r+", encoding="utf-8") as file: + with open(test_json_file, 'r+', encoding='utf-8') as file: data = json.load(file) - del data["grid"] # Remove the 'grid' field + del data['grid'] # Remove the 'grid' field file.seek(0) json.dump(data, file, indent=4) file.truncate() - new_grid_label = "updated_label" - new_grid = "updated_grid" - new_nom_res = "updated_nom_res" + new_grid_label = 'updated_label' + new_grid = 'updated_grid' + new_nom_res = 'updated_nom_res' # Act & Assert - with pytest.raises(KeyError, match="Error while updating 'grid'"): - update_grid_and_label(temp_json_file, new_grid_label, new_grid, new_nom_res) + with pytest.raises(KeyError, match='Error while updating grid. Ensure the field exists and is modifiable.'): + update_grid_and_label(test_json_file, new_grid_label, new_grid, new_nom_res) def test_invalid_json_file(tmp_path): """ Test behavior when the input file is not a valid JSON file. """ # Arrange - invalid_json_file = tmp_path / "invalid_file.json" - with open(invalid_json_file, "w", encoding="utf-8") as file: - file.write("This is not a valid JSON!") + invalid_json_file = tmp_path / 'invalid_file.json' + with open(invalid_json_file, 'w', encoding='utf-8') as file: + file.write('This is not a valid JSON!') - new_grid_label = "updated_label" - new_grid = "updated_grid" - new_nom_res = "updated_nom_res" + new_grid_label = 'updated_label' + new_grid = 'updated_grid' + new_nom_res = 'updated_nom_res' # Act & Assert with pytest.raises(json.JSONDecodeError): @@ -138,10 +138,10 @@ def test_nonexistent_file(): Test behavior when the specified JSON file does not exist. """ # Arrange - nonexistent_file = Path("nonexistent.json") - new_grid_label = "updated_label" - new_grid = "updated_grid" - new_nom_res = "updated_nom_res" + nonexistent_file = Path('nonexistent.json') + new_grid_label = 'updated_label' + new_grid = 'updated_grid' + new_nom_res = 'updated_nom_res' # Act & Assert with pytest.raises(FileNotFoundError): diff --git a/fremorizer/tests/test_cmor_init_subtool.py b/fremorizer/tests/test_cmor_init_subtool.py index bb33649..c076b1a 100644 --- a/fremorizer/tests/test_cmor_init_subtool.py +++ b/fremorizer/tests/test_cmor_init_subtool.py @@ -1,11 +1,12 @@ -'''Unit tests for cmor_init module +""" +Unit tests for cmor_init module Tests the cmor_init_subtool and its helper functions including: - _fetch_tables_git: git clone table fetching - _fetch_tables_curl: curl tarball table fetching - cmor_init_subtool: main initialization logic - ValueError handling for invalid mip_era -''' +""" import json @@ -20,23 +21,26 @@ def test_cmor_init_invalid_mip_era(): - '''Test that invalid mip_era raises ValueError''' - with pytest.raises(ValueError, match="mip_era must be 'cmip6' or 'cmip7'"): + """ + Test that invalid mip_era raises ValueError + """ + with pytest.raises(ValueError, match='mip_era must be cmip6 or cmip7'): cmor_init_subtool(mip_era='cmip5') - with pytest.raises(ValueError, match="mip_era must be 'cmip6' or 'cmip7'"): + with pytest.raises(ValueError, match='mip_era must be cmip6 or cmip7'): cmor_init_subtool(mip_era='invalid') - with pytest.raises(ValueError, match="mip_era must be 'cmip6' or 'cmip7'"): + with pytest.raises(ValueError, match='mip_era must be cmip6 or cmip7'): cmor_init_subtool(mip_era='CMIP8') def test_cmor_init_tables_dir_with_curl(tmp_path): - '''Test fetching tables with curl (fast mode) when tables_dir is provided. + """ + Test fetching tables with curl (fast mode) when tables_dir is provided. This test actually fetches the CMIP6 tables from GitHub using curl, without mocking, to ensure the full integration works. - ''' + """ tables_dir = tmp_path / 'cmip6_tables_curl' result = cmor_init_subtool( @@ -70,11 +74,12 @@ def test_cmor_init_tables_dir_with_curl(tmp_path): def test_cmor_init_tables_dir_with_git(tmp_path): - '''Test fetching tables with git clone when tables_dir is provided. + """ + Test fetching tables with git clone when tables_dir is provided. This test actually clones the CMIP7 tables from GitHub using git, without mocking, to ensure the full integration works. - ''' + """ tables_dir = tmp_path / 'cmip7_tables_git' result = cmor_init_subtool( @@ -107,11 +112,12 @@ def test_cmor_init_tables_dir_with_git(tmp_path): def test_fetch_tables_git_directly(tmp_path): - '''Test _fetch_tables_git function directly. + """ + Test _fetch_tables_git function directly. This test exercises the git clone functionality without mocking, using a small repository to keep the test fast. - ''' + """ tables_dir = tmp_path / 'direct_git_test' repo_url = MIP_TABLE_REPOS['cmip6'] @@ -128,11 +134,12 @@ def test_fetch_tables_git_directly(tmp_path): def test_fetch_tables_curl_directly(tmp_path): - '''Test _fetch_tables_curl function directly. + """ + Test _fetch_tables_curl function directly. This test exercises the curl + tarball extraction functionality without mocking, using the actual CMIP7 repository. - ''' + """ tables_dir = tmp_path / 'direct_curl_test' repo_url = MIP_TABLE_REPOS['cmip7'] @@ -152,7 +159,9 @@ def test_fetch_tables_curl_directly(tmp_path): def test_cmor_init_tables_dir_and_exp_config(tmp_path): - '''Test that both exp_config and tables_dir can be provided together.''' + """ + Test that both exp_config and tables_dir can be provided together. + """ tables_dir = tmp_path / 'tables' exp_config = tmp_path / 'experiment.json' @@ -179,7 +188,9 @@ def test_cmor_init_tables_dir_and_exp_config(tmp_path): def test_cmor_init_tables_dir_only_no_exp_config(tmp_path): - '''Test that when only tables_dir is provided, no exp_config is created.''' + """ + Test that when only tables_dir is provided, no exp_config is created. + """ tables_dir = tmp_path / 'tables_only' result = cmor_init_subtool( @@ -200,11 +211,12 @@ def test_cmor_init_tables_dir_only_no_exp_config(tmp_path): def test_fetch_tables_curl_with_tag(tmp_path): - '''Test fetching tables with a specific git tag using curl. + """ + Test fetching tables with a specific git tag using curl. This verifies that the tag parameter works correctly. Uses a known tag from the CMIP6 repository (6.9.33). - ''' + """ tables_dir = tmp_path / 'tables_with_tag' repo_url = MIP_TABLE_REPOS['cmip6'] @@ -219,11 +231,12 @@ def test_fetch_tables_curl_with_tag(tmp_path): def test_fetch_tables_git_with_tag(tmp_path): - '''Test fetching tables with a specific git tag using git clone. + """ + Test fetching tables with a specific git tag using git clone. This verifies that the --branch tag parameter works correctly. Uses a known tag from the CMIP6 repository (6.9.33). - ''' + """ tables_dir = tmp_path / 'tables_git_with_tag' repo_url = MIP_TABLE_REPOS['cmip6'] diff --git a/fremorizer/tests/test_cmor_mixer_calendar_integration.py b/fremorizer/tests/test_cmor_mixer_calendar_integration.py index 098671c..8c33c3f 100644 --- a/fremorizer/tests/test_cmor_mixer_calendar_integration.py +++ b/fremorizer/tests/test_cmor_mixer_calendar_integration.py @@ -77,9 +77,9 @@ def fake_nc_filenames(tmp_path): # Shared set of mocks – everything that would touch the filesystem or CMOR _MOCKS = [ - 'fremorizer.cmor_mixer.update_calendar_type', - 'fremorizer.cmor_mixer.cmorize_all_variables_in_dir', - 'fremorizer.cmor_mixer.glob.glob', + "fremorizer.cmor_mixer.update_calendar_type", + "fremorizer.cmor_mixer.cmorize_all_variables_in_dir", + "fremorizer.cmor_mixer.glob.glob", ] diff --git a/fremorizer/tests/test_cmor_mixer_omission_tracking.py b/fremorizer/tests/test_cmor_mixer_omission_tracking.py index 6333c7f..a4c115f 100644 --- a/fremorizer/tests/test_cmor_mixer_omission_tracking.py +++ b/fremorizer/tests/test_cmor_mixer_omission_tracking.py @@ -18,14 +18,14 @@ # --------------------------------------------------------------------------- DUMMY_ARGS = { - "indir": '/fake/indir', - "iso_datetime_range_arr": ['00010101-00041231'], - "name_of_set": 'component', - "json_exp_config": '/fake/exp.json', - "outdir": '/fake/outdir', - "mip_var_cfgs": {'variable_entry': {}}, - "json_table_config": '/fake/table.json', - "run_one_mode": False, + 'indir': '/fake/indir', + 'iso_datetime_range_arr': ['00010101-00041231'], + 'name_of_set': 'component', + 'json_exp_config': '/fake/exp.json', + 'outdir': '/fake/outdir', + 'mip_var_cfgs': {'variable_entry': {}}, + 'json_table_config': '/fake/table.json', + 'run_one_mode': False, } diff --git a/fremorizer/tests/test_cmor_run_subtool.py b/fremorizer/tests/test_cmor_run_subtool.py index b7981cc..a1c9817 100644 --- a/fremorizer/tests/test_cmor_run_subtool.py +++ b/fremorizer/tests/test_cmor_run_subtool.py @@ -1,10 +1,9 @@ -''' +""" tests for fremorizer.cmor_run_subtool -''' +""" from datetime import date import json -import os from pathlib import Path import subprocess import shutil @@ -14,6 +13,7 @@ import pytest from fremorizer import cmor_run_subtool +from fremorizer.tests.conftest import _CMIP6_EXP_CONFIG_DATA # where are we? we're running pytest from the base directory of this repo @@ -26,9 +26,9 @@ f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_Omon.json' def test_setup_cmor_cmip_table_repo(): - ''' + """ setup routine, make sure the recursively cloned tables exist - ''' + """ assert all( [ Path(CMIP6_TABLE_REPO_PATH).exists(), Path(TABLE_CONFIG).exists() ] ) @@ -48,7 +48,7 @@ def test_setup_cmor_cmip_table_repo(): # input file details. if calendar matches data, the dates should be preserved or equiv. DATETIMES_INPUTFILE='199301-199302' FILENAME = f'reduced_ocean_monthly_1x1deg.{DATETIMES_INPUTFILE}.sos' -FULL_INPUTFILE=f"{INDIR}/{FILENAME}.nc" +FULL_INPUTFILE=f'{INDIR}/{FILENAME}.nc' CALENDAR_TYPE = 'julian' # determined by cmor_run_subtool @@ -56,9 +56,9 @@ def test_setup_cmor_cmip_table_repo(): CMOR_CREATES_DIR = \ f'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1/Omon/sos/{GRID_LABEL}' FULL_OUTPUTDIR = \ - f"{OUTDIR}/{CMOR_CREATES_DIR}/v{YYYYMMDD}" + f'{OUTDIR}/{CMOR_CREATES_DIR}/v{YYYYMMDD}' FULL_OUTPUTFILE = \ -f"{FULL_OUTPUTDIR}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_{GRID_LABEL}_{DATETIMES_INPUTFILE}.nc" +f'{FULL_OUTPUTDIR}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_{GRID_LABEL}_{DATETIMES_INPUTFILE}.nc' # CMIP6-required global attributes that must be present in CMOR output CMIP6_REQUIRED_GLOBAL_ATTRS = [ @@ -68,54 +68,54 @@ def test_setup_cmor_cmip_table_repo(): def _assert_data_matches(ds_in, ds_out): - ''' + """ helper: assert that science variable data, coordinate data, and shapes are preserved between input and CMOR output datasets. - ''' + """ # the science variable data must be preserved exactly assert np.array_equal(ds_in.variables['sos'][:], ds_out.variables['sos'][:]), \ - "sos data values differ between input and CMOR output" + 'sos data values differ between input and CMOR output' # coordinate data must be preserved assert np.allclose(ds_in.variables['lat'][:], ds_out.variables['lat'][:]), \ - "latitude data differs between input and CMOR output" + 'latitude data differs between input and CMOR output' assert np.allclose(ds_in.variables['lon'][:], ds_out.variables['lon'][:]), \ - "longitude data differs between input and CMOR output" + 'longitude data differs between input and CMOR output' assert np.allclose(ds_in.variables['time'][:], ds_out.variables['time'][:]), \ - "time data differs between input and CMOR output" + 'time data differs between input and CMOR output' # variable shapes must be preserved assert ds_in.variables['sos'][:].shape == ds_out.variables['sos'][:].shape, \ - "sos data shape differs between input and CMOR output" + 'sos data shape differs between input and CMOR output' def _assert_metadata_matches(ds_in, ds_out): - ''' + """ helper: assert that CMIP6-required global attributes are present and that key variable-level metadata is preserved between input and CMOR output datasets. - ''' + """ # CMOR output must contain CMIP6-required global attributes for required_attr in CMIP6_REQUIRED_GLOBAL_ATTRS: assert required_attr in ds_out.ncattrs(), \ - f"CMOR output missing required global attribute '{required_attr}'" + f'CMOR output missing required global attribute {required_attr}' # science variable standard_name and long_name must be preserved assert ds_in.variables['sos'].standard_name == ds_out.variables['sos'].standard_name, \ - "sos standard_name differs between input and CMOR output" + 'sos standard_name differs between input and CMOR output' assert ds_in.variables['sos'].long_name == ds_out.variables['sos'].long_name, \ - "sos long_name differs between input and CMOR output" + 'sos long_name differs between input and CMOR output' # _FillValue and missing_value must be preserved assert ds_in.variables['sos']._FillValue == ds_out.variables['sos']._FillValue, \ - "sos _FillValue differs between input and CMOR output" # pylint: disable=protected-access + 'sos _FillValue differs between input and CMOR output' # pylint: disable=protected-access assert ds_in.variables['sos'].missing_value == ds_out.variables['sos'].missing_value, \ - "sos missing_value differs between input and CMOR output" + 'sos missing_value differs between input and CMOR output' def _write_fake_table(tmp_path, filename, mip_era): - ''' + """ helper: create a minimal MIP table JSON with a mip_era header for mismatch tests - ''' + """ table_path = tmp_path / filename table_path.write_text(json.dumps({ 'Header': {'mip_era': mip_era}, @@ -125,9 +125,9 @@ def _write_fake_table(tmp_path, filename, mip_era): def test_cmip6_exp_with_cmip7_table_raises(tmp_path): - ''' + """ ValueError with clear message when CMIP6 experiment uses a CMIP7-format table. - ''' + """ table_path = _write_fake_table(tmp_path, 'CMIP7_fake.json', 'CMIP7') indir = tmp_path / 'indir' outdir = tmp_path / 'outdir' @@ -145,9 +145,9 @@ def test_cmip6_exp_with_cmip7_table_raises(tmp_path): def test_cmip7_exp_with_cmip6_table_raises(tmp_path): - ''' + """ ValueError with clear message when CMIP7 experiment uses a CMIP6-format table. - ''' + """ table_path = _write_fake_table(tmp_path, 'CMIP6_fake.json', 'CMIP6') indir = tmp_path / 'indir' outdir = tmp_path / 'outdir' @@ -165,14 +165,14 @@ def test_cmip7_exp_with_cmip6_table_raises(tmp_path): def test_setup_fre_cmor_run_subtool(capfd): - ''' + """ The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen output file from prev pytest runs, removes it if it's present, and ensures the new file is created without error. - ''' + """ - ncgen_input = f"{ROOTDIR}/reduced_ascii_files/{FILENAME}.cdl" - ncgen_output = f"{ROOTDIR}/ocean_sos_var_file/{FILENAME}.nc" + ncgen_input = f'{ROOTDIR}/reduced_ascii_files/{FILENAME}.cdl' + ncgen_output = f'{ROOTDIR}/ocean_sos_var_file/{FILENAME}.nc' Path(ncgen_output).parent.mkdir(parents=True, exist_ok=True) if Path(ncgen_output).exists(): @@ -192,20 +192,20 @@ def test_setup_fre_cmor_run_subtool(capfd): _out, _err = capfd.readouterr() def test_fre_cmor_run_subtool_case1(capfd): - ''' fre cmor run, test-use case ''' + """ fre cmor run, test-use case """ #import sys #assert False, f'{sys.path}' #debug #print( - # f"cmor_run_subtool(" - # f"\'{INDIR}\'," - # f"\'{VARLIST}\'," - # f"\'{TABLE_CONFIG}\'," - # f"\'{EXP_CONFIG}\'," - # f"\'{OUTDIR}\'" - # ")" + # f'cmor_run_subtool(' + # f'\'{INDIR}\',' + # f'\'{VARLIST}\',' + # f'\'{TABLE_CONFIG}\',' + # f'\'{EXP_CONFIG}\',' + # f'\'{OUTDIR}\'' + # ')' #) # test call, where meat of the workload gets done @@ -227,7 +227,7 @@ def test_fre_cmor_run_subtool_case1(capfd): _out, _err = capfd.readouterr() def test_fre_cmor_run_subtool_case1_output_compare_data(capfd): - ''' I/O data-only comparison of test case1 ''' + """ I/O data-only comparison of test case1 """ print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') print(f'FULL_INPUTFILE={FULL_INPUTFILE}') @@ -235,13 +235,13 @@ def test_fre_cmor_run_subtool_case1_output_compare_data(capfd): netCDF4.Dataset(FULL_OUTPUTFILE) as ds_out: # file formats should differ: CMOR converts input to NETCDF4_CLASSIC assert ds_in.file_format != ds_out.file_format, \ - f"expected file formats to differ, got input={ds_in.file_format}, output={ds_out.file_format}" + f'expected file formats to differ, got input={ds_in.file_format}, output={ds_out.file_format}' _assert_data_matches(ds_in, ds_out) _out, _err = capfd.readouterr() def test_fre_cmor_run_subtool_case1_output_compare_metadata(capfd): - ''' I/O metadata-only comparison of test case1 ''' + """ I/O metadata-only comparison of test case1 """ print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') print(f'FULL_INPUTFILE={FULL_INPUTFILE}') @@ -249,7 +249,7 @@ def test_fre_cmor_run_subtool_case1_output_compare_metadata(capfd): netCDF4.Dataset(FULL_OUTPUTFILE) as ds_out: # CMOR processing should add/change global attributes assert set(ds_in.ncattrs()) != set(ds_out.ncattrs()), \ - "expected global attributes to differ between input and CMOR output" + 'expected global attributes to differ between input and CMOR output' _assert_metadata_matches(ds_in, ds_out) _out, _err = capfd.readouterr() @@ -259,13 +259,13 @@ def test_fre_cmor_run_subtool_case1_output_compare_metadata(capfd): FILENAME_DIFF = \ f'reduced_ocean_monthly_1x1deg.{DATETIMES_INPUTFILE}.sosV2.nc' FULL_INPUTFILE_DIFF = \ - f"{INDIR}/{FILENAME_DIFF}" + f'{INDIR}/{FILENAME_DIFF}' VARLIST_DIFF = \ f'{ROOTDIR}/varlist_local_target_vars_differ' def test_setup_fre_cmor_run_subtool_case2(capfd): - ''' make a copy of the input file to the slightly different name. + """ make a copy of the input file to the slightly different name. checks for outputfile from prev pytest runs, removes it if it's present. - this routine also checks to make sure the desired input file is present''' + this routine also checks to make sure the desired input file is present""" if Path(FULL_OUTPUTFILE).exists(): Path(FULL_OUTPUTFILE).unlink() assert not Path(FULL_OUTPUTFILE).exists() @@ -308,17 +308,17 @@ def test_setup_fre_cmor_run_subtool_case2(capfd): _out, _err = capfd.readouterr() def test_fre_cmor_run_subtool_case2(capfd): - ''' fre cmor run, test-use case2 ''' + """ fre cmor run, test-use case2 """ #debug #print( - # f"cmor_run_subtool(" - # f"\'{INDIR}\'," - # f"\'{VARLIST_DIFF}\'," - # f"\'{TABLE_CONFIG}\'," - # f"\'{EXP_CONFIG}\'," - # f"\'{OUTDIR}\'" - # ")" + # f'cmor_run_subtool(' + # f'\'{INDIR}\',' + # f'\'{VARLIST_DIFF}\',' + # f'\'{TABLE_CONFIG}\',' + # f'\'{EXP_CONFIG}\',' + # f'\'{OUTDIR}\'' + # ')' #) # test call, where meat of the workload gets done @@ -342,7 +342,7 @@ def test_fre_cmor_run_subtool_case2(capfd): def test_fre_cmor_run_subtool_case2_output_compare_data(capfd): - ''' I/O data-only comparison of test case2 ''' + """ I/O data-only comparison of test case2 """ print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') print(f'FULL_INPUTFILE_DIFF={FULL_INPUTFILE_DIFF}') @@ -350,13 +350,13 @@ def test_fre_cmor_run_subtool_case2_output_compare_data(capfd): netCDF4.Dataset(FULL_OUTPUTFILE) as ds_out: # file formats should differ: CMOR converts input to NETCDF4_CLASSIC assert ds_in.file_format != ds_out.file_format, \ - f"expected file formats to differ, got input={ds_in.file_format}, output={ds_out.file_format}" + f'expected file formats to differ, got input={ds_in.file_format}, output={ds_out.file_format}' _assert_data_matches(ds_in, ds_out) _out, _err = capfd.readouterr() def test_fre_cmor_run_subtool_case2_output_compare_metadata(capfd): - ''' I/O metadata-only comparison of test case2 ''' + """ I/O metadata-only comparison of test case2 """ print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') print(f'FULL_INPUTFILE_DIFF={FULL_INPUTFILE_DIFF}') @@ -364,34 +364,28 @@ def test_fre_cmor_run_subtool_case2_output_compare_metadata(capfd): netCDF4.Dataset(FULL_OUTPUTFILE) as ds_out: # CMOR processing should add/change global attributes assert set(ds_in.ncattrs()) != set(ds_out.ncattrs()), \ - "expected global attributes to differ between input and CMOR output" + 'expected global attributes to differ between input and CMOR output' _assert_metadata_matches(ds_in, ds_out) _out, _err = capfd.readouterr() -def test_git_cleanup(): - ''' - Performs a git restore on EXP_CONFIG to avoid false positives from - git's record of changed files. It's supposed to change as part of the test. - ''' - is_ci = os.environ.get("GITHUB_WORKSPACE") is not None - if not is_ci: - git_cmd = f"git restore {EXP_CONFIG}" - restore = subprocess.run(git_cmd, - shell=True, - check=False) - check_cmd = f"git status | grep {EXP_CONFIG}" - check = subprocess.run(check_cmd, - shell = True, - check = False) - #first command completed, second found no file in git status - assert all([restore.returncode == 0, - check.returncode == 1]) +def test_exp_config_cleanup(): + """ + Restores the CMIP6 experiment config to its pristine state after tests + that mutate it in-place (e.g. grid / calendar updates). + + The config is no longer tracked by git — it is materialised by a + session-scoped conftest fixture — so we rewrite it from the canonical + fixture data instead of running ``git restore``. + """ + Path(EXP_CONFIG).write_text( + json.dumps(_CMIP6_EXP_CONFIG_DATA, indent=4), encoding='utf-8' + ) def test_cmor_run_subtool_raise_value_error(): - ''' + """ test that ValueError raised when required args are absent - ''' + """ with pytest.raises(ValueError): cmor_run_subtool( indir = None, json_var_list = None, @@ -400,9 +394,9 @@ def test_cmor_run_subtool_raise_value_error(): outdir = None ) def test_fre_cmor_run_subtool_no_exp_config(): - ''' + """ fre cmor run, exception, json_exp_config DNE - ''' + """ # test call, where meat of the workload gets done with pytest.raises(FileNotFoundError): @@ -417,9 +411,9 @@ def test_fre_cmor_run_subtool_no_exp_config(): VARLIST_EMPTY = \ f'{ROOTDIR}/empty_varlist' def test_fre_cmor_run_subtool_empty_varlist(): - ''' + """ fre cmor run, exception, variable list is empty - ''' + """ # test call, where meat of the workload gets done with pytest.raises(ValueError): @@ -433,7 +427,7 @@ def test_fre_cmor_run_subtool_empty_varlist(): def test_fre_cmor_run_subtool_opt_var_name_not_in_table(): - ''' fre cmor run, exception, ''' + """ fre cmor run, exception, """ # test call, where meat of the workload gets done with pytest.raises(ValueError): @@ -443,17 +437,17 @@ def test_fre_cmor_run_subtool_opt_var_name_not_in_table(): json_table_config = TABLE_CONFIG, json_exp_config = EXP_CONFIG, outdir = OUTDIR, - opt_var_name="difmxybo" + opt_var_name='difmxybo' ) def test_fre_cmor_run_subtool_missing_mip_era(tmp_path): - ''' + """ KeyError when the exp config JSON has no mip_era entry. - ''' + """ # create a minimal exp config that is missing 'mip_era' bad_exp = tmp_path / 'no_mip_era.json' - exp_data = {"institution_id": "TEST", "source_id": "TEST-1-0"} + exp_data = {'institution_id': 'TEST', 'source_id': 'TEST-1-0'} bad_exp.write_text(json.dumps(exp_data)) with pytest.raises(KeyError, match='noncompliant'): @@ -467,9 +461,9 @@ def test_fre_cmor_run_subtool_missing_mip_era(tmp_path): def test_fre_cmor_run_subtool_unsupported_mip_era(tmp_path): - ''' + """ ValueError when mip_era is present but not CMIP6 or CMIP7. - ''' + """ # create an exp config with an unsupported mip_era value bad_exp = tmp_path / 'bad_mip_era.json' with open(EXP_CONFIG, 'r', encoding='utf-8') as f: diff --git a/fremorizer/tests/test_cmor_run_subtool_cmip7.py b/fremorizer/tests/test_cmor_run_subtool_cmip7.py new file mode 100644 index 0000000..14b9093 --- /dev/null +++ b/fremorizer/tests/test_cmor_run_subtool_cmip7.py @@ -0,0 +1,351 @@ +""" +CMIP7-flavored tests for fremorizer.cmor_run_subtool + +Each test mirrors a corresponding CMIP6 test in test_cmor_run_subtool.py but +targets the CMIP7 experiment-configuration JSON and CMIP7-format CMOR tables. +""" + +from datetime import date +from pathlib import Path +import subprocess +import shutil + +import netCDF4 +import numpy as np +import pytest + +from fremorizer import cmor_run_subtool + + +# where are we? we're running pytest from the base directory of this repo +ROOTDIR = 'fremorizer/tests/test_files' + +# setup- cmip/cmor variable table(s) +CMIP7_TABLE_REPO_PATH = \ + f'{ROOTDIR}/cmip7-cmor-tables' +TABLE_CONFIG = \ + f'{CMIP7_TABLE_REPO_PATH}/tables/CMIP7_ocean.json' + +# explicit inputs to tool +GRID = 'regridded to FOO grid from native' #placeholder value +GRID_LABEL = 'g999' +NOM_RES = '10000 km' #placeholder value + +INDIR = f'{ROOTDIR}/ocean_sos_var_file' +VARLIST = f'{ROOTDIR}/varlist' +CMIP7_EXP_CONFIG = f'{ROOTDIR}/CMOR_CMIP7_input_example.json' +OUTDIR = f'{ROOTDIR}/outdir' +TMPDIR = f'{OUTDIR}/tmp' + +# input file details. if calendar matches data, the dates should be preserved or equiv. +DATETIMES_INPUTFILE='199301-199302' +FILENAME = f'reduced_ocean_monthly_1x1deg.{DATETIMES_INPUTFILE}.sos' +FULL_INPUTFILE=f'{INDIR}/{FILENAME}.nc' +CALENDAR_TYPE = 'julian' + +# determined by cmor_run_subtool +YYYYMMDD = date.today().strftime('%Y%m%d') + +# CMIP7 output path follows output_path_template: +# +CMOR_CREATES_DIR = \ + f'CMIP/DUMMY-MODEL/historical/r3i1p1f3/sos/tavg-u-hxy-sea/{GRID_LABEL}' +FULL_OUTPUTDIR = \ + f'{OUTDIR}/{CMOR_CREATES_DIR}' +FULL_OUTPUTFILE = \ + f'{FULL_OUTPUTDIR}/sos_tavg-u-hxy-sea_mon_glb_{GRID_LABEL}_DUMMY-MODEL_historical_r3i1p1f3_{DATETIMES_INPUTFILE}.nc' + +# CMIP7-required global attributes that must be present in CMOR output +# note: CMIP7 uses 'table_info' instead of 'table_id' +CMIP7_REQUIRED_GLOBAL_ATTRS = [ + 'variable_id', 'mip_era', 'table_info', + 'experiment_id', 'institution_id', 'source_id' +] + + +def _assert_data_matches(ds_in, ds_out): + """ + helper: assert that science variable data, coordinate data, and shapes + are preserved between input and CMOR output datasets. + """ + # the science variable data must be preserved exactly + assert np.array_equal(ds_in.variables['sos'][:], ds_out.variables['sos'][:]), \ + 'sos data values differ between input and CMOR output' + + # coordinate data must be preserved + assert np.allclose(ds_in.variables['lat'][:], ds_out.variables['lat'][:]), \ + 'latitude data differs between input and CMOR output' + assert np.allclose(ds_in.variables['lon'][:], ds_out.variables['lon'][:]), \ + 'longitude data differs between input and CMOR output' + assert np.allclose(ds_in.variables['time'][:], ds_out.variables['time'][:]), \ + 'time data differs between input and CMOR output' + + # variable shapes must be preserved + assert ds_in.variables['sos'][:].shape == ds_out.variables['sos'][:].shape, \ + 'sos data shape differs between input and CMOR output' + + +def _assert_metadata_matches(ds_in, ds_out): + """ + helper: assert that CMIP7-required global attributes are present and that + key variable-level metadata is preserved between input and CMOR output datasets. + """ + # CMOR output must contain CMIP7-required global attributes + for required_attr in CMIP7_REQUIRED_GLOBAL_ATTRS: + assert required_attr in ds_out.ncattrs(), \ + f'CMOR output missing required global attribute {required_attr}' + + # CMIP7 uses 'table_info' instead of 'table_id' + assert 'table_info' in ds_out.ncattrs(), \ + 'CMOR output missing table_info attribute (CMIP7-specific)' + assert 'table_id' not in ds_out.ncattrs(), \ + 'CMOR output should not have table_id for CMIP7 (uses table_info instead)' + + # science variable standard_name and long_name must be preserved + assert ds_in.variables['sos'].standard_name == ds_out.variables['sos'].standard_name, \ + 'sos standard_name differs between input and CMOR output' + assert ds_in.variables['sos'].long_name == ds_out.variables['sos'].long_name, \ + 'sos long_name differs between input and CMOR output' + + # _FillValue and missing_value must be preserved + assert ds_in.variables['sos']._FillValue == ds_out.variables['sos']._FillValue, \ + 'sos _FillValue differs between input and CMOR output' # pylint: disable=protected-access + assert ds_in.variables['sos'].missing_value == ds_out.variables['sos'].missing_value, \ + 'sos missing_value differs between input and CMOR output' + + +# --------------------------------------------------------------------------- +# CMIP7 table repo setup +# --------------------------------------------------------------------------- +def test_setup_cmip7_cmor_table_repo(): + """ + setup routine, make sure the recursively cloned CMIP7 tables exist + """ + assert all( [ Path(CMIP7_TABLE_REPO_PATH).exists(), + Path(TABLE_CONFIG).exists() + ] ) + + +# --------------------------------------------------------------------------- +# CMIP7 case 1: basic CMORization run +# --------------------------------------------------------------------------- +def test_setup_fre_cmor_run_subtool_cmip7(capfd): + """ + The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen + output file from prev pytest runs, removes it if it's present, and ensures the new file is + created without error. (CMIP7 version) + """ + ncgen_input = f'{ROOTDIR}/reduced_ascii_files/{FILENAME}.cdl' + ncgen_output = f'{ROOTDIR}/ocean_sos_var_file/{FILENAME}.nc' + + Path(ncgen_output).parent.mkdir(parents=True, exist_ok=True) + if Path(ncgen_output).exists(): + Path(ncgen_output).unlink() + assert Path(ncgen_input).exists() + + ex = [ 'ncgen3', '-k', 'netCDF-4', '-o', ncgen_output, ncgen_input ] + + sp = subprocess.run(ex, check = True) + + assert all( [ sp.returncode == 0, Path(ncgen_output).exists() ] ) + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_cmip7_case1(capfd): + """ + fre cmor run, CMIP7 test-use case + """ + + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST, + json_table_config = TABLE_CONFIG, + json_exp_config = CMIP7_EXP_CONFIG, + outdir = OUTDIR, + run_one_mode = True, + grid_label = GRID_LABEL, + grid = GRID, + nom_res = NOM_RES, + calendar_type = CALENDAR_TYPE + ) + + assert all( [ Path(FULL_OUTPUTFILE).exists(), + Path(FULL_INPUTFILE).exists() ] ) + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_cmip7_case1_output_compare_data(capfd): + """ + I/O data-only comparison of CMIP7 test case1 + """ + print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') + print(f'FULL_INPUTFILE={FULL_INPUTFILE}') + + with netCDF4.Dataset(FULL_INPUTFILE) as ds_in, \ + netCDF4.Dataset(FULL_OUTPUTFILE) as ds_out: + # file formats should differ: CMOR converts input to NETCDF4_CLASSIC + assert ds_in.file_format != ds_out.file_format, \ + f'expected file formats to differ, got input={ds_in.file_format}, output={ds_out.file_format}' + + _assert_data_matches(ds_in, ds_out) + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_cmip7_case1_output_compare_metadata(capfd): + """ + I/O metadata-only comparison of CMIP7 test case1 + """ + print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') + print(f'FULL_INPUTFILE={FULL_INPUTFILE}') + + with netCDF4.Dataset(FULL_INPUTFILE) as ds_in, \ + netCDF4.Dataset(FULL_OUTPUTFILE) as ds_out: + # CMOR processing should add/change global attributes + assert set(ds_in.ncattrs()) != set(ds_out.ncattrs()), \ + 'expected global attributes to differ between input and CMOR output' + + _assert_metadata_matches(ds_in, ds_out) + _out, _err = capfd.readouterr() + + +# --------------------------------------------------------------------------- +# CMIP7 case 2: differing local vs target variable names +# --------------------------------------------------------------------------- +FILENAME_DIFF = \ + f'reduced_ocean_monthly_1x1deg.{DATETIMES_INPUTFILE}.sosV2.nc' +FULL_INPUTFILE_DIFF = \ + f'{INDIR}/{FILENAME_DIFF}' +VARLIST_DIFF = \ + f'{ROOTDIR}/varlist_local_target_vars_differ' + +def test_setup_fre_cmor_run_subtool_cmip7_case2(capfd): + """ + make a copy of the input file to the slightly different name. + checks for outputfile from prev pytest runs, removes it if it's present. + this routine also checks to make sure the desired input file is present (CMIP7 version) + """ + + if Path(OUTDIR).exists(): + try: + shutil.rmtree(OUTDIR) + except OSError as exc: + print(f'WARNING: OUTDIR={OUTDIR} could not be removed: {exc}') + + # make a copy of the usual test file. + if not Path(FULL_INPUTFILE_DIFF).exists(): + shutil.copy( + Path(FULL_INPUTFILE), + Path(FULL_INPUTFILE_DIFF) ) + assert Path(FULL_INPUTFILE_DIFF).exists() + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_cmip7_case2(capfd): + """ + fre cmor run, CMIP7 test-use case2 + """ + + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST_DIFF, + json_table_config = TABLE_CONFIG, + json_exp_config = CMIP7_EXP_CONFIG, + outdir = OUTDIR, + run_one_mode = True, + grid_label = GRID_LABEL, + grid = GRID, + nom_res = NOM_RES, + calendar_type = CALENDAR_TYPE + ) + + # check we ran on the right input file and output was created. + assert all( [ Path(FULL_OUTPUTFILE).exists(), + Path(FULL_INPUTFILE_DIFF).exists() ] ) + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_cmip7_case2_output_compare_data(capfd): + """ + I/O data-only comparison of CMIP7 test case2 + """ + print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') + print(f'FULL_INPUTFILE_DIFF={FULL_INPUTFILE_DIFF}') + + with netCDF4.Dataset(FULL_INPUTFILE_DIFF) as ds_in, \ + netCDF4.Dataset(FULL_OUTPUTFILE) as ds_out: + # file formats should differ: CMOR converts input to NETCDF4_CLASSIC + assert ds_in.file_format != ds_out.file_format, \ + f'expected file formats to differ, got input={ds_in.file_format}, output={ds_out.file_format}' + + _assert_data_matches(ds_in, ds_out) + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_cmip7_case2_output_compare_metadata(capfd): + """ + I/O metadata-only comparison of CMIP7 test case2 + """ + print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') + print(f'FULL_INPUTFILE_DIFF={FULL_INPUTFILE_DIFF}') + + with netCDF4.Dataset(FULL_INPUTFILE_DIFF) as ds_in, \ + netCDF4.Dataset(FULL_OUTPUTFILE) as ds_out: + # CMOR processing should add/change global attributes + assert set(ds_in.ncattrs()) != set(ds_out.ncattrs()), \ + 'expected global attributes to differ between input and CMOR output' + + _assert_metadata_matches(ds_in, ds_out) + _out, _err = capfd.readouterr() + + +# --------------------------------------------------------------------------- +# CMIP7 error handling tests +# --------------------------------------------------------------------------- +def test_cmor_run_subtool_cmip7_raise_value_error(): + """ + test that ValueError raised when required args are absent (CMIP7 version) + """ + with pytest.raises(ValueError): + cmor_run_subtool( indir = None, + json_var_list = None, + json_table_config = None, + json_exp_config = None, + outdir = None ) + +def test_fre_cmor_run_subtool_cmip7_no_exp_config(): + """ + fre cmor run, exception, json_exp_config DNE (CMIP7 version) + """ + + with pytest.raises(FileNotFoundError): + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST_DIFF, + json_table_config = TABLE_CONFIG, + json_exp_config = 'DOES NOT EXIST', + outdir = OUTDIR + ) + +def test_fre_cmor_run_subtool_cmip7_empty_varlist(): + """ + fre cmor run, exception, variable list is empty (CMIP7 version) + """ + varlist_empty = f'{ROOTDIR}/empty_varlist' + + with pytest.raises(ValueError): + cmor_run_subtool( + indir = INDIR, + json_var_list = varlist_empty, + json_table_config = TABLE_CONFIG, + json_exp_config = CMIP7_EXP_CONFIG, + outdir = OUTDIR + ) + +def test_fre_cmor_run_subtool_cmip7_opt_var_name_not_in_table(): + """ + fre cmor run, exception, opt_var_name not in CMIP7 table + """ + + with pytest.raises(ValueError): + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST, + json_table_config = TABLE_CONFIG, + json_exp_config = CMIP7_EXP_CONFIG, + outdir = OUTDIR, + opt_var_name='zzzz_nonexistent_var' + ) diff --git a/fremorizer/tests/test_cmor_run_subtool_cmip7_further_examples.py b/fremorizer/tests/test_cmor_run_subtool_cmip7_further_examples.py new file mode 100644 index 0000000..b2f5c62 --- /dev/null +++ b/fremorizer/tests/test_cmor_run_subtool_cmip7_further_examples.py @@ -0,0 +1,201 @@ +''' +Expanded set of CMIP7 tests for fremor run — cases beyond test_cmor_run_subtool_cmip7.py. + +These tests exercise cmor_run_subtool against a variety of variables, tables, +and grid labels drawn from a mock pp-archive, targeting CMIP7 experiment +configuration and CMIP7-format CMOR tables. + +.. tip:: pytest temp directories + By default pytest removes temp directories after the session. To keep + them around for debugging, run:: + + pytest --basetemp=/tmp/fremorizer-debug -k test_case_cmip7 -x + + Output files will then persist under ``/tmp/fremorizer-debug``. +''' + +from datetime import date +import glob +from pathlib import Path + +import pytest + +from fremorizer import cmor_run_subtool +from fremorizer.tests.conftest import ncgen + + +# ── path constants ────────────────────────────────────────────────────────── +ROOTDIR = 'fremorizer/tests/test_files' +CMORBITE_VARLIST = f'{ROOTDIR}/CMORbite_var_list.json' + +# cmip7 table repo +CMIP7_TABLE_REPO_PATH = f'{ROOTDIR}/cmip7-cmor-tables' + +# experiment config (materialised by conftest._write_exp_configs) +EXP_CONFIG_CMIP7 = f'{ROOTDIR}/CMOR_CMIP7_input_example.json' + +# determined by cmor_run_subtool +YYYYMMDD = date.today().strftime('%Y%m%d') + +# mock-archive base paths +MOCK_ARCHIVE_ROOT = f'{ROOTDIR}/ascii_files/mock_archive' +ESM4_DECK_PP_DIR = ( + 'cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp' +) +ESM4_DEV_PP_DIR = ( + 'USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC' + '/gfdl.ncrc5-intel23-prod-openmp/pp' +) + +# CMIP7 output dir structure +# (activity_id/source_id/experiment_id/member_id/variable_id/branding_suffix/grid_label) +CMOR_CREATES_DIR_BASE_CMIP7 = ( + 'CMIP/DUMMY-MODEL/historical/r3i1p1f3' +) + + +# ── helper: convert CDLs to NC in a test directory ───────────────────────── +def _ncgen_for_case(testfile_dir, opt_var_name): + '''Convert the CDL file(s) for *opt_var_name* to NetCDF-4 inside *testfile_dir*. + + Returns the path to the primary NC file. Also generates ancillary files + (e.g. ``ps.nc`` for ``cl`` / ``mc``, ``ocean_monthly.static.nc`` for native + ocean grid variables). + ''' + cdl_files = glob.glob(f'{testfile_dir}*.{opt_var_name}.cdl') + assert len(cdl_files) >= 1, ( + f'no CDL file found for variable {opt_var_name} in {testfile_dir}' + ) + cdl_file = cdl_files[0] + nc_file = cdl_file.replace('.cdl', '.nc') + ncgen(cdl_file, nc_file) + + # ancillary files required by specific variables + if opt_var_name in ('cl', 'mc'): + ps_cdl = cdl_file.replace(f'{opt_var_name}.cdl', 'ps.cdl') + assert Path(ps_cdl).exists(), f'ps CDL not found: {ps_cdl}' + ncgen(ps_cdl, ps_cdl.replace('.cdl', '.nc')) + + elif opt_var_name == 'sos': + static_cdl = testfile_dir.replace( + 'ts/monthly/5yr/', 'ocean_monthly.static.cdl' + ) + if Path(static_cdl).exists(): + ncgen(static_cdl, static_cdl.replace('.cdl', '.nc')) + + return nc_file + + +# ── CMIP7 parametrized tests ────────────────────────────────────────────── +@pytest.mark.parametrize( + 'testfile_dir,table,opt_var_name,grid_label,start,calendar', + [ + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/atmos_plev39_cmip/ts/monthly/5yr/zonavg/', + 'CMIP7_atmos', 'ta', 'g999', '1850', 'noleap', + id='atmos_ta_g999', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/atmos_scalar/ts/monthly/5yr/', + 'CMIP7_atmosChem', 'ch4global', 'g999', '1850', 'noleap', + id='atmosChem_ch4global_g999', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/LUmip_refined/ts/monthly/5yr/', + 'CMIP7_land', 'gppLut', 'g999', '1850', 'noleap', + id='land_gppLut_g999', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/atmos_level_cmip/ts/monthly/5yr/', + 'CMIP7_atmos', 'cl', 'g999', '1850', 'noleap', + id='atmos_cl_g999', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/atmos_level_cmip/ts/monthly/5yr/', + 'CMIP7_atmos', 'mc', 'g999', '1850', 'noleap', + id='atmos_mc_g999', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}' + '/ocean_monthly_z_1x1deg/ts/monthly/5yr/', + 'CMIP7_ocean', 'so', 'g999', '0001', 'noleap', + id='ocean_so_g999', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}' + '/ocean_monthly/ts/monthly/5yr/', + 'CMIP7_ocean', 'sos', 'g999', '0001', 'noleap', + id='ocean_sos_g999', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}' + '/land/ts/monthly/5yr/', + 'CMIP7_land', 'lai', 'g999', '0001', 'noleap', + id='land_lai_g999', + ), + ], +) +def test_case_cmip7( # pylint: disable=too-many-arguments,too-many-positional-arguments + testfile_dir, table, opt_var_name, grid_label, start, calendar, + tmp_path, monkeypatch, +): + '''Run cmor_run_subtool for a single CMIP7 variable and assert output exists.''' + # ── conditional skips for cases that cannot yet pass ──────────────────── + if opt_var_name == 'ch4global': + pytest.skip( + 'ch4global does not exist in any CMIP7 table ' + '(CMIP7 uses ch4 variants instead); ' + 'needs new mock data or a different variable mapping' + ) + if opt_var_name == 'gppLut': + pytest.skip( + 'gppLut_tavg-u-hxy-multi cmor.axis fails: ' + 'CMIP7 landuse coordinate definition is incompatible ' + 'with the mock archive landuse axis values' + ) + + # native-grid ocean tests: prevent gold statics lookup from finding /archive files + if grid_label == 'gn': + monkeypatch.setattr( + 'fremorizer.cmor_mixer.find_gold_ocean_statics_file', + lambda **kw: None, + ) + + _ncgen_for_case(testfile_dir, opt_var_name) + + table_file = f'{CMIP7_TABLE_REPO_PATH}/tables/{table}.json' + outdir = str(tmp_path / 'outdir') + + cmor_run_subtool( + indir=testfile_dir, + json_var_list=CMORBITE_VARLIST, + json_table_config=table_file, + json_exp_config=EXP_CONFIG_CMIP7, + outdir=outdir, + run_one_mode=True, + opt_var_name=opt_var_name, + grid='FOO_PLACEHOLDER', + grid_label=grid_label, + nom_res='10000 km', + start=start, + calendar_type=calendar, + ) + + # CMIP7 output_path_template: + # //// + # // + # Use recursive glob to find output regardless of branding suffix. + cmor_output_glob = ( + f'{outdir}/{CMOR_CREATES_DIR_BASE_CMIP7}' + f'/{opt_var_name}/**/*{opt_var_name}*{grid_label}*.nc' + ) + cmor_output_files = glob.glob(cmor_output_glob, recursive=True) + assert len(cmor_output_files) >= 1, ( + f'no CMOR output found matching {cmor_output_glob}' + ) + assert Path(cmor_output_files[0]).exists() diff --git a/fremorizer/tests/test_cmor_run_subtool_further_examples.py b/fremorizer/tests/test_cmor_run_subtool_further_examples.py index bf5a2c6..342fd84 100644 --- a/fremorizer/tests/test_cmor_run_subtool_further_examples.py +++ b/fremorizer/tests/test_cmor_run_subtool_further_examples.py @@ -1,196 +1,185 @@ ''' -expanded set of tests for fremor run focus on cases beyond test_cmor_run_subtool.py +Expanded set of tests for fremor run — cases beyond test_cmor_run_subtool.py. + +These tests exercise cmor_run_subtool against a variety of variables, tables, +and grid labels drawn from a mock pp-archive. + +.. tip:: pytest temp directories + By default pytest removes temp directories after the session. To keep + them around for debugging, run:: + + pytest --basetemp=/tmp/fremorizer-debug -k test_case_cmip6 -x + + Output files will then persist under ``/tmp/fremorizer-debug``. ''' from datetime import date import glob -import os from pathlib import Path -import shutil -import subprocess import pytest from fremorizer import cmor_run_subtool +from fremorizer.tests.conftest import ncgen -# global consts for these tests, with no/trivial impact on the results -ROOTDIR='fremorizer/tests/test_files' -CMORBITE_VARLIST=f'{ROOTDIR}/CMORbite_var_list.json' +# ── path constants ────────────────────────────────────────────────────────── +ROOTDIR = 'fremorizer/tests/test_files' +CMORBITE_VARLIST = f'{ROOTDIR}/CMORbite_var_list.json' -# cmip6 variable table(s) -CMIP6_TABLE_REPO_PATH = \ - f'{ROOTDIR}/cmip6-cmor-tables' +# cmip6 table repo +CMIP6_TABLE_REPO_PATH = f'{ROOTDIR}/cmip6-cmor-tables' -# outputs -OUTDIR = f'{ROOTDIR}/outdir_ppan_only' -TMPDIR = f'{OUTDIR}/tmp' +# experiment config (materialised by conftest._write_exp_configs) +EXP_CONFIG_CMIP6 = f'{ROOTDIR}/CMOR_input_example.json' # determined by cmor_run_subtool YYYYMMDD = date.today().strftime('%Y%m%d') -CMOR_CREATES_DIR_BASE = \ + +# mock-archive base paths +MOCK_ARCHIVE_ROOT = f'{ROOTDIR}/ascii_files/mock_archive' +ESM4_DECK_PP_DIR = ( + 'cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp' +) +ESM4_DEV_PP_DIR = ( + 'USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC' + '/gfdl.ncrc5-intel23-prod-openmp/pp' +) + +# CMIP6 output dir structure +CMOR_CREATES_DIR_BASE_CMIP6 = ( 'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1' +) -# this file exists basically for users to specify their own information to append to the netcdf file -# i.e., it fills in FOO/BAR/BAZ style values, and what they are currently is totally irrelevant -EXP_CONFIG_DEFAULT=f'{ROOTDIR}/CMOR_input_example.json' # this likely is not sufficient - -CLEANUP_AFTER_EVERY_TEST = False - -def _cleanup(): - # clean up from previous tests - #time.sleep(60) # busy disk issue? possible non-closing netcdf file problem in code - print(OUTDIR) - if Path(f'{OUTDIR}').exists(): - try: - shutil.rmtree(f'{OUTDIR}') - except OSError: - #time.sleep(60) - shutil.rmtree(f'{OUTDIR}') - assert not Path(f'{OUTDIR}').exists() - -MOCK_ARCHIVE_ROOT='fremorizer/tests/test_files/ascii_files/mock_archive' -ESM4_DECK_PP_DIR='cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp' -ESM4_DEV_PP_DIR='USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/pp' -@pytest.mark.parametrize( "testfile_dir,table,opt_var_name,grid_label,start,calendar", - [ - pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/atmos_plev39_cmip/ts/monthly/5yr/zonavg/', - 'AERmonZ', 'ta', 'gr1','1850','noleap', id='AERmonZ_ta_gr1' ), - pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/atmos_scalar/ts/monthly/5yr/', - 'Amon', 'ch4global', 'gr', '1850','noleap', id='Amon_ch4global_gr' ), - pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/LUmip_refined/ts/monthly/5yr/', - 'Emon', 'gppLut', 'gr1','1850','noleap', id='Emon_gppLut_gr1' ), - pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/atmos_level_cmip/ts/monthly/5yr/', - 'Amon', 'cl', 'gr1','1850','noleap', id='Amon_cl_gr1' ), - pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/atmos_level_cmip/ts/monthly/5yr/', - 'Amon', 'mc', 'gr1','1850','noleap', id='Amon_mc_gr1' ), - pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}/ocean_monthly_z_1x1deg/ts/monthly/5yr/', - 'Omon', 'so', 'gr', '0001','noleap', id='Omon_so_gr' ), - pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}/ocean_monthly/ts/monthly/5yr/', - 'Omon', 'sos', 'gn', '0001','noleap', id='Omon_sos_gn' ), - pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}/land/ts/monthly/5yr/', - 'Lmon', 'lai', 'gr1','0001','noleap', id='Lmon_lai_gr1' ), - ] ) - -def test_case_function(testfile_dir,table,opt_var_name,grid_label,start,calendar,monkeypatch): - ''' - Should be iterating over the test dictionary - ''' - # for native-grid ocean tests, prevent the gold statics lookup from finding - # /archive files so the test uses its own locally-generated statics file +# ── helper: convert CDLs to NC in a test directory ───────────────────────── +def _ncgen_for_case(testfile_dir, opt_var_name): + '''Convert the CDL file(s) for *opt_var_name* to NetCDF-4 inside *testfile_dir*. + + Returns the path to the primary NC file. Also generates ancillary files + (e.g. ``ps.nc`` for ``cl`` / ``mc``, ``ocean_monthly.static.nc`` for native + ocean grid variables). + ''' + cdl_files = glob.glob(f'{testfile_dir}*.{opt_var_name}.cdl') + assert len(cdl_files) >= 1, ( + f'no CDL file found for variable {opt_var_name} in {testfile_dir}' + ) + cdl_file = cdl_files[0] + nc_file = cdl_file.replace('.cdl', '.nc') + ncgen(cdl_file, nc_file) + + # ancillary files required by specific variables + if opt_var_name in ('cl', 'mc'): + ps_cdl = cdl_file.replace(f'{opt_var_name}.cdl', 'ps.cdl') + assert Path(ps_cdl).exists(), f'ps CDL not found: {ps_cdl}' + ncgen(ps_cdl, ps_cdl.replace('.cdl', '.nc')) + + elif opt_var_name == 'sos': + static_cdl = testfile_dir.replace( + 'ts/monthly/5yr/', 'ocean_monthly.static.cdl' + ) + if Path(static_cdl).exists(): + ncgen(static_cdl, static_cdl.replace('.cdl', '.nc')) + + return nc_file + + +# ── CMIP6 parametrized tests ─────────────────────────────────────────────── +@pytest.mark.parametrize( + 'testfile_dir,table,opt_var_name,grid_label,start,calendar', + [ + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/atmos_plev39_cmip/ts/monthly/5yr/zonavg/', + 'AERmonZ', 'ta', 'gr1', '1850', 'noleap', + id='AERmonZ_ta_gr1', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/atmos_scalar/ts/monthly/5yr/', + 'Amon', 'ch4global', 'gr', '1850', 'noleap', + id='Amon_ch4global_gr', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/LUmip_refined/ts/monthly/5yr/', + 'Emon', 'gppLut', 'gr1', '1850', 'noleap', + id='Emon_gppLut_gr1', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/atmos_level_cmip/ts/monthly/5yr/', + 'Amon', 'cl', 'gr1', '1850', 'noleap', + id='Amon_cl_gr1', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}' + '/atmos_level_cmip/ts/monthly/5yr/', + 'Amon', 'mc', 'gr1', '1850', 'noleap', + id='Amon_mc_gr1', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}' + '/ocean_monthly_z_1x1deg/ts/monthly/5yr/', + 'Omon', 'so', 'gr', '0001', 'noleap', + id='Omon_so_gr', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}' + '/ocean_monthly/ts/monthly/5yr/', + 'Omon', 'sos', 'gn', '0001', 'noleap', + id='Omon_sos_gn', + ), + pytest.param( + f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}' + '/land/ts/monthly/5yr/', + 'Lmon', 'lai', 'gr1', '0001', 'noleap', + id='Lmon_lai_gr1', + ), + ], +) +def test_case_cmip6( # pylint: disable=too-many-arguments,too-many-positional-arguments + testfile_dir, table, opt_var_name, grid_label, start, calendar, + tmp_path, monkeypatch, +): + '''Run cmor_run_subtool for a single CMIP6 variable and assert output exists.''' + # native-grid ocean tests: prevent gold statics lookup from finding /archive files if grid_label == 'gn': monkeypatch.setattr( - 'fremorizer.cmor_mixer.find_gold_ocean_statics_file', lambda **kw: None) + 'fremorizer.cmor_mixer.find_gold_ocean_statics_file', + lambda **kw: None, + ) - # define inputs to the cmor run tool - indir = testfile_dir - table_file = f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_{table}.json' + _ncgen_for_case(testfile_dir, opt_var_name) - # execute the test - try: - cdl_input_files=glob.glob(indir+'*.'+opt_var_name+'.cdl') - assert len(cdl_input_files)>=1 - - cdl_input_file=cdl_input_files[0] - assert Path(cdl_input_file).exists() - - nc_input_file=cdl_input_file.replace('.cdl','.nc') - if Path(nc_input_file).exists(): - Path(nc_input_file).unlink() - subprocess.run(['ncgen3','-k','netCDF-4','-o', nc_input_file, cdl_input_file], - check=True) - assert Path(nc_input_file).exists() - - # exception: these files need a ps file to be around, so extra ncgen step for these: - if opt_var_name in [ 'cl', 'mc' ]: - cdl_input_ps_file = cdl_input_file.replace( opt_var_name+'.cdl', 'ps.cdl') - assert Path(cdl_input_ps_file).exists() - - nc_input_ps_file = cdl_input_ps_file.replace('.cdl','.nc') - if Path(nc_input_ps_file).exists(): - Path(nc_input_ps_file).unlink() - subprocess.run(['ncgen3','-k','netCDF-4','-o', nc_input_ps_file, cdl_input_ps_file], - check=True) - assert Path(nc_input_ps_file).exists() - - elif opt_var_name == 'sos': - cdl_ocn_statics_file=testfile_dir.replace('ts/monthly/5yr/','ocean_monthly.static.cdl') - assert Path(cdl_ocn_statics_file).exists() - - nc_ocn_statics_file=cdl_ocn_statics_file.replace('.cdl','.nc') - if Path(nc_ocn_statics_file).exists(): - Path(nc_ocn_statics_file).unlink() - subprocess.run(['ncgen3','-k','netCDF-4','-o', nc_ocn_statics_file, cdl_ocn_statics_file], - check=True) - assert Path(nc_ocn_statics_file).exists() - - ##assert False - ## Debug, please keep. -Ian - #print( - #f'fre -vv cmor run \\\n' - #f' -d {indir} \\\n' - #f' -l {CMORBITE_VARLIST} \\\n' - #f' -r {table_file} \\\n' - #f' -p {EXP_CONFIG_DEFAULT} \\\n' - #f' -o {OUTDIR} \\\n' - #f' --run_one \\\n' - #f' -v {opt_var_name} \\\n' - #f' --grid_desc \'FOO_PLACEHOLDER\' \\\n' - #f' -g {grid_label} \\\n' - #f' --nom_res \'10000 km\' \\\n' - #f' --start {start} \\\n' - #f' --calendar {calendar}\n' - #f'') - #assert False - cmor_run_subtool( - indir = indir, - json_var_list = CMORBITE_VARLIST, - json_table_config = table_file, - json_exp_config = EXP_CONFIG_DEFAULT, - outdir = OUTDIR, - run_one_mode = True, - opt_var_name = opt_var_name, - grid = 'FOO_PLACEHOLDER', - grid_label = grid_label, - nom_res = '10000 km' ,# placeholder - start = start, - calendar_type=calendar - ) - #assert False - some_return = 0 - except Exception as exc: - raise Exception(f'exception caught: exc=\n{exc}') from exc - - # outputs that should be created - cmor_output_dir = f'{OUTDIR}/{CMOR_CREATES_DIR_BASE}/{table}/{opt_var_name}/{grid_label}/v{YYYYMMDD}' - cmor_output_file_glob = f'{cmor_output_dir}/' + \ - f'{opt_var_name}_{table}_PCMDI-test-1-0_piControl-withism_r3i1p1f1_{grid_label}_??????-??????.nc' - cmor_output_file = glob.glob( cmor_output_file_glob )[0] - - # success criteria - assert all( [ some_return == 0, - Path(cmor_output_dir).exists(), - Path(cmor_output_file).exists() ] ) - - if CLEANUP_AFTER_EVERY_TEST: - _cleanup() - -def test_git_cleanup(): - ''' - Performs a git restore on EXP_CONFIG to avoid false positives from - git's record of changed files. It's supposed to change as part of the test. - ''' - is_ci = os.environ.get("GITHUB_WORKSPACE") is not None - if not is_ci: - git_cmd = f"git restore {EXP_CONFIG_DEFAULT}" - restore = subprocess.run(git_cmd, - shell=True, - check=False) - check_cmd = f"git status | grep {EXP_CONFIG_DEFAULT}" - check = subprocess.run(check_cmd, - shell = True, - check = False) - #first command completed, second found no file in git status - assert all( [ restore.returncode == 0, - check.returncode == 1 ] ) + table_file = f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_{table}.json' + outdir = str(tmp_path / 'outdir') + + cmor_run_subtool( + indir=testfile_dir, + json_var_list=CMORBITE_VARLIST, + json_table_config=table_file, + json_exp_config=EXP_CONFIG_CMIP6, + outdir=outdir, + run_one_mode=True, + opt_var_name=opt_var_name, + grid='FOO_PLACEHOLDER', + grid_label=grid_label, + nom_res='10000 km', + start=start, + calendar_type=calendar, + ) + + cmor_output_dir = ( + f'{outdir}/{CMOR_CREATES_DIR_BASE_CMIP6}' + f'/{table}/{opt_var_name}/{grid_label}/v{YYYYMMDD}' + ) + cmor_output_glob = ( + f'{cmor_output_dir}/{opt_var_name}_{table}_PCMDI-test-1-0' + f'_piControl-withism_r3i1p1f1_{grid_label}_??????-??????.nc' + ) + cmor_output_files = glob.glob(cmor_output_glob) + assert len(cmor_output_files) >= 1, ( + f'no CMOR output found matching {cmor_output_glob}' + ) + assert Path(cmor_output_files[0]).exists() diff --git a/fremorizer/tests/test_cmor_yamler_freq_validation.py b/fremorizer/tests/test_cmor_yamler_freq_validation.py index 92616a6..e39aa27 100644 --- a/fremorizer/tests/test_cmor_yamler_freq_validation.py +++ b/fremorizer/tests/test_cmor_yamler_freq_validation.py @@ -1,6 +1,6 @@ -''' +""" tests for fremorizer.cmor_helpers.conv_mip_to_bronx_freq -''' +""" import pytest @@ -12,38 +12,38 @@ def test_conv_mip_to_bronx_freq_valid_frequencies(): """ # Test cases from the mapping dictionary test_cases = [ - ("1hr", "1hr"), - ("1hrCM", None), - ("1hrPt", None), - ("3hr", "3hr"), - ("3hrPt", None), - ("6hr", "6hr"), - ("6hrPt", None), - ("day", "daily"), - ("dec", None), - ("fx", None), - ("mon", "monthly"), - ("monC", None), - ("monPt", None), - ("subhrPt", None), - ("yr", "annual"), - ("yrPt", None) # Should return None according to mapping + ('1hr', '1hr'), + ('1hrCM', None), + ('1hrPt', None), + ('3hr', '3hr'), + ('3hrPt', None), + ('6hr', '6hr'), + ('6hrPt', None), + ('day', 'daily'), + ('dec', None), + ('fx', None), + ('mon', 'monthly'), + ('monC', None), + ('monPt', None), + ('subhrPt', None), + ('yr', 'annual'), + ('yrPt', None) # Should return None according to mapping ] for cmor_freq, expected_bronx_freq in test_cases: result = conv_mip_to_bronx_freq(cmor_freq) - assert result == expected_bronx_freq, f"Failed for {cmor_freq}: expected {expected_bronx_freq}, got {result}" + assert result == expected_bronx_freq, f'Failed for {cmor_freq}: expected {expected_bronx_freq}, got {result}' def test_conv_mip_to_bronx_freq_invalid_frequency(): """ Test that invalid frequencies (not 'fx') raise KeyError. """ # Arrange - invalid_frequencies = ["invalid", "unknown", "bad_freq", "yearly", "weekly"] + invalid_frequencies = ['invalid', 'unknown', 'bad_freq', 'yearly', 'weekly'] for invalid_freq in invalid_frequencies: # Act & Assert - with pytest.raises(KeyError, match=f'MIP table frequency = "{invalid_freq}" is not a valid MIP frequency'): + with pytest.raises(KeyError, match=f'MIP table frequency = {invalid_freq} is not a valid MIP frequency'): conv_mip_to_bronx_freq(invalid_freq) def test_conv_mip_to_bronx_freq_edge_cases(): @@ -52,7 +52,7 @@ def test_conv_mip_to_bronx_freq_edge_cases(): """ # Test empty string with pytest.raises(KeyError): - conv_mip_to_bronx_freq("") + conv_mip_to_bronx_freq('') # Test None input - should raise KeyError with pytest.raises(KeyError): @@ -63,7 +63,7 @@ def test_conv_mip_to_bronx_freq_case_sensitivity(): Test that the function is case-sensitive. """ # These should raise KeyError because they're not exact matches - case_variants = ["1HR", "Mon", "DAY", "YR"] + case_variants = ['1HR', 'Mon', 'DAY', 'YR'] for variant in case_variants: with pytest.raises(KeyError): diff --git a/fremorizer/tests/test_cmor_yamler_subtool.py b/fremorizer/tests/test_cmor_yamler_subtool.py index 60b982a..69396e4 100644 --- a/fremorizer/tests/test_cmor_yamler_subtool.py +++ b/fremorizer/tests/test_cmor_yamler_subtool.py @@ -1,10 +1,10 @@ -''' +""" tests for fremorizer.cmor_yamler.cmor_yaml_subtool Covers: - full end-to-end run (dry_run_mode=False) via mocked consolidate_yamls - every documented exception path in the function -''' +""" import json import shutil @@ -41,7 +41,7 @@ def _build_cmor_dict(*, pp_dir, table_dir, outdir, exp_config, chunk='P5Y', data_series_type='ts', gridding=None, start='1993', stop='1993', calendar_type='julian'): - '''Build the dictionary that consolidate_yamls would return.''' + """Build the dictionary that consolidate_yamls would return.""" if gridding is None: gridding = { 'grid_label': GRID_LABEL, @@ -81,10 +81,10 @@ def _build_cmor_dict(*, pp_dir, table_dir, outdir, exp_config, @pytest.fixture def yamler_env(tmp_path): - ''' + """ Set up a temporary pp directory tree and output directory that cmor_yaml_subtool can use for a real (non-dry-run) CMIP6 Omon/sos test. - ''' + """ component = 'ocean_monthly_1x1deg' freq = 'monthly' chunk_bronx = '5yr' @@ -129,10 +129,10 @@ def yamler_env(tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_cmor_yaml_subtool_dry_run_false(mock_consolidate, yamler_env): # pylint: disable=redefined-outer-name - ''' + """ Full end-to-end: cmor_yaml_subtool with dry_run_mode=False should call cmor_run_subtool and produce at least one CMOR-ised .nc file. - ''' + """ mock_consolidate.return_value = _build_cmor_dict( pp_dir=yamler_env['pp_dir'], table_dir=yamler_env['table_dir'], outdir=yamler_env['outdir'], @@ -162,7 +162,7 @@ def test_cmor_yaml_subtool_dry_run_false(mock_consolidate, yamler_env): # pylint # ================================================================ def test_yamlfile_does_not_exist(): - ''' FileNotFoundError when yamlfile path does not exist ''' + """ FileNotFoundError when yamlfile path does not exist """ with pytest.raises(FileNotFoundError): cmor_yaml_subtool( yamlfile='DOES_NOT_EXIST.yaml', @@ -172,7 +172,7 @@ def test_yamlfile_does_not_exist(): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_pp_dir_does_not_exist(mock_consolidate, tmp_path): - ''' FileNotFoundError when pp_dir does not exist ''' + """ FileNotFoundError when pp_dir does not exist """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -197,7 +197,7 @@ def test_pp_dir_does_not_exist(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_table_dir_does_not_exist(mock_consolidate, tmp_path): - ''' FileNotFoundError when cmip_cmor_table_dir does not exist ''' + """ FileNotFoundError when cmip_cmor_table_dir does not exist """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -224,7 +224,7 @@ def test_table_dir_does_not_exist(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_exp_json_does_not_exist(mock_consolidate, tmp_path): - ''' FileNotFoundError when exp_json path does not exist ''' + """ FileNotFoundError when exp_json path does not exist """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') pp_dir = tmp_path / 'pp' @@ -249,7 +249,7 @@ def test_exp_json_does_not_exist(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_mip_table_file_does_not_exist(mock_consolidate, tmp_path): - ''' FileNotFoundError when the derived json_mip_table_config does not exist ''' + """ FileNotFoundError when the derived json_mip_table_config does not exist """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -278,7 +278,7 @@ def test_mip_table_file_does_not_exist(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_cmip7_freq_none_raises(mock_consolidate, tmp_path): - ''' ValueError when mip_era=CMIP7 and freq is None ''' + """ ValueError when mip_era=CMIP7 and freq is None """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -310,10 +310,10 @@ def test_cmip7_freq_none_raises(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_cmip6_freq_none_no_derivation_raises(mock_consolidate, tmp_path): - ''' + """ ValueError when mip_era=CMIP6, freq is None, and the MIP table frequency cannot be derived (e.g. fx table). - ''' + """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -353,13 +353,13 @@ def test_cmip6_freq_none_no_derivation_raises(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_cmip6_freq_none_derivation_exception_caught(mock_consolidate, tmp_path): - ''' + """ When mip_era=CMIP6, freq is None, and get_bronx_freq_from_mip_table raises a KeyError (e.g. the MIP table JSON has no variable_entry key), the except (KeyError, TypeError) branch catches it, sets freq = None, and the subsequent check raises ValueError. Covers the except branch around get_bronx_freq_from_mip_table. - ''' + """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -398,7 +398,7 @@ def test_cmip6_freq_none_derivation_exception_caught(mock_consolidate, tmp_path) @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_gridding_dict_has_none_value_raises(mock_consolidate, tmp_path): - ''' ValueError when a gridding field is None ''' + """ ValueError when a gridding field is None """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -430,11 +430,11 @@ def test_gridding_dict_has_none_value_raises(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_outdir_creation_when_missing(mock_consolidate, tmp_path): - ''' + """ When cmorized_outdir does not exist, the function should create it (rather than raising). Verify with a dry-run so we only test the path-creation logic without running CMOR. - ''' + """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -462,10 +462,10 @@ def test_outdir_creation_when_missing(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_outdir_creation_failure_raises_oserror(mock_consolidate, tmp_path): - ''' + """ OSError when cmorized_outdir does not exist and Path.mkdir fails. Covers the except branch in the outdir-creation block. - ''' + """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -494,11 +494,11 @@ def test_outdir_creation_failure_raises_oserror(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_start_stop_calendar_missing_from_yaml(mock_consolidate, tmp_path): - ''' + """ When start, stop, and calendar_type are None on the CLI AND absent from the YAML dict, the function should log warnings and continue (dry-run mode). Covers the KeyError branches for start/stop/calendar_type. - ''' + """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -534,12 +534,12 @@ def test_start_stop_calendar_missing_from_yaml(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_cmip6_freq_none_derivation_succeeds(mock_consolidate, tmp_path): - ''' + """ When mip_era=CMIP6 and freq is None, but the MIP table carries a derivable frequency (e.g. Omon → "mon" → "monthly"), the function should successfully derive freq and continue (dry-run mode). Covers the successful get_bronx_freq_from_mip_table path. - ''' + """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -569,10 +569,10 @@ def test_cmip6_freq_none_derivation_succeeds(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_dry_run_prints_cli_call(mock_consolidate, tmp_path): - ''' + """ dry_run_mode=True with print_cli_call=True should log the CLI invocation and never call cmor_run_subtool. - ''' + """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -604,10 +604,10 @@ def test_dry_run_prints_cli_call(mock_consolidate, tmp_path): @patch('fremorizer.cmor_yamler.consolidate_yamls') def test_dry_run_prints_python_call(mock_consolidate, tmp_path): - ''' + """ dry_run_mode=True with print_cli_call=False should log the Python cmor_run_subtool(...) invocation. - ''' + """ dummy_yaml = tmp_path / 'model.yaml' dummy_yaml.write_text('placeholder') local_exp = tmp_path / 'exp.json' @@ -635,12 +635,12 @@ def test_dry_run_prints_python_call(mock_consolidate, tmp_path): assert len(output_nc) == 0 def test_modelyaml_dne_raise_filenotfound(): - ''' + """ tests that a FileNotFoundError is raised when the model yaml does not exist - ''' + """ with pytest.raises(FileNotFoundError): - cmor_yaml_subtool( yamlfile = "MODEL YAML DOESN'T EXIST", - exp_name = "FOO", - platform = "BAR", - target = "BAZ", + cmor_yaml_subtool( yamlfile = 'MODEL YAML DOESN\'T EXIST', + exp_name = 'FOO', + platform = 'BAR', + target = 'BAZ', dry_run_mode = True ) diff --git a/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json b/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json deleted file mode 100644 index 9adab5f..0000000 --- a/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "#_TESTING_ONLY": " ***** This is for unit-test functionality of NOAA-GDFL's fremorizer module for CMIP7, they do not reflect values used in actual production *****", - "contact ": "MIP participant mipmember@foobar.c.om", - "comment": "additional important information not fitting into other fields can be placed here", - "license_id": "CC-BY-4.0", - "license_url": "https://creativecommons.org/licenses/by/4.0", - "license": "; CMIP7 data produced by is licensed under a License (). Consult https://wcrp-cmip.github.io/cmip7-guidance/docs/CMIP7/Guidance_for_users/#2-terms-of-use-and-citations-requirements for terms of use governing CMIP7 output, including citation requirements and proper acknowledgment. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", - "#_TRACKING": "***** anything to do with citing this data, accreditation, licensing, and references go here *****", - "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html. The ssp245 simulation is described in Dorkey et al. '(Clim. Dyn., 2003, 323-357.)'", - "drs_specs": "MIP-DRS7", - "archive_id": "WCRP", - "tracking_prefix": "hdl:21.14107", - "#_MIP_DETAILS": "***** anything to do with identifying the specific MIP activity this configuration file is for *****", - "_cmip7_option": 1, - "mip_era": "CMIP7", - "parent_mip_era": "CMIP7", - "activity_id": "CMIP", - "parent_activity_id": "CMIP", - "#_SOURCE_SECTION": "***** anything to do with identifying this experiment, it's relationships to other experiments, the producers, and their institution *****", - "institution": "", - "institution_id": "CCCma", - "source": "DUMMY-MODEL: aerosol: Dummy Aerosol; atmosphere: Dummy Atmosphere; atmospheric_chemistry: Dummy Atmospheric Chemistry; land_surface: Dummy Land Surface; ocean: Dummy Ocean; ocean_biogeochemistry: Dummy Ocean Biogeochemistry; sea_ice: Dummy Sea Ice", - "parent_source_id": "DUMMY-MODEL", - "source_id": "DUMMY-MODEL", - "source_type": "AOGCM ISM AER", - "parent_experiment_id": "piControl", - "experiment_id": "historical", - "sub_experiment": "none", - "sub_experiment_id": "none", - "#_INDICES": "***** changed from ints to strings for CMIP7 *****", - "realization_index": "r3", - "initialization_index": "i1", - "physics_index": "p1", - "forcing_index": "f3", - "run_variant": "3rd realization", - "parent_variant_label": "r3i1p1f3", - "#_TEMPORAL_INFO": "***** anything to do with describing temporal aspects of the experiment *****", - "parent_time_units": "days since 1850-01-01", - "branch_method": "no parent", - "branch_time_in_child": 59400.0, - "branch_time_in_parent": 0.0, - "calendar": "julian", - "#_SPATIAL_INFO": "***** anything to do with describing physical aspects of the experiment *****", - "grid": "FOO_BAR_PLACEHOLD", - "grid_label": "g999", - "frequency": "mon", - "region": "glb", - "nominal_resolution": "10000 km", - "#_HISTORY_METADATA": "history attribute string and template, to create history field for output file", - "history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.", - "_history_template": "%s ;rewrote data to be consistent with for variable found in table .", - "#_OUTPUT_PATHS": "***** pathing/templates for output files *****", - "#_output_template_NOTE": "***** PCMDI/cmor 4e7f1f3d731077b7f65c188edefac924cc3e2779 Test/test_cmor_CMIP7.py L47 *****", - "#_output": "***** Root directory for output (can be either a relative or full path) *****", - "outpath": ".", - "#_output_path_template": "***** Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax *****", - "output_path_template": "", - "#_output_file_template": "***** Template for output filename using tables keys or global attributes, these should follow the relevant data reference syntax *****", - "output_file_template": "", - "#_INPUT_CONFIG_PATHS": "***** pathing/templates for input configuration files holding controlled vocabularies *****", - "_controlled_vocabulary_file": "../tables-cvs/cmor-cvs.json", - "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", - "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json" -} diff --git a/fremorizer/tests/test_files/CMOR_input_example.json b/fremorizer/tests/test_files/CMOR_input_example.json deleted file mode 100644 index dc40b81..0000000 --- a/fremorizer/tests/test_files/CMOR_input_example.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "#note": " **** The following are set correctly for CMIP6 and should not normally need editing", - "source_type": "AOGCM ISM AER", - "experiment_id": "piControl-withism", - "activity_id": "ISMIP6", - "sub_experiment_id": "none", - "realization_index": "3", - "initialization_index": "1", - "physics_index": "1", - "forcing_index": "1", - "run_variant": "3rd realization", - "parent_experiment_id": "no parent", - "parent_activity_id": "no parent", - "parent_source_id": "no parent", - "parent_variant_label": "no parent", - "parent_time_units": "no parent", - "branch_method": "no parent", - "branch_time_in_child": 59400.0, - "branch_time_in_parent": 0.0, - "institution_id": "PCMDI", - "source_id": "PCMDI-test-1-0", - "calendar": "julian", - "grid": "FOO_BAR_PLACEHOLD", - "grid_label": "gr", - "nominal_resolution": "10000 km", - "license": "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", - "#output": "Root directory for output (can be either a relative or full path)", - "outpath": "CMIP6", - "contact ": "Python Coder (coder@a.b.c.com)", - "history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.", - "comment": "", - "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html. The ssp245 simulation is described in Dorkey et al. '(Clim. Dyn., 2003, 323-357.)'", - "sub_experiment": "none", - "institution": "", - "source": "PCMDI-test 1.0 (1989)", - "_controlled_vocabulary_file": "CMIP6_CV.json", - "_AXIS_ENTRY_FILE": "CMIP6_coordinate.json", - "_FORMULA_VAR_FILE": "CMIP6_formula_terms.json", - "_cmip6_option": "CMIP6", - "mip_era": "CMIP6", - "parent_mip_era": "no parent", - "tracking_prefix": "hdl:21.14100", - "_history_template": "%s ;rewrote data to be consistent with for variable found in table .", - "#output_path_template": "Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax", - "output_path_template": "<_member_id>
", - "output_file_template": "
<_member_id>" -} \ No newline at end of file diff --git a/pylintrc b/pylintrc index ec7d727..027b461 100644 --- a/pylintrc +++ b/pylintrc @@ -39,7 +39,7 @@ extension-pkg-whitelist= fail-on= # Specify a score threshold under which the program will exit with error. -fail-under=9.6 +fail-under=9.7 # Interpret the stdin as a python script, whose filename needs to be passed as # the module_or_package argument. @@ -553,7 +553,7 @@ spelling-store-unknown-words=no # This flag controls whether inconsistent-quotes generates a warning when the # character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=no +check-quote-consistency=yes # This flag controls whether the implicit-str-concat should generate a warning # on implicit string concatenation in sequences defined over several lines.