diff --git a/.gitmodules b/.gitmodules index d4c20ce..0203da1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,4 @@ [submodule "fremorizer/tests/test_files/cmip7-cmor-tables"] path = fremorizer/tests/test_files/cmip7-cmor-tables url = https://github.com/WCRP-CMIP/cmip7-cmor-tables.git + branch = DR-1.2.2.3-v1.0.5 diff --git a/README.md b/README.md index f6467de..ff3ce16 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # `fremorizer` +`fremorizer` CMORizes FRE output with `CMOR`. It is a `conda` package and it's documentation can be found on +[`readthedocs`](https://fremorizer.readthedocs.io/en/latest/). + [![Anaconda-Server Badge](https://anaconda.org/ilaflott/fremorizer/badges/version.svg)](https://anaconda.org/ilaflott/fremorizer) [![Anaconda-Server Badge](https://anaconda.org/ilaflott/fremorizer/badges/latest_release_date.svg)](https://anaconda.org/ilaflott/fremorizer) [![Anaconda-Server Badge](https://anaconda.org/ilaflott/fremorizer/badges/latest_release_relative_date.svg)](https://anaconda.org/ilaflott/fremorizer) @@ -11,83 +14,98 @@ [![codecov](https://codecov.io/gh/ilaflott/fremorizer/branch/main/graph/badge.svg)](https://codecov.io/gh/ilaflott/fremorizer) -Simply put, `fremorizer` CMORizes FRE output with `CMOR`. +## Background and Purpose +`fremorizer` is a model output rewriter (CMORizer) for FRE/FMS based models and output. It was originally the `fre.cmor` +submodule of [`NOAA-GFDL/fre-cli`](https://github.com/NOAA-GFDL/fre-cli). `fremorizer` (or `fremor` for short) is geared +for rewriting NOAA-GFDL datasets for further quality control checks, assessments and data publishing pipelines in the +context of CMIP7 using the [`CMOR`](https://cmor.llnl.gov/) library. -`fremorizer` is a `conda` package, it's documentation can be found on [`readthedocs`](https://fremorizer.readthedocs.io/en/latest/). +## Installation / Access -## Background and Purpose -`fremorizer` is a model output rewriter (CMORizer) for FRE/FMS based models and output. It was originally the `fre.cmor` submodule of -[`NOAA-GFDL/fre-cli`](https://github.com/NOAA-GFDL/fre-cli). `fremorizer` (or `fremor` for short) is geared for rewriting NOAA-GFDL -datasets for further quality control checks, assessments and data publishing pipelines in the context of CMIP7 using the -[`CMOR`](https://cmor.llnl.gov/) library. +### Requirements +- `python>=3.11` +- `click>=8.2` +- `cmor>=3.14.2` +- `netCDF4>=1.7` +- `numpy>=2` +- `pyyaml` +For development and testing, `pylint`, `pytest`, and `pytest-cov` are all highly recommended as helpful additions. -## Installation / Access ### via PPAN / modules (COMING SOON/TODO) +If you're trying to gain access to `fremor` functionality as quickly as possible: ```bash -# this will be whatever is in main of this repository (COMING SOON/TODO) +# the current post-release in main module load fremorizer/test -# this will be a tagged version of the package in this repository -module load fremorizer/YYYY.XX.[ZZ]{-alpha,-beta} +# a tagged version of fremorizer, post-releases will never be named modules +module load fremorizer/X.Y.Z ``` -### From source/checkout into a virtual environment (`conda` or `venv`) with `pip` -If you're trying to develop the package, or edit the code for contributing in either a big or small way, this is for you. If you're just -trying to use `fremorizer` and you want to deal with as few technical details as possible, *this is not for you*. -Add `-e` to the `pip` step for an editable install. `--recursive` pulls in required configuration in the form of CMIP CMOR tables, and -can be omitted if desired at the cost of local testing functionality out-of-box +### via `conda` +If you have a path to a `fremorizer` environment you can activate it like so: ```bash -# this does work, right now -git clone --recursive https://github.com/ilaflott/fremorizer.git -cd fremorizer -conda env create -f environment.yaml -pip install . +conda activate some/path/to/fremorizer_env ``` -### Via `conda` (COMING SOON) -If you just want an environment named `fremorizer` with the package: +If you want your own `fremorizer` environment: ```bash -# does not work yet, TODO -conda create -n fremorizer noaa-gfdl::fremorizer -conda activate fremorizer +# the environment will be named fremorizer_en +conda create -n fremorizer_env ilaflott::fremorizer + +# see fremorizer_env in the list --> activate it by name +conda env list +conda activate fremorizer_env ``` or, if you've already activated a `conda` environment ```bash -# does not work yet, TODO -conda install -c noaa-gfdl fremorizer +conda create -n empty_env +conda activate empty_env +conda install -c ilaflott fremorizer -# does not work yet, TODO # equivalent syntax -conda install noaa-gfdl::fremorizer +conda install ilaflott::fremorizer ``` -## Usage -### as a command line interface (CLI) -The CLI entry point is `fremor`. It maps directly from the `fre cmor` subcommand: +### `pip` install source/checkout into a virtual environment (`conda`/`venv`) +If you're trying to develop `fremorizer` capabilities, or edit the code to your liking in either a big or small way, +**this is for you**. This checks out the code, creates and activates an environment, installs into the environment, +and runs all unit-tests and `pylint` checks: ```bash -# past fre-cli command -fre -vv -l logfile.txt cmor [OPTIONS] +# omit --recursive if you don't want tables as submodules +git clone --recursive https://github.com/ilaflott/fremorizer.git +cd fremorizer -# fremorizer equivalent -fremor -vv -l logfile.txt [OPTIONS] +# create an environment and install the local checkout +conda env create -f environment.yaml +conda actiavte fremorizer +pip install -e . + +# Run tests +pytest fremorizer/tests/ + +# Run linter +pylint --rcfile pylintrc fremorizer/ ``` -### as a `python` module -Each CLI subcommand (`run`, `yaml`, etc.) maps to an API under under `fremor`, so the CLI functionality -is equivalently available via `import` in scripts as a proper `python` module -### Subcommands +## Usage + + +### as a command line interface (CLI) +The CLI entry point is `fremor`, currently a suite of (currently) six routines for facillitating data preparation for +CMIP7. ```bash +# The full list of subcommands fremor init # Initialize CMOR configuration resources: generate template user config, fetch tables fremor find # Find and print variables in MIP tables according to your variable lists or other input fremor varlist # Create a simple variable list of netCDF files in a directory @@ -96,85 +114,40 @@ fremor yaml # Bulk routine for processing data based on a CMOR YAML config, fremor run # Lowest-level routine, no CMOR YAML needed, rewrites output files in a directory with CMOR ``` -For a concise overview of required inputs and sample commands, see the -[CMOR Quickstart](docs/quickstart.rst). - -### Getting Started: Initialize CMOR Resources - -Before CMORizing data, you need an experiment configuration template and MIP tables. -The `fremor init` command helps set up these resources: - +The CLI offers full logging and verbosity control independent of the command chosen: ```bash -# Generate a CMIP6 experiment config template and fetch CMIP6 tables -fremor init -m cmip6 -e exp_config.json -t cmip6-tables - -# Generate a CMIP7 experiment config template and fetch CMIP7 tables (fast mode) -fremor init -m cmip7 -e exp_config.json -t cmip7-tables --fast - -# Fetch tables for a specific release tag -fremor init -m cmip6 -t cmip6-tables --tag 6.9.33 +# verbosity and logging +fremor -v ... # INFO level logging +fremor -vv ... # DEBUG level logging +fremor -q ... # ERROR level only (quiet) +fremor -l mylog.txt ... # Log to file (appends) ``` -This command: -- Generates an experiment configuration JSON template with required CMIP metadata fields -- Fetches official MIP tables from trusted GitHub repositories (CMIP6: `pcmdi/cmip6-cmor-tables`, CMIP7: `WCRP-CMIP/cmip7-cmor-tables`) -- Supports both `git clone` (default) and `curl` (`--fast`) methods for downloading official configurations - -### Verbosity and Logging - +If you've used the previous `fre cmor` command, there is a direct mapping of syntax: ```bash -fremor -v run ... # INFO level logging -fremor -vv run ... # DEBUG level logging -fremor -q run ... # ERROR level only (quiet) -fremor -l log.txt run ... # Log to file (appends) -``` - -### Example: CMORize ocean data +# past fre-cli command +fre -vv -l logfile.txt cmor [OPTIONS] -```bash -fremor run \ - -d /path/to/input/netcdf/dir \ - -l /path/to/varlist.json \ - -r /path/to/CMIP6_Omon.json \ - -p /path/to/exp_config.json \ - -o /path/to/output/dir +# fremorizer equivalent +fremor -vv -l logfile.txt [OPTIONS] ``` -## Requirements - -- `python>=3.11` -- `click>=8.2` -- `cmor>=3.14.2` -- `netCDF4>=1.7` -- `numpy>=2` -- `pyyaml` -For development and testing, `pylint`, `pytest`, and `pytest-cov` are all highly recommended as helpful additions. +### as a `python` module +Each CLI subcommand (`run`, `yaml`, etc.) maps to an API under under `fremor`, so the CLI functionality +is equivalently available via `import` in scripts as a proper `python` module -## Development -```bash -# Checkout code -git clone --recursive https://github.com/ilaflott/fremorizer.git -cd fremorizer -# Create a conda environment -conda env create -f environment.yaml -conda activate fremorizer +## Getting started +For an overview of required inputs and sample commands, see the [CMOR Quickstart](docs/quickstart.rst). -# Install in editable mode -pip install -e . -# Run tests -pytest fremorizer/tests/ -# Run linter -pylint --rcfile pylintrc fremorizer/ -``` +## CI/CD Workflows and QA -## Quality Assurance -### WCRP Compliance Checking +### WCRP Compliance Checking (under development) [![wcrp_compliance_check](https://github.com/ilaflott/fremorizer/actions/workflows/wcrp_compliance_check.yml/badge.svg?branch=main)](https://github.com/ilaflott/fremorizer/actions/workflows/wcrp_compliance_check.yml) @@ -193,28 +166,27 @@ To view compliance results from a workflow/CI run: 2. Select the `wcrp_compliance_check` workflow run 3. Download the `wcrp-compliance-reports` artifact -### pipeline badges + +### `conda` environment tests | Python 3.11 | Python 3.12 | Python 3.13 | Python 3.14 | |-------------|-------------|-------------|-------------| | [![3.11](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.11) | [![3.12](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.12) | [![3.13](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.13) | [![3.14](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/ilaflott/fremorizer/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.14) | -## License + +## License Apache License 2.0 — see [LICENSE.md](LICENSE.md) -## Conda-forge feedstock -See `CONDA_FORGE_FEEDSTOCK_PLAN.md` for the steps and follow-up tasks to submit and maintain the conda-forge feedstock for `fremorizer`. ## Releases and Versioning - `fremorizer` uses a post-release scheme to identify development beyond the latest tagged version and tie the current `main` branch to a `conda` package versioned as `develop`. To avoid confusion with `fre-workflows` and `fre-cli`, which often demand that the version tags match, `fremorizer`'s version format is `X.Y.Z[.post]`. -### new published release procedure +### new published release procedure To publish new release carefully follow the below procedure: 1. create a new branch off of `main`, which is already published to `conda` under `develop`/the previous tagged version + `.post` 2. edit the version number in `fremorizer/_version.py` from the current one, to the desired version tag, remove `.post`, then open a PR. edit nothing else (usually). diff --git a/environment.yaml b/environment.yaml index 4d13fba..cae9556 100644 --- a/environment.yaml +++ b/environment.yaml @@ -5,7 +5,7 @@ channels: dependencies: - python>=3.11 - conda-forge::click>=8.2 - - conda-forge::cmor>=3.14 + - conda-forge::cmor>=3.14.2 - conda-forge::netcdf4>=1.7 - conda-forge::numpy>=2 - conda-forge::pyyaml diff --git a/fremorizer/cli.py b/fremorizer/cli.py index 984e7c1..594bb46 100644 --- a/fremorizer/cli.py +++ b/fremorizer/cli.py @@ -271,8 +271,8 @@ def varlist(dir_targ, output_variable_list, mip_table): help="Temporal frequency string, e.g. 'monthly', 'daily'. Default 'monthly'.") @click.option("--chunk", type=str, default="5yr", help="Time chunk string, e.g. '5yr', '10yr'. Default '5yr'.") -@click.option("--grid", type=str, default="g99", - help="Grid label anchor name, e.g. 'g99', 'gn'. Default 'g99'.") +@click.option("--grid", type=str, default="g999", + help="Grid label anchor name, e.g. 'g999', 'gn'. Default 'g999'.") @click.option("--overwrite", is_flag=True, default=False, help="Overwrite existing variable list files.") @click.option("--calendar", type=str, default="noleap", diff --git a/fremorizer/cmor_config.py b/fremorizer/cmor_config.py index 633a5e2..952347a 100644 --- a/fremorizer/cmor_config.py +++ b/fremorizer/cmor_config.py @@ -61,7 +61,7 @@ def cmor_config_subtool( varlist_dir: str, freq: str = 'monthly', chunk: str = '5yr', - grid: str = 'g99', + grid: str = 'g999', overwrite: bool = False, calendar_type: str = 'noleap' ): @@ -90,7 +90,7 @@ def cmor_config_subtool( :type freq: str :param chunk: Time chunk string, e.g. '5yr', '10yr'. Default '5yr'. :type chunk: str - :param grid: Grid label anchor name, e.g. 'g99', 'gn'. Default 'g99'. + :param grid: Grid label anchor name, e.g. 'g999', 'gn'. Default 'g999'. :type grid: str :param overwrite: If True, overwrite existing variable list files. Default False. :type overwrite: bool diff --git a/fremorizer/cmor_init.py b/fremorizer/cmor_init.py index dde23ff..5352d95 100644 --- a/fremorizer/cmor_init.py +++ b/fremorizer/cmor_init.py @@ -118,9 +118,7 @@ def _cmip7_exp_config_template(): "tracking_prefix": "hdl:21.14107", "_cmip7_option": 1, "mip_era": "CMIP7", - "parent_mip_era": "CMIP7", "activity_id": "CMIP", - "parent_activity_id": "CMIP", "institution": "", "institution_id": "", "source": "", @@ -129,21 +127,17 @@ def _cmip7_exp_config_template(): "experiment_id": "", "sub_experiment": "none", "sub_experiment_id": "none", - "parent_source_id": "", - "parent_experiment_id": "", "realization_index": "r1", "initialization_index": "i1", "physics_index": "p1", "forcing_index": "f1", "run_variant": "", - "parent_variant_label": "", - "parent_time_units": "", "branch_method": "no parent", "branch_time_in_child": 0.0, "branch_time_in_parent": 0.0, "calendar": "", "grid": "PLACEHOLD", - "grid_label": "g99", + "grid_label": "g999", "frequency": "", "region": "", "nominal_resolution": "", diff --git a/fremorizer/tests/test_cli.py b/fremorizer/tests/test_cli.py index 354b3d2..66506a5 100644 --- a/fremorizer/tests/test_cli.py +++ b/fremorizer/tests/test_cli.py @@ -223,7 +223,7 @@ def test_cli_fremor_run_cmip7_case1(cli_sos_nc_file, tmp_path): # pylint: disabl "--exp_config", str(EXP_CONFIG_CMIP7), "--outdir", outdir, "--calendar", "julian", - "--grid_label", "g99", + "--grid_label", "g999", "--grid_desc", "FOO_BAR_PLACEHOLD", "--nom_res", "10000 km" ] ) assert result.exit_code == 0, f'cmip7 case1 failed: {result.output}' @@ -245,7 +245,7 @@ def test_cli_fremor_run_cmip7_case2(cli_sosv2_nc_file, tmp_path): # pylint: disa "--exp_config", str(EXP_CONFIG_CMIP7), "--outdir", outdir, "--calendar", "julian", - "--grid_label", "g99", + "--grid_label", "g999", "--grid_desc", "FOO_BAR_PLACEHOLD", "--nom_res", "10000 km" ] ) assert result.exit_code == 0, f'cmip7 case2 failed: {result.output}' diff --git a/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json b/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json index 20e9679..9adab5f 100644 --- a/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json +++ b/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json @@ -2,12 +2,13 @@ "#_TESTING_ONLY": " ***** This is for unit-test functionality of NOAA-GDFL's fremorizer module for CMIP7, they do not reflect values used in actual production *****", "contact ": "MIP participant mipmember@foobar.c.om", "comment": "additional important information not fitting into other fields can be placed here", + "license_id": "CC-BY-4.0", + "license_url": "https://creativecommons.org/licenses/by/4.0", + "license": "; CMIP7 data produced by is licensed under a License (). Consult https://wcrp-cmip.github.io/cmip7-guidance/docs/CMIP7/Guidance_for_users/#2-terms-of-use-and-citations-requirements for terms of use governing CMIP7 output, including citation requirements and proper acknowledgment. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", "#_TRACKING": "***** anything to do with citing this data, accreditation, licensing, and references go here *****", - "license": "CC-BY-4-0; CMIP7 data produced by CCCma is licensed under a Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0). Consult [TODO terms of use link] for terms of use governing CMIP7 output, including citation requirements and proper acknowledgment. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html. The ssp245 simulation is described in Dorkey et al. '(Clim. Dyn., 2003, 323-357.)'", "drs_specs": "MIP-DRS7", "archive_id": "WCRP", - "license_id": "CC-BY-4-0", "tracking_prefix": "hdl:21.14107", "#_MIP_DETAILS": "***** anything to do with identifying the specific MIP activity this configuration file is for *****", "_cmip7_option": 1, @@ -18,14 +19,14 @@ "#_SOURCE_SECTION": "***** anything to do with identifying this experiment, it's relationships to other experiments, the producers, and their institution *****", "institution": "", "institution_id": "CCCma", - "source": "CanESM6-MR:", - "source_id": "CanESM6-MR", + "source": "DUMMY-MODEL: aerosol: Dummy Aerosol; atmosphere: Dummy Atmosphere; atmospheric_chemistry: Dummy Atmospheric Chemistry; land_surface: Dummy Land Surface; ocean: Dummy Ocean; ocean_biogeochemistry: Dummy Ocean Biogeochemistry; sea_ice: Dummy Sea Ice", + "parent_source_id": "DUMMY-MODEL", + "source_id": "DUMMY-MODEL", "source_type": "AOGCM ISM AER", - "experiment_id": "esm-piControl", + "parent_experiment_id": "piControl", + "experiment_id": "historical", "sub_experiment": "none", "sub_experiment_id": "none", - "parent_source_id": "CanESM6-MR", - "parent_experiment_id": "esm-piControl-spinup", "#_INDICES": "***** changed from ints to strings for CMIP7 *****", "realization_index": "r3", "initialization_index": "i1", @@ -41,7 +42,7 @@ "calendar": "julian", "#_SPATIAL_INFO": "***** anything to do with describing physical aspects of the experiment *****", "grid": "FOO_BAR_PLACEHOLD", - "grid_label": "g99", + "grid_label": "g999", "frequency": "mon", "region": "glb", "nominal_resolution": "10000 km", @@ -53,9 +54,9 @@ "#_output": "***** Root directory for output (can be either a relative or full path) *****", "outpath": ".", "#_output_path_template": "***** Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax *****", - "output_path_template": "", + "output_path_template": "", "#_output_file_template": "***** Template for output filename using tables keys or global attributes, these should follow the relevant data reference syntax *****", - "output_file_template": "[].nc", + "output_file_template": "", "#_INPUT_CONFIG_PATHS": "***** pathing/templates for input configuration files holding controlled vocabularies *****", "_controlled_vocabulary_file": "../tables-cvs/cmor-cvs.json", "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", diff --git a/fremorizer/tests/test_files/CMOR_input_example.json b/fremorizer/tests/test_files/CMOR_input_example.json index 66dfb83..dc40b81 100644 --- a/fremorizer/tests/test_files/CMOR_input_example.json +++ b/fremorizer/tests/test_files/CMOR_input_example.json @@ -19,9 +19,9 @@ "branch_time_in_parent": 0.0, "institution_id": "PCMDI", "source_id": "PCMDI-test-1-0", - "calendar": "noleap", - "grid": "FOO_PLACEHOLDER", - "grid_label": "gr1", + "calendar": "julian", + "grid": "FOO_BAR_PLACEHOLD", + "grid_label": "gr", "nominal_resolution": "10000 km", "license": "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", "#output": "Root directory for output (can be either a relative or full path)", diff --git a/fremorizer/tests/test_files/cmip7-cmor-tables b/fremorizer/tests/test_files/cmip7-cmor-tables index 1a31c4d..fa8d442 160000 --- a/fremorizer/tests/test_files/cmip7-cmor-tables +++ b/fremorizer/tests/test_files/cmip7-cmor-tables @@ -1 +1 @@ -Subproject commit 1a31c4d48d4706e4f37dd39265736587b4ef9c89 +Subproject commit fa8d44268a8466bc51244e0789dfe3dba930f5b7 diff --git a/meta.yaml b/meta.yaml index 89eb61e..1ca69ed 100644 --- a/meta.yaml +++ b/meta.yaml @@ -23,7 +23,7 @@ requirements: run: - python>=3.11 - click>=8.2 - - cmor>=3.14 + - cmor>=3.14.2 - netcdf4>=1.7 - numpy>=2 - pyyaml diff --git a/pyproject.toml b/pyproject.toml index be3cac8..6675a32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,11 +21,11 @@ keywords = [ ] dependencies = [ - 'click', - 'cmor', - 'netCDF4', - 'numpy', - 'pyyaml', + "click", + "cmor", + "netCDF4", + "numpy", + "pyyaml", ] classifiers = [